diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f059a397..cd030d3f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -8,35 +8,8 @@ on: pull_request: branches: [ main ] jobs: - test-docker-images: - if: github.event.pull_request - name: Test Images - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build Alpine image - uses: docker/build-push-action@v5 - with: - context: . - file: ./packaging/docker/alpine/Dockerfile - push: false - tags: ${{ github.repository_owner }}/miniflux:alpine-dev - - name: Test Alpine Docker image - run: docker run --rm ${{ github.repository_owner }}/miniflux:alpine-dev miniflux -i - - name: Build Distroless image - uses: docker/build-push-action@v5 - with: - context: . - file: ./packaging/docker/distroless/Dockerfile - push: false - tags: ${{ github.repository_owner }}/miniflux:distroless-dev - - name: Test Distroless Docker image - run: docker run --rm ${{ github.repository_owner }}/miniflux:distroless-dev miniflux -i - - publish-docker-images: - if: ${{ ! github.event.pull_request }} - name: Publish Images + docker-images: + name: Docker Images permissions: packages: write runs-on: ubuntu-latest @@ -46,33 +19,31 @@ jobs: with: fetch-depth: 0 - - name: Generate Alpine Docker tag - id: docker_alpine_tag - run: | - DOCKER_IMAGE=${{ github.repository_owner }}/miniflux - DOCKER_VERSION=dev - if [ "${{ github.event_name }}" = "schedule" ]; then - DOCKER_VERSION=nightly - TAGS="docker.io/${DOCKER_IMAGE}:${DOCKER_VERSION},ghcr.io/${DOCKER_IMAGE}:${DOCKER_VERSION},quay.io/${DOCKER_IMAGE}:${DOCKER_VERSION}" - elif [[ $GITHUB_REF == refs/tags/* ]]; then - DOCKER_VERSION=${GITHUB_REF#refs/tags/} - TAGS="docker.io/${DOCKER_IMAGE}:${DOCKER_VERSION},ghcr.io/${DOCKER_IMAGE}:${DOCKER_VERSION},quay.io/${DOCKER_IMAGE}:${DOCKER_VERSION},docker.io/${DOCKER_IMAGE}:latest,ghcr.io/${DOCKER_IMAGE}:latest,quay.io/${DOCKER_IMAGE}:latest" - fi - echo ::set-output name=tags::${TAGS} + - name: Generate Alpine Docker tags + id: docker_alpine_tags + uses: docker/metadata-action@v5 + with: + images: | + docker.io/${{ github.repository_owner }}/miniflux + ghcr.io/${{ github.repository_owner }}/miniflux + quay.io/${{ github.repository_owner }}/miniflux + tags: | + type=ref,event=pr + type=schedule,pattern=nightly + type=semver,pattern={{raw}} - - name: Generate Distroless Docker tag - id: docker_distroless_tag - run: | - DOCKER_IMAGE=${{ github.repository_owner }}/miniflux - DOCKER_VERSION=dev-distroless - if [ "${{ github.event_name }}" = "schedule" ]; then - DOCKER_VERSION=nightly-distroless - TAGS="docker.io/${DOCKER_IMAGE}:${DOCKER_VERSION},ghcr.io/${DOCKER_IMAGE}:${DOCKER_VERSION},quay.io/${DOCKER_IMAGE}:${DOCKER_VERSION}" - elif [[ $GITHUB_REF == refs/tags/* ]]; then - DOCKER_VERSION=${GITHUB_REF#refs/tags/}-distroless - TAGS="docker.io/${DOCKER_IMAGE}:${DOCKER_VERSION},ghcr.io/${DOCKER_IMAGE}:${DOCKER_VERSION},quay.io/${DOCKER_IMAGE}:${DOCKER_VERSION},docker.io/${DOCKER_IMAGE}:latest-distroless,ghcr.io/${DOCKER_IMAGE}:latest-distroless,quay.io/${DOCKER_IMAGE}:latest-distroless" - fi - echo ::set-output name=tags::${TAGS} + - name: Generate Distroless Docker tags + id: docker_distroless_tags + uses: docker/metadata-action@v5 + with: + images: | + docker.io/${{ github.repository_owner }}/miniflux + ghcr.io/${{ github.repository_owner }}/miniflux + quay.io/${{ github.repository_owner }}/miniflux + tags: | + type=ref,event=pr,suffix=-distroless + type=schedule,pattern=nightly,suffix=-distroless + type=semver,pattern={{raw}},suffix=-distroless - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -81,12 +52,14 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Login to DockerHub + if: github.event_name != 'pull_request' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to GitHub Container Registry + if: github.event_name != 'pull_request' uses: docker/login-action@v3 with: registry: ghcr.io @@ -94,6 +67,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Login to Quay Container Registry + if: github.event_name != 'pull_request' uses: docker/login-action@v3 with: registry: quay.io @@ -106,8 +80,8 @@ jobs: context: . file: ./packaging/docker/alpine/Dockerfile platforms: linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64 - push: true - tags: ${{ steps.docker_alpine_tag.outputs.tags }} + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.docker_alpine_tags.outputs.tags }} - name: Build and Push Distroless images uses: docker/build-push-action@v5 @@ -115,5 +89,5 @@ jobs: context: . file: ./packaging/docker/distroless/Dockerfile platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ steps.docker_distroless_tag.outputs.tags }} + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.docker_distroless_tags.outputs.tags }} diff --git a/go.mod b/go.mod index 934f9aec..a63c6c2f 100644 --- a/go.mod +++ b/go.mod @@ -11,11 +11,11 @@ require ( github.com/lib/pq v1.10.9 github.com/prometheus/client_golang v1.19.0 github.com/tdewolff/minify/v2 v2.20.19 - github.com/yuin/goldmark v1.7.0 - golang.org/x/crypto v0.21.0 - golang.org/x/net v0.22.0 - golang.org/x/oauth2 v0.18.0 - golang.org/x/term v0.18.0 + github.com/yuin/goldmark v1.7.1 + golang.org/x/crypto v0.22.0 + golang.org/x/net v0.24.0 + golang.org/x/oauth2 v0.19.0 + golang.org/x/term v0.19.0 golang.org/x/text v0.14.0 mvdan.cc/xurls/v2 v2.5.0 ) @@ -32,7 +32,6 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/fxamacker/cbor/v2 v2.6.0 // indirect github.com/go-jose/go-jose/v4 v4.0.1 // indirect - github.com/golang/protobuf v1.5.3 // indirect github.com/google/uuid v1.6.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect @@ -40,8 +39,7 @@ require ( github.com/prometheus/procfs v0.12.0 // indirect github.com/tdewolff/parse/v2 v2.7.12 // indirect github.com/x448/float16 v0.8.4 // indirect - golang.org/x/sys v0.18.0 // indirect - google.golang.org/appengine v1.6.8 // indirect + golang.org/x/sys v0.19.0 // indirect google.golang.org/protobuf v1.33.0 // indirect ) diff --git a/go.sum b/go.sum index 6cd0c212..f2013f52 100644 --- a/go.sum +++ b/go.sum @@ -22,11 +22,6 @@ github.com/go-webauthn/x v0.1.9 h1:v1oeLmoaa+gPOaZqUdDentu6Rl7HkSSsmOT6gxEQHhE= github.com/go-webauthn/x v0.1.9/go.mod h1:pJNMlIMP1SU7cN8HNlKJpLEnFHCygLCvaLZ8a1xeoQA= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-tpm v0.9.0 h1:sQF6YqWMi+SCXpsmS3fd21oPy/vSddwZry4JnmltHVk= @@ -61,12 +56,12 @@ github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739/go.mod h1:XPuWBzv github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yuin/goldmark v1.7.0 h1:EfOIvIMZIzHdB/R/zVrikYLPPwJlfMcNczJFMs1m6sA= -github.com/yuin/goldmark v1.7.0/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= +github.com/yuin/goldmark v1.7.1 h1:3bajkSilaCbjdKVsKdZjZCLBNPL9pYzrCakKaf4U49U= +github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -74,10 +69,10 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= -golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/oauth2 v0.19.0 h1:9+E/EZBCbTLNrbN35fHv/a/d/mOBatymz1zbtQrXpIg= +golang.org/x/oauth2 v0.19.0/go.mod h1:vYi7skDa1x015PmRRYZ7+s1cWyPgrPiSYRe4rnsexc8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -88,18 +83,17 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= +golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= @@ -109,11 +103,6 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/database/migrations.go b/internal/database/migrations.go index d40e5d2f..fa3c3972 100644 --- a/internal/database/migrations.go +++ b/internal/database/migrations.go @@ -882,4 +882,10 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, + func(tx *sql.Tx) (err error) { + // Entry URLs can exceeds btree maximum size + // Checking entry existence is now using entries_feed_id_status_hash_idx index + _, err = tx.Exec(`DROP INDEX entries_feed_url_idx`) + return err + }, } diff --git a/internal/integration/matrixbot/matrixbot.go b/internal/integration/matrixbot/matrixbot.go index 3e29d83d..8c0599be 100644 --- a/internal/integration/matrixbot/matrixbot.go +++ b/internal/integration/matrixbot/matrixbot.go @@ -10,7 +10,7 @@ import ( "miniflux.app/v2/internal/model" ) -// PushEntry pushes entries to matrix chat using integration settings provided +// PushEntries pushes entries to matrix chat using integration settings provided func PushEntries(feed *model.Feed, entries model.Entries, matrixBaseURL, matrixUsername, matrixPassword, matrixRoomID string) error { client := NewClient(matrixBaseURL) discovery, err := client.DiscoverEndpoints() diff --git a/internal/reader/processor/processor.go b/internal/reader/processor/processor.go index 913ae0b3..c92550d2 100644 --- a/internal/reader/processor/processor.go +++ b/internal/reader/processor/processor.go @@ -23,6 +23,8 @@ import ( "miniflux.app/v2/internal/storage" "github.com/PuerkitoBio/goquery" + "github.com/tdewolff/minify/v2" + "github.com/tdewolff/minify/v2/html" ) var ( @@ -36,14 +38,18 @@ var ( func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, user *model.User, forceRefresh bool) { var filteredEntries model.Entries + minifier := minify.New() + minifier.AddFunc("text/html", html.Minify) + // Process older entries first for i := len(feed.Entries) - 1; i >= 0; i-- { entry := feed.Entries[i] slog.Debug("Processing entry", slog.Int64("user_id", user.ID), - slog.Int64("entry_id", entry.ID), slog.String("entry_url", entry.URL), + slog.String("entry_hash", entry.Hash), + slog.String("entry_title", entry.Title), slog.Int64("feed_id", feed.ID), slog.String("feed_url", feed.FeedURL), ) @@ -52,14 +58,18 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, user *model.Us } websiteURL := getUrlFromEntry(feed, entry) - entryIsNew := !store.EntryURLExists(feed.ID, entry.URL) + entryIsNew := store.IsNewEntry(feed.ID, entry.Hash) if feed.Crawler && (entryIsNew || forceRefresh) { slog.Debug("Scraping entry", slog.Int64("user_id", user.ID), - slog.Int64("entry_id", entry.ID), slog.String("entry_url", entry.URL), + slog.String("entry_hash", entry.Hash), + slog.String("entry_title", entry.Title), slog.Int64("feed_id", feed.ID), slog.String("feed_url", feed.FeedURL), + slog.Bool("entry_is_new", entryIsNew), + slog.Bool("force_refresh", forceRefresh), + slog.String("website_url", websiteURL), ) startTime := time.Now() @@ -90,7 +100,6 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, user *model.Us if scraperErr != nil { slog.Warn("Unable to scrape entry", slog.Int64("user_id", user.ID), - slog.Int64("entry_id", entry.ID), slog.String("entry_url", entry.URL), slog.Int64("feed_id", feed.ID), slog.String("feed_url", feed.FeedURL), @@ -98,7 +107,11 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, user *model.Us ) } else if content != "" { // We replace the entry content only if the scraper doesn't return any error. - entry.Content = content + if minifiedHTML, err := minifier.String("text/html", content); err == nil { + entry.Content = minifiedHTML + } else { + entry.Content = content + } } } @@ -134,7 +147,6 @@ func isBlockedEntry(feed *model.Feed, entry *model.Entry) bool { if compiledBlocklist.MatchString(entry.URL) || compiledBlocklist.MatchString(entry.Title) || compiledBlocklist.MatchString(entry.Author) || containsBlockedTag { slog.Debug("Blocking entry based on rule", - slog.Int64("entry_id", entry.ID), slog.String("entry_url", entry.URL), slog.Int64("feed_id", feed.ID), slog.String("feed_url", feed.FeedURL), @@ -165,7 +177,6 @@ func isAllowedEntry(feed *model.Feed, entry *model.Entry) bool { if compiledKeeplist.MatchString(entry.URL) || compiledKeeplist.MatchString(entry.Title) || compiledKeeplist.MatchString(entry.Author) || containsAllowedTag { slog.Debug("Allow entry based on rule", - slog.Int64("entry_id", entry.ID), slog.String("entry_url", entry.URL), slog.Int64("feed_id", feed.ID), slog.String("feed_url", feed.FeedURL), @@ -178,6 +189,9 @@ func isAllowedEntry(feed *model.Feed, entry *model.Entry) bool { // ProcessEntryWebPage downloads the entry web page and apply rewrite rules. func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry, user *model.User) error { + minifier := minify.New() + minifier.AddFunc("text/html", html.Minify) + startTime := time.Now() websiteURL := getUrlFromEntry(feed, entry) @@ -209,7 +223,11 @@ func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry, user *model.User) } if content != "" { - entry.Content = content + if minifiedHTML, err := minifier.String("text/html", content); err == nil { + entry.Content = minifiedHTML + } else { + entry.Content = content + } if user.ShowReadingTime { entry.ReadingTime = readingtime.EstimateReadingTime(entry.Content, user.DefaultReadingSpeed, user.CJKReadingSpeed) } @@ -230,7 +248,6 @@ func getUrlFromEntry(feed *model.Feed, entry *model.Entry) string { re := regexp.MustCompile(parts[1]) url = re.ReplaceAllString(entry.URL, parts[2]) slog.Debug("Rewriting entry URL", - slog.Int64("entry_id", entry.ID), slog.String("original_entry_url", entry.URL), slog.String("rewritten_entry_url", url), slog.Int64("feed_id", feed.ID), @@ -238,7 +255,6 @@ func getUrlFromEntry(feed *model.Feed, entry *model.Entry) string { ) } else { slog.Debug("Cannot find search and replace terms for replace rule", - slog.Int64("entry_id", entry.ID), slog.String("original_entry_url", entry.URL), slog.String("rewritten_entry_url", url), slog.Int64("feed_id", feed.ID), @@ -251,6 +267,11 @@ func getUrlFromEntry(feed *model.Feed, entry *model.Entry) string { } func updateEntryReadingTime(store *storage.Storage, feed *model.Feed, entry *model.Entry, entryIsNew bool, user *model.User) { + if !user.ShowReadingTime { + slog.Debug("Skip reading time estimation for this user", slog.Int64("user_id", user.ID)) + return + } + if shouldFetchYouTubeWatchTime(entry) { if entryIsNew { watchTime, err := fetchYouTubeWatchTime(entry.URL) @@ -266,7 +287,7 @@ func updateEntryReadingTime(store *storage.Storage, feed *model.Feed, entry *mod } entry.ReadingTime = watchTime } else { - entry.ReadingTime = store.GetReadTime(entry, feed) + entry.ReadingTime = store.GetReadTime(feed.ID, entry.Hash) } } @@ -285,14 +306,13 @@ func updateEntryReadingTime(store *storage.Storage, feed *model.Feed, entry *mod } entry.ReadingTime = watchTime } else { - entry.ReadingTime = store.GetReadTime(entry, feed) + entry.ReadingTime = store.GetReadTime(feed.ID, entry.Hash) } } + // Handle YT error case and non-YT entries. if entry.ReadingTime == 0 { - if user.ShowReadingTime { - entry.ReadingTime = readingtime.EstimateReadingTime(entry.Content, user.DefaultReadingSpeed, user.CJKReadingSpeed) - } + entry.ReadingTime = readingtime.EstimateReadingTime(entry.Content, user.DefaultReadingSpeed, user.CJKReadingSpeed) } } diff --git a/internal/storage/entry.go b/internal/storage/entry.go index 1a7cc6d7..f22a424b 100644 --- a/internal/storage/entry.go +++ b/internal/storage/entry.go @@ -225,24 +225,27 @@ func (s *Storage) entryExists(tx *sql.Tx, entry *model.Entry) (bool, error) { return result, nil } -// GetReadTime fetches the read time of an entry based on its hash, and the feed id and user id from the feed. -// It's intended to be used on entries objects created by parsing a feed as they don't contain much information. -// The feed param helps to scope the search to a specific user and feed in order to avoid hash clashes. -func (s *Storage) GetReadTime(entry *model.Entry, feed *model.Feed) int { +func (s *Storage) IsNewEntry(feedID int64, entryHash string) bool { + var result bool + s.db.QueryRow(`SELECT true FROM entries WHERE feed_id=$1 AND hash=$2`, feedID, entryHash).Scan(&result) + return !result +} + +func (s *Storage) GetReadTime(feedID int64, entryHash string) int { var result int + + // Note: This query uses entries_feed_id_hash_key index s.db.QueryRow( `SELECT reading_time FROM entries WHERE - user_id=$1 AND - feed_id=$2 AND - hash=$3 + feed_id=$1 AND + hash=$2 `, - feed.UserID, - feed.ID, - entry.Hash, + feedID, + entryHash, ).Scan(&result) return result } @@ -575,14 +578,6 @@ func (s *Storage) MarkCategoryAsRead(userID, categoryID int64, before time.Time) return nil } -// EntryURLExists returns true if an entry with this URL already exists. -func (s *Storage) EntryURLExists(feedID int64, entryURL string) bool { - var result bool - query := `SELECT true FROM entries WHERE feed_id=$1 AND url=$2` - s.db.QueryRow(query, feedID, entryURL).Scan(&result) - return result -} - // EntryShareCode returns the share code of the provided entry. // It generates a new one if not already defined. func (s *Storage) EntryShareCode(userID int64, entryID int64) (shareCode string, err error) { diff --git a/internal/ui/static/js/app.js b/internal/ui/static/js/app.js index 79ffb4b5..02911194 100644 --- a/internal/ui/static/js/app.js +++ b/internal/ui/static/js/app.js @@ -86,7 +86,8 @@ function onClickMainMenuListItem(event) { if (element.tagName === "A") { window.location.href = element.getAttribute("href"); } else { - window.location.href = element.querySelector("a").getAttribute("href"); + const linkElement = element.querySelector("a") || element.closest("a"); + window.location.href = linkElement.getAttribute("href"); } } diff --git a/packaging/docker/alpine/Dockerfile b/packaging/docker/alpine/Dockerfile index 9fc93858..93355295 100644 --- a/packaging/docker/alpine/Dockerfile +++ b/packaging/docker/alpine/Dockerfile @@ -1,14 +1,10 @@ -FROM golang:alpine AS build -ENV CGO_ENABLED=0 -RUN apk add --no-cache --update git +FROM docker.io/library/golang:alpine3.19 AS build +RUN apk add --no-cache build-base git make ADD . /go/src/app WORKDIR /go/src/app -RUN go build \ - -o miniflux \ - -ldflags="-s -w -X 'miniflux.app/v2/internal/version.Version=`git describe --tags --abbrev=0`' -X 'miniflux.app/v2/internal/version.Commit=`git rev-parse --short HEAD`' -X 'miniflux.app/v2/internal/version.BuildDate=`date +%FT%T%z`'" \ - main.go +RUN make miniflux -FROM alpine:latest +FROM docker.io/library/alpine:3.19 LABEL org.opencontainers.image.title=Miniflux LABEL org.opencontainers.image.description="Miniflux is a minimalist and opinionated feed reader" diff --git a/packaging/docker/distroless/Dockerfile b/packaging/docker/distroless/Dockerfile index 5d50dd98..a4080891 100644 --- a/packaging/docker/distroless/Dockerfile +++ b/packaging/docker/distroless/Dockerfile @@ -1,13 +1,9 @@ -FROM golang:latest AS build -ENV CGO_ENABLED=0 +FROM docker.io/library/golang:bookworm AS build ADD . /go/src/app WORKDIR /go/src/app -RUN go build \ - -o miniflux \ - -ldflags="-s -w -X 'miniflux.app/v2/internal/version.Version=`git describe --tags --abbrev=0`' -X 'miniflux.app/v2/internal/version.Commit=`git rev-parse --short HEAD`' -X 'miniflux.app/v2/internal/version.BuildDate=`date +%FT%T%z`'" \ - main.go +RUN make miniflux -FROM gcr.io/distroless/base:nonroot +FROM gcr.io/distroless/base-debian12:nonroot LABEL org.opencontainers.image.title=Miniflux LABEL org.opencontainers.image.description="Miniflux is a minimalist and opinionated feed reader"