From 51fb9495ad6117ba6926fb9279275efbcabd3ac4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Guillot?= Date: Tue, 16 Mar 2021 22:15:40 -0700 Subject: [PATCH] Improve large feed deletion First patch to avoid the app hanging when deleting large feeds --- storage/feed.go | 33 ++++++++++++------ storage/user.go | 90 +++++++++++++++---------------------------------- 2 files changed, 49 insertions(+), 74 deletions(-) diff --git a/storage/feed.go b/storage/feed.go index 9c4fec3d..bd366b1d 100644 --- a/storage/feed.go +++ b/storage/feed.go @@ -8,8 +8,10 @@ import ( "database/sql" "errors" "fmt" + "runtime" "miniflux.app/config" + "miniflux.app/logger" "miniflux.app/model" ) @@ -353,21 +355,30 @@ func (s *Storage) UpdateFeedError(feed *model.Feed) (err error) { return nil } -// RemoveFeed removes a feed. +// RemoveFeed removes a feed and all entries. +// This operation can takes time if the feed has lot of entries. func (s *Storage) RemoveFeed(userID, feedID int64) error { - query := `DELETE FROM feeds WHERE id = $1 AND user_id = $2` - result, err := s.db.Exec(query, feedID, userID) + rows, err := s.db.Query(`SELECT id FROM entries WHERE user_id=$1 AND feed_id=$2`, userID, feedID) if err != nil { - return fmt.Errorf(`store: unable to remove feed #%d: %v`, feedID, err) + return fmt.Errorf(`store: unable to get user feed entries: %v`, err) + } + defer rows.Close() + + for rows.Next() { + var entryID int64 + if err := rows.Scan(&entryID); err != nil { + return fmt.Errorf(`store: unable to read user feed entry ID: %v`, err) + } + + logger.Debug(`[FEED DELETION] Deleting entry #%d of feed #%d for user #%d (%d GoRoutines)`, entryID, feedID, userID, runtime.NumGoroutine()) + + if _, err := s.db.Exec(`DELETE FROM entries WHERE id=$1 AND user_id=$2`, entryID, userID); err != nil { + return fmt.Errorf(`store: unable to delete user feed entries #%d: %v`, entryID, err) + } } - count, err := result.RowsAffected() - if err != nil { - return fmt.Errorf(`store: unable to remove feed #%d: %v`, feedID, err) - } - - if count == 0 { - return errors.New(`store: no feed has been removed`) + if _, err := s.db.Exec(`DELETE FROM feeds WHERE id=$1`, feedID); err != nil { + return fmt.Errorf(`store: unable to delete feed #%d: %v`, feedID, err) } return nil diff --git a/storage/user.go b/storage/user.go index e86f4bb2..32e9162a 100644 --- a/storage/user.go +++ b/storage/user.go @@ -428,13 +428,39 @@ func (s *Storage) RemoveUser(userID int64) error { // RemoveUserAsync deletes user data without locking the database. func (s *Storage) RemoveUserAsync(userID int64) { go func() { - deleteUserFeeds(s.db, userID) + if err := s.deleteUserFeeds(userID); err != nil { + logger.Error(`%v`, err) + return + } + s.db.Exec(`DELETE FROM users WHERE id=$1`, userID) s.db.Exec(`DELETE FROM integrations WHERE user_id=$1`, userID) + logger.Debug(`[MASS DELETE] User #%d has been deleted (%d GoRoutines)`, userID, runtime.NumGoroutine()) }() } +func (s *Storage) deleteUserFeeds(userID int64) error { + rows, err := s.db.Query(`SELECT id FROM feeds WHERE user_id=$1`, userID) + if err != nil { + return fmt.Errorf(`store: unable to get user feeds: %v`, err) + } + defer rows.Close() + + for rows.Next() { + var feedID int64 + rows.Scan(&feedID) + + logger.Debug(`[USER DELETION] Deleting feed #%d for user #%d (%d GoRoutines)`, feedID, userID, runtime.NumGoroutine()) + + if err := s.RemoveFeed(userID, feedID); err != nil { + return err + } + } + + return nil +} + // Users returns all users. func (s *Storage) Users() (model.Users, error) { query := ` @@ -538,65 +564,3 @@ func hashPassword(password string) (string, error) { bytes, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) return string(bytes), err } - -func deleteUserFeeds(db *sql.DB, userID int64) { - query := `SELECT id FROM feeds WHERE user_id=$1` - rows, err := db.Query(query, userID) - if err != nil { - logger.Error(`store: unable to get user feeds: %v`, err) - return - } - defer rows.Close() - - var feedIDs []int64 - for rows.Next() { - var feedID int64 - rows.Scan(&feedID) - feedIDs = append(feedIDs, feedID) - } - - worker := func(jobs <-chan int64, results chan<- bool) { - for feedID := range jobs { - logger.Debug(`[MASS DELETE] Deleting feed #%d for user #%d (%d GoRoutines)`, feedID, userID, runtime.NumGoroutine()) - deleteUserEntries(db, userID, feedID) - db.Exec(`DELETE FROM feeds WHERE id=$1`, feedID) - results <- true - } - } - - numJobs := len(feedIDs) - jobs := make(chan int64, numJobs) - results := make(chan bool, numJobs) - - for w := 0; w < 2; w++ { - go worker(jobs, results) - } - - for j := 0; j < numJobs; j++ { - jobs <- feedIDs[j] - } - close(jobs) - - for a := 1; a <= numJobs; a++ { - <-results - } -} - -func deleteUserEntries(db *sql.DB, userID int64, feedID int64) { - rows, err := db.Query(`SELECT id FROM entries WHERE user_id=$1 AND feed_id=$2`, userID, feedID) - if err != nil { - logger.Error(`store: unable to get user feed entries: %v`, err) - return - } - defer rows.Close() - - for rows.Next() { - var entryID int64 - rows.Scan(&entryID) - - logger.Debug(`[MASS DELETE] Deleting entry #%d for user #%d (%d GoRoutines)`, entryID, userID, runtime.NumGoroutine()) - - db.Exec(`DELETE FROM enclosures WHERE entry_id=$1 AND user_id=$2`, entryID, userID) - db.Exec(`DELETE FROM entries WHERE id=$1 AND user_id=$2`, entryID, userID) - } -}