mirror of https://github.com/miniflux/v2.git
Prevent Youtube scraping if entry already exists
This commit is contained in:
parent
d092fc8436
commit
1d80c12e18
|
@ -45,8 +45,8 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if feed.Crawler {
|
entryIsNew := !store.EntryURLExists(feed.ID, entry.URL)
|
||||||
if !store.EntryURLExists(feed.ID, entry.URL) {
|
if feed.Crawler && entryIsNew {
|
||||||
logger.Debug("[Processor] Crawling entry %q from feed %q", entry.URL, feed.FeedURL)
|
logger.Debug("[Processor] Crawling entry %q from feed %q", entry.URL, feed.FeedURL)
|
||||||
|
|
||||||
startTime := time.Now()
|
startTime := time.Now()
|
||||||
|
@ -72,27 +72,13 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed) {
|
||||||
entry.Content = content
|
entry.Content = content
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
entry.Content = rewrite.Rewriter(entry.URL, entry.Content, feed.RewriteRules)
|
entry.Content = rewrite.Rewriter(entry.URL, entry.Content, feed.RewriteRules)
|
||||||
|
|
||||||
// The sanitizer should always run at the end of the process to make sure unsafe HTML is filtered.
|
// The sanitizer should always run at the end of the process to make sure unsafe HTML is filtered.
|
||||||
entry.Content = sanitizer.Sanitize(entry.URL, entry.Content)
|
entry.Content = sanitizer.Sanitize(entry.URL, entry.Content)
|
||||||
|
|
||||||
if config.Opts.FetchYouTubeWatchTime() {
|
updateEntryReadingTime(store, feed, entry, entryIsNew)
|
||||||
if matches := youtubeRegex.FindStringSubmatch(entry.URL); len(matches) == 2 {
|
|
||||||
watchTime, err := fetchYouTubeWatchTime(entry.URL)
|
|
||||||
if err != nil {
|
|
||||||
logger.Error("[Processor] Unable to fetch YouTube watch time: %q => %v", entry.URL, err)
|
|
||||||
}
|
|
||||||
entry.ReadingTime = watchTime
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if entry.ReadingTime == 0 {
|
|
||||||
entry.ReadingTime = calculateReadingTime(entry.Content)
|
|
||||||
}
|
|
||||||
|
|
||||||
filteredEntries = append(filteredEntries, entry)
|
filteredEntries = append(filteredEntries, entry)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -155,6 +141,34 @@ func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func updateEntryReadingTime(store *storage.Storage, feed *model.Feed, entry *model.Entry, entryIsNew bool) {
|
||||||
|
if shouldFetchYouTubeWatchTime(entry) {
|
||||||
|
if entryIsNew {
|
||||||
|
watchTime, err := fetchYouTubeWatchTime(entry.URL)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("[Processor] Unable to fetch YouTube watch time: %q => %v", entry.URL, err)
|
||||||
|
}
|
||||||
|
entry.ReadingTime = watchTime
|
||||||
|
} else {
|
||||||
|
entry.ReadingTime = store.GetReadTime(entry, feed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle YT error case and non-YT entries.
|
||||||
|
if entry.ReadingTime == 0 {
|
||||||
|
entry.ReadingTime = calculateReadingTime(entry.Content)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func shouldFetchYouTubeWatchTime(entry *model.Entry) bool {
|
||||||
|
if !config.Opts.FetchYouTubeWatchTime() {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
matches := youtubeRegex.FindStringSubmatch(entry.URL)
|
||||||
|
urlMatchesYouTubePattern := len(matches) == 2
|
||||||
|
return urlMatchesYouTubePattern
|
||||||
|
}
|
||||||
|
|
||||||
func fetchYouTubeWatchTime(url string) (int, error) {
|
func fetchYouTubeWatchTime(url string) (int, error) {
|
||||||
clt := client.NewClientWithConfig(url, config.Opts)
|
clt := client.NewClientWithConfig(url, config.Opts)
|
||||||
response, browserErr := browser.Exec(clt)
|
response, browserErr := browser.Exec(clt)
|
||||||
|
|
|
@ -225,6 +225,20 @@ func (s *Storage) entryExists(tx *sql.Tx, entry *model.Entry) bool {
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetReadTime fetches the read time of an entry based on its hash, and the feed id and user id from the feed.
|
||||||
|
// It's intended to be used on entries objects created by parsing a feed as they don't contain much information.
|
||||||
|
// The feed param helps to scope the search to a specific user and feed in order to avoid hash clashes.
|
||||||
|
func (s *Storage) GetReadTime(entry *model.Entry, feed *model.Feed) int {
|
||||||
|
var result int
|
||||||
|
s.db.QueryRow(
|
||||||
|
`SELECT reading_time FROM entries WHERE user_id=$1 AND feed_id=$2 AND hash=$3`,
|
||||||
|
feed.UserID,
|
||||||
|
feed.ID,
|
||||||
|
entry.Hash,
|
||||||
|
).Scan(&result)
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
// cleanupEntries deletes from the database entries marked as "removed" and not visible anymore in the feed.
|
// cleanupEntries deletes from the database entries marked as "removed" and not visible anymore in the feed.
|
||||||
func (s *Storage) cleanupEntries(feedID int64, entryHashes []string) error {
|
func (s *Storage) cleanupEntries(feedID int64, entryHashes []string) error {
|
||||||
query := `
|
query := `
|
||||||
|
|
Loading…
Reference in New Issue