diff --git a/locale/translations.go b/locale/translations.go
index 217541ee..7124f1b6 100644
--- a/locale/translations.go
+++ b/locale/translations.go
@@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
-// 2017-12-10 18:56:24.387844114 -0800 PST m=+0.029823201
+// 2017-12-10 20:08:14.447304303 -0800 PST m=+0.040286758
package locale
@@ -167,12 +167,13 @@ var translations = map[string]string{
"Activate Fever API": "Activer l'API de Fever",
"Fever Username": "Nom d'utilisateur pour l'API de Fever",
"Fever Password": "Mot de passe pour l'API de Fever",
- "Fetch original content": "Récupérer le contenu original"
+ "Fetch original content": "Récupérer le contenu original",
+ "Scraper Rules": "Règles pour récupérer le contenu original"
}
`,
}
var translationsChecksums = map[string]string{
"en_US": "6fe95384260941e8a5a3c695a655a932e0a8a6a572c1e45cb2b1ae8baa01b897",
- "fr_FR": "fd629b171aefa50dd0a6100acaac8fbecbdf1a1d53e3fce984234565ec5bb5d5",
+ "fr_FR": "4426cea875ee2c9acb1a2b0619cb82f3a32f71aabe5d07657eaf2f6b7387c5f9",
}
diff --git a/locale/translations/fr_FR.json b/locale/translations/fr_FR.json
index cc82efed..0a51ec3c 100644
--- a/locale/translations/fr_FR.json
+++ b/locale/translations/fr_FR.json
@@ -151,5 +151,6 @@
"Activate Fever API": "Activer l'API de Fever",
"Fever Username": "Nom d'utilisateur pour l'API de Fever",
"Fever Password": "Mot de passe pour l'API de Fever",
- "Fetch original content": "Récupérer le contenu original"
+ "Fetch original content": "Récupérer le contenu original",
+ "Scraper Rules": "Règles pour récupérer le contenu original"
}
diff --git a/model/feed.go b/model/feed.go
index dbdb9d6a..fb2819da 100644
--- a/model/feed.go
+++ b/model/feed.go
@@ -22,6 +22,7 @@ type Feed struct {
LastModifiedHeader string `json:"last_modified_header,omitempty"`
ParsingErrorMsg string `json:"parsing_error_message,omitempty"`
ParsingErrorCount int `json:"parsing_error_count,omitempty"`
+ ScraperRules string `json:"scraper_rules"`
Category *Category `json:"category,omitempty"`
Entries Entries `json:"entries,omitempty"`
Icon *FeedIcon `json:"icon,omitempty"`
diff --git a/reader/scraper/rules.go b/reader/scraper/rules.go
new file mode 100644
index 00000000..ae6c4a57
--- /dev/null
+++ b/reader/scraper/rules.go
@@ -0,0 +1,16 @@
+// Copyright 2017 Frédéric Guillot. All rights reserved.
+// Use of this source code is governed by the Apache 2.0
+// license that can be found in the LICENSE file.
+
+package scraper
+
+// List of predefined scraper rules (alphabetically sorted)
+// domain => CSS selectors
+var predefinedRules = map[string]string{
+ "lemonde.fr": "div#articleBody",
+ "lesjoiesducode.fr": ".blog-post-content img",
+ "linux.com": "div.content, div[property]",
+ "opensource.com": "div[property]",
+ "phoronix.com": "div.content",
+ "techcrunch.com": "div.article-entry",
+}
diff --git a/reader/scraper/scraper.go b/reader/scraper/scraper.go
index 6c518621..b79a088a 100644
--- a/reader/scraper/scraper.go
+++ b/reader/scraper/scraper.go
@@ -6,14 +6,19 @@ package scraper
import (
"errors"
+ "io"
+ "log"
+ "strings"
+ "github.com/PuerkitoBio/goquery"
"github.com/miniflux/miniflux2/http"
"github.com/miniflux/miniflux2/reader/readability"
"github.com/miniflux/miniflux2/reader/sanitizer"
+ "github.com/miniflux/miniflux2/url"
)
// Fetch download a web page a returns relevant contents.
-func Fetch(websiteURL string) (string, error) {
+func Fetch(websiteURL, rules string) (string, error) {
client := http.NewClient(websiteURL)
response, err := client.Get()
if err != nil {
@@ -29,10 +34,57 @@ func Fetch(websiteURL string) (string, error) {
return "", err
}
- content, err := readability.ExtractContent(page)
+ var content string
+ if rules == "" {
+ rules = getPredefinedScraperRules(websiteURL)
+ }
+
+ if rules != "" {
+ log.Printf(`[Scraper] Using rules "%s" for "%s"`, rules, websiteURL)
+ content, err = scrapContent(page, rules)
+ } else {
+ log.Printf(`[Scraper] Using readability for "%s"`, websiteURL)
+ content, err = readability.ExtractContent(page)
+ }
+
if err != nil {
return "", err
}
return sanitizer.Sanitize(websiteURL, content), nil
}
+
+func scrapContent(page io.Reader, rules string) (string, error) {
+ document, err := goquery.NewDocumentFromReader(page)
+ if err != nil {
+ return "", err
+ }
+
+ contents := ""
+ document.Find(rules).Each(func(i int, s *goquery.Selection) {
+ var content string
+
+ // For some inline elements, we get the parent.
+ if s.Is("img") {
+ content, _ = s.Parent().Html()
+ } else {
+ content, _ = s.Html()
+ }
+
+ contents += content
+ })
+
+ return contents, nil
+}
+
+func getPredefinedScraperRules(websiteURL string) string {
+ urlDomain := url.Domain(websiteURL)
+
+ for domain, rules := range predefinedRules {
+ if strings.Contains(urlDomain, domain) {
+ return rules
+ }
+ }
+
+ return ""
+}
diff --git a/reader/scraper/scraper_test.go b/reader/scraper/scraper_test.go
new file mode 100644
index 00000000..b493e25c
--- /dev/null
+++ b/reader/scraper/scraper_test.go
@@ -0,0 +1,21 @@
+// Copyright 2017 Frédéric Guillot. All rights reserved.
+// Use of this source code is governed by the Apache 2.0
+// license that can be found in the LICENSE file.
+
+package scraper
+
+import "testing"
+
+func TestGetPredefinedRules(t *testing.T) {
+ if getPredefinedScraperRules("http://www.phoronix.com/") == "" {
+ t.Error("Unable to find rule for phoronix.com")
+ }
+
+ if getPredefinedScraperRules("https://www.linux.com/") == "" {
+ t.Error("Unable to find rule for linux.com")
+ }
+
+ if getPredefinedScraperRules("https://example.org/") != "" {
+ t.Error("A rule not defined should not return anything")
+ }
+}
diff --git a/server/template/html/edit_feed.html b/server/template/html/edit_feed.html
index fac2a9b7..04950926 100644
--- a/server/template/html/edit_feed.html
+++ b/server/template/html/edit_feed.html
@@ -45,6 +45,9 @@
+
+
+