From 351bbe3497517f631d1248e95cb16239ea468f02 Mon Sep 17 00:00:00 2001 From: hermes-lol Date: Tue, 5 May 2026 11:23:22 +0000 Subject: [PATCH 1/2] feat: add Keywords and Description support for RSS articles - Add Keywords and Description fields to Article model - Implement custom XML parser to capture multiple tags - Add database migration for new columns - Update scanner to pass new fields to database --- internal/model/model.go | 2 + internal/rss/rss.go | 158 ++++++++++++++++++++++++++++++----- internal/scanner/scanner.go | 2 + internal/storage/database.go | 36 +++++--- 4 files changed, 164 insertions(+), 34 deletions(-) diff --git a/internal/model/model.go b/internal/model/model.go index 7187a12..fe8a427 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -20,4 +20,6 @@ type Article struct { PublishedDate *time.Time DiscoveredDate *time.Time IsRead bool + Keywords string + Description string } diff --git a/internal/rss/rss.go b/internal/rss/rss.go index 188d51b..f5efdd4 100644 --- a/internal/rss/rss.go +++ b/internal/rss/rss.go @@ -1,8 +1,10 @@ package rss import ( + "encoding/xml" "errors" "fmt" + "io" "mime" "net/http" "net/url" @@ -17,6 +19,8 @@ type FeedArticle struct { Title string URL string PublishedDate *time.Time + Keywords string + Description string } type FeedParseError struct { @@ -38,29 +42,124 @@ func ParseFeed(feedURL string, timeout time.Duration, userAgent string) ([]FeedA return nil, FeedParseError{Message: fmt.Sprintf("failed to fetch feed: status %d", response.StatusCode)} } - parser := gofeed.NewParser() - feed, err := parser.Parse(response.Body) + // Use custom XML parser to capture all tags + articles, err := parseFeedXML(response.Body) if err != nil { return nil, FeedParseError{Message: fmt.Sprintf("failed to parse feed: %v", err)} } + return articles, nil +} + +// parseFeedXML uses xml.Decoder to capture all custom elements including multiple tags +func parseFeedXML(reader io.Reader) ([]FeedArticle, error) { + decoder := xml.NewDecoder(reader) var articles []FeedArticle - for _, item := range feed.Items { - title := strings.TrimSpace(item.Title) - link := strings.TrimSpace(item.Link) - if title == "" || link == "" { - continue + var currentItem *xml.StartElement + var currentTitle, currentLink, currentDescription string + var currentPubDate string + var currentKeywords []string + + for { + token, err := decoder.Token() + if err != nil { + if err == io.EOF { + break + } + return nil, err + } + + switch elem := token.(type) { + case xml.StartElement: + if elem.Name.Local == "item" { + // Start of new item, reset + currentItem = &elem + currentTitle = "" + currentLink = "" + currentDescription = "" + currentPubDate = "" + currentKeywords = nil + } else if currentItem != nil { + // Inside item, look for specific elements + switch elem.Name.Local { + case "title": + if text := readTextContent(decoder, elem); text != "" { + currentTitle = text + } + case "link": + if text := readTextContent(decoder, elem); text != "" { + currentLink = text + } + case "description": + if text := readTextContent(decoder, elem); text != "" { + currentDescription = text + } + case "pubDate": + if text := readTextContent(decoder, elem); text != "" { + currentPubDate = text + } + case "keyword": + if text := readTextContent(decoder, elem); text != "" { + currentKeywords = append(currentKeywords, text) + } + } + } + case xml.EndElement: + if elem.Name.Local == "item" && currentTitle != "" && currentLink != "" { + pubDate := parseRSSDate(currentPubDate) + desc := stripHTML(currentDescription) + articles = append(articles, FeedArticle{ + Title: strings.TrimSpace(currentTitle), + URL: strings.TrimSpace(currentLink), + PublishedDate: pubDate, + Keywords: strings.Join(currentKeywords, ","), + Description: strings.TrimSpace(desc), + }) + currentItem = nil + } } - articles = append(articles, FeedArticle{ - Title: title, - URL: link, - PublishedDate: pickPublishedDate(item), - }) } return articles, nil } +func readTextContent(decoder *xml.Decoder, elem xml.StartElement) string { + var text string + for { + token, err := decoder.Token() + if err != nil { + break + } + if char, ok := token.(xml.CharData); ok { + text += string(char) + } + if _, ok := token.(xml.EndElement); ok { + break + } + } + return text +} + +func parseRSSDate(dateStr string) *time.Time { + if dateStr == "" { + return nil + } + // Try multiple date formats + formats := []string{ + time.RFC1123Z, // "Mon, 02 Jan 2006 15:04:05 -0700" + time.RFC1123, // "Mon, 02 Jan 2006 15:04:05 GMT" + "Mon, 02 Jan 2006 15:04:05 -0700", + "Mon, 02 Jan 2006 15:04:05 GMT", + time.RFC3339, + } + for _, format := range formats { + if t, err := time.Parse(format, dateStr); err == nil { + return &t + } + } + return nil +} + func DiscoverFeedURL(blogURL string, timeout time.Duration, userAgent string) (string, error) { client := &http.Client{Timeout: timeout} response, err := getWithOptionalUserAgent(client, blogURL, userAgent) @@ -189,17 +288,32 @@ func resolveURL(base *url.URL, href string) string { return base.ResolveReference(parsed).String() } -func pickPublishedDate(item *gofeed.Item) *time.Time { - if item == nil { - return nil - } - if item.PublishedParsed != nil { - return item.PublishedParsed - } - if item.UpdatedParsed != nil { - return item.UpdatedParsed +func stripHTML(html string) string { + re := strings.NewReplacer( + "
", " ", + "
", " ", + "
", " ", + "

", " ", + "

", " ", + "
", " ", + "
", " ", + " ", " ", + "&", "&", + "<", "<", + ">", ">", + """, `"`, + ) + html = re.Replace(html) + for strings.Contains(html, "<") && strings.Contains(html, ">") { + start := strings.Index(html, "<") + end := strings.Index(html, ">") + if end > start { + html = html[:start] + html[end+1:] + } else { + break + } } - return nil + return strings.TrimSpace(html) } func IsFeedError(err error) bool { diff --git a/internal/scanner/scanner.go b/internal/scanner/scanner.go index dbd56c6..ce5d19c 100644 --- a/internal/scanner/scanner.go +++ b/internal/scanner/scanner.go @@ -184,6 +184,8 @@ func convertFeedArticles(blogID int64, articles []rss.FeedArticle) []model.Artic URL: article.URL, PublishedDate: article.PublishedDate, IsRead: false, + Keywords: article.Keywords, + Description: article.Description, }) } return result diff --git a/internal/storage/database.go b/internal/storage/database.go index b366577..7c5ee49 100644 --- a/internal/storage/database.go +++ b/internal/storage/database.go @@ -118,6 +118,10 @@ func (db *Database) migrate() error { "blogs": { {"user_agent", "TEXT"}, }, + "articles": { + {"keywords", "TEXT"}, + {"description", "TEXT"}, + }, } for table, cols := range migrations { // Fetch all existing columns for this table. @@ -248,14 +252,16 @@ func (db *Database) RemoveBlog(id int64) (bool, error) { func (db *Database) AddArticle(article model.Article) (model.Article, error) { result, err := db.conn.Exec( - `INSERT INTO articles (blog_id, title, url, published_date, discovered_date, is_read) - VALUES (?, ?, ?, ?, ?, ?)`, + `INSERT INTO articles (blog_id, title, url, published_date, discovered_date, is_read, keywords, description) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, article.BlogID, article.Title, article.URL, formatTimePtr(article.PublishedDate), formatTimePtr(article.DiscoveredDate), article.IsRead, + nullIfEmpty(article.Keywords), + nullIfEmpty(article.Description), ) if err != nil { return article, err @@ -276,7 +282,7 @@ func (db *Database) AddArticlesBulk(articles []model.Article) (int, error) { if err != nil { return 0, err } - stmt, err := _tx.Prepare(`INSERT INTO articles (blog_id, title, url, published_date, discovered_date, is_read) VALUES (?, ?, ?, ?, ?, ?)`) + stmt, err := _tx.Prepare(`INSERT INTO articles (blog_id, title, url, published_date, discovered_date, is_read, keywords, description) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`) if err != nil { _ = _tx.Rollback() return 0, err @@ -291,6 +297,8 @@ func (db *Database) AddArticlesBulk(articles []model.Article) (int, error) { formatTimePtr(article.PublishedDate), formatTimePtr(article.DiscoveredDate), article.IsRead, + nullIfEmpty(article.Keywords), + nullIfEmpty(article.Description), ) if err != nil { _ = _tx.Rollback() @@ -304,12 +312,12 @@ func (db *Database) AddArticlesBulk(articles []model.Article) (int, error) { } func (db *Database) GetArticle(id int64) (*model.Article, error) { - row := db.conn.QueryRow(`SELECT id, blog_id, title, url, published_date, discovered_date, is_read FROM articles WHERE id = ?`, id) + row := db.conn.QueryRow(`SELECT id, blog_id, title, url, published_date, discovered_date, is_read, keywords, description FROM articles WHERE id = ?`, id) return scanArticle(row) } func (db *Database) GetArticleByURL(url string) (*model.Article, error) { - row := db.conn.QueryRow(`SELECT id, blog_id, title, url, published_date, discovered_date, is_read FROM articles WHERE url = ?`, url) + row := db.conn.QueryRow(`SELECT id, blog_id, title, url, published_date, discovered_date, is_read, keywords, description FROM articles WHERE url = ?`, url) return scanArticle(row) } @@ -363,7 +371,7 @@ func (db *Database) GetExistingArticleURLs(urls []string) (map[string]struct{}, } func (db *Database) ListArticles(unreadOnly bool, blogID *int64) ([]model.Article, error) { - query := `SELECT id, blog_id, title, url, published_date, discovered_date, is_read FROM articles WHERE 1=1` + query := `SELECT id, blog_id, title, url, published_date, discovered_date, is_read, keywords, description FROM articles WHERE 1=1` var args []interface{} if unreadOnly { query += " AND is_read = 0" @@ -459,8 +467,10 @@ func scanArticle(scanner interface{ Scan(dest ...any) error }) (*model.Article, publishedDate sql.NullString discovered sql.NullString isRead bool + keywords sql.NullString + description sql.NullString ) - if err := scanner.Scan(&id, &blogID, &title, &url, &publishedDate, &discovered, &isRead); err != nil { + if err := scanner.Scan(&id, &blogID, &title, &url, &publishedDate, &discovered, &isRead, &keywords, &description); err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, nil } @@ -468,11 +478,13 @@ func scanArticle(scanner interface{ Scan(dest ...any) error }) (*model.Article, } article := &model.Article{ - ID: id, - BlogID: blogID, - Title: title, - URL: url, - IsRead: isRead, + ID: id, + BlogID: blogID, + Title: title, + URL: url, + IsRead: isRead, + Keywords: keywords.String, + Description: description.String, } if publishedDate.Valid { if parsed, err := parseTime(publishedDate.String); err == nil { From fb0090c75af16b465d2c50c0f9ee36da8f72d417 Mon Sep 17 00:00:00 2001 From: hermes-lol Date: Tue, 5 May 2026 11:35:30 +0000 Subject: [PATCH 2/2] feat: display Keywords and Description in articles list --- internal/cli/commands.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/internal/cli/commands.go b/internal/cli/commands.go index e56c186..a6c69ca 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -392,6 +392,17 @@ func printArticle(article model.Article, blogName string) { if article.PublishedDate != nil { fmt.Printf(" Published: %s\n", article.PublishedDate.Format("2006-01-02")) } + if article.Keywords != "" { + fmt.Printf(" Keywords: %s\n", article.Keywords) + } + if article.Description != "" { + // Truncate description if too long + desc := article.Description + if len(desc) > 100 { + desc = desc[:100] + "..." + } + fmt.Printf(" Description: %s\n", desc) + } fmt.Println() }