Skip to content

add 36kr news command: latest articles from RSS feed #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tamnd wants to merge 1 commit into
base: main
Choose a base branch
from
Open

add 36kr news command: latest articles from RSS feed #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cli/cmd_news.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (
func (a *App) newsCmd() *cobra.Command {
return &cobra.Command{
Use: "news",
Short: "List the latest 36kr articles",
Short: "List the latest articles from 36kr",
RunE: func(cmd *cobra.Command, _ []string) error {
n := a.effectiveLimit(20)
a.progressf("fetching %d articles from 36kr RSS...", n)
Expand Down
10 changes: 5 additions & 5 deletions kr36/kr36.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,11 +104,11 @@ func (c *Client) News(ctx context.Context, limit int) ([]Article, error) {
out := make([]Article, 0, len(items))
for i, it := range items {
out = append(out, Article{
Rank: i + 1,
Title: strings.TrimSpace(it.Title),
Summary: stripHTML(it.Description),
PubDate: parsePubDate(it.PubDate),
URL: strings.TrimSpace(it.Link),
Rank: i + 1,
Title: strings.TrimSpace(it.Title),
Summary: stripHTML(it.Description),
Published: parsePubDate(it.PubDate),
URL: strings.TrimSpace(it.Link),
})
}
return out, nil
Expand Down
290 changes: 179 additions & 111 deletions kr36/kr36_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import (
"context"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"

Expand All @@ -12,167 +14,233 @@ import (

const mockRSS = `<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0">
<channel>
<title>36氪</title>
<link>http://36kr.com</link>
<item>
<title>曼联，要被卖了</title>
<link><![CDATA[https://36kr.com/p/3200000001]]></link>
<pubDate>2026-06-13 16:37:21 +0800</pubDate>
<description><![CDATA[<p>曼联俱乐部<b>正式宣布</b>出售，估值超过50亿英镑。</p>]]></description>
</item>
<item>
<title>OpenAI再融资</title>
<link><![CDATA[https://36kr.com/p/3200000002]]></link>
<pubDate>2026-06-12 10:00:00 +0800</pubDate>
<description><![CDATA[OpenAI完成新一轮融资，估值超过3000亿美元。]]></description>
</item>
<item>
<title>字节跳动出海</title>
<link><![CDATA[https://36kr.com/p/3200000003]]></link>
<pubDate>2026-06-11 09:00:00 +0800</pubDate>
<description><![CDATA[字节跳动加速<em>全球化</em>布局。]]></description>
</item>
<item>
<title>华为最新发布</title>
<link><![CDATA[https://36kr.com/p/3200000004]]></link>
<pubDate>2026-06-10 08:00:00 +0800</pubDate>
<description><![CDATA[华为发布最新旗舰手机。]]></description>
</item>
<item>
<title>比亚迪销量新高</title>
<link><![CDATA[https://36kr.com/p/3200000005]]></link>
<pubDate>2026-06-09 07:00:00 +0800</pubDate>
<description><![CDATA[比亚迪创下单月销量新纪录。]]></description>
</item>
</channel>
<channel>
<title>36Kr</title>
<description>36氪最新文章</description>
<item>
<title>OpenAI raises $6.6 billion</title>
<link><![CDATA[https://36kr.com/p/1234567890]]></link>
<guid>https://36kr.com/p/1234567890</guid>
<pubDate>2024-03-15 10:30:00 +0800</pubDate>
<description><![CDATA[<p><b>OpenAI</b> announced a <em>record</em> funding round today.</p>]]></description>
</item>
<item>
<title>BYD surpasses Tesla in Q1 deliveries</title>
<link><![CDATA[https://36kr.com/p/9876543210]]></link>
<guid>https://36kr.com/p/9876543210</guid>
<pubDate>2024-03-14 08:00:00 +0800</pubDate>
<description><![CDATA[<p>BYD delivered 300,000 vehicles in Q1 2024.</p>]]></description>
</item>
<item>
<title>Third article title</title>
<link><![CDATA[https://36kr.com/p/1111111111]]></link>
<guid>https://36kr.com/p/1111111111</guid>
<pubDate>2024-03-13 12:00:00 +0800</pubDate>
<description><![CDATA[Third article summary text.]]></description>
</item>
</channel>
</rss>`

const emptyRSS = `<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0">
<channel>
<title>36Kr</title>
<description>36氪最新文章</description>
</channel>
</rss>`

func newTestClient(ts *httptest.Server) *kr36.Client {
cfg := kr36.DefaultConfig()
cfg.BaseURL = ts.URL
cfg.Rate = 0
return kr36.NewClient(cfg)
return kr36.NewClient(kr36.Config{
BaseURL: ts.URL,
UserAgent: "test-agent/1.0",
Rate: 0,
Timeout: 5 * time.Second,
Retries: 0,
})
}

func TestNewsSendsUserAgent(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("User-Agent") == "" {
t.Error("request carried no User-Agent")
}
func TestNewsParsesItems(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/rss+xml")
_, _ = w.Write([]byte(mockRSS))
}))
defer srv.Close()

c := newTestClient(srv)
_, err := c.News(context.Background(), 5)
defer ts.Close()
c := newTestClient(ts)
articles, err := c.News(context.Background(), 0)
if err != nil {
t.Fatal(err)
}
if len(articles) != 3 {
t.Fatalf("want 3 articles, got %d", len(articles))
}
a := articles[0]
if a.Rank != 1 {
t.Errorf("rank: want 1, got %d", a.Rank)
}
if a.Title != "OpenAI raises $6.6 billion" {
t.Errorf("title: got %q", a.Title)
}
if a.URL != "https://36kr.com/p/1234567890" {
t.Errorf("url: got %q", a.URL)
}
if a.Published != "2024-03-15" {
t.Errorf("published: got %q, want 2024-03-15", a.Published)
}
}

func TestNewsParsesItems(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
func TestNewsLimitRespected(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(mockRSS))
}))
defer srv.Close()

c := newTestClient(srv)
arts, err := c.News(context.Background(), 0)
defer ts.Close()
c := newTestClient(ts)
articles, err := c.News(context.Background(), 2)
if err != nil {
t.Fatal(err)
}
if len(arts) != 5 {
t.Fatalf("got %d articles, want 5", len(arts))
if len(articles) != 2 {
t.Fatalf("want 2 articles, got %d", len(articles))
}

a := arts[0]
if a.Rank != 1 {
t.Errorf("rank = %d, want 1", a.Rank)
if articles[1].Rank != 2 {
t.Errorf("articles[1].Rank = %d, want 2", articles[1].Rank)
}
if a.Title != "曼联，要被卖了" {
t.Errorf("title = %q", a.Title)
}

func TestNewsHTMLStripped(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(mockRSS))
}))
defer ts.Close()
c := newTestClient(ts)
articles, err := c.News(context.Background(), 1)
if err != nil {
t.Fatal(err)
}
if a.URL != "https://36kr.com/p/3200000001" {
t.Errorf("url = %q", a.URL)
summary := articles[0].Summary
if strings.Contains(summary, "<") || strings.Contains(summary, ">") {
t.Errorf("HTML not stripped from summary: %q", summary)
}
if a.PubDate != "2026-06-13" {
t.Errorf("pub_date = %q, want 2026-06-13", a.PubDate)
if !strings.Contains(summary, "OpenAI") {
t.Errorf("expected text content in summary, got: %q", summary)
}
}

func TestNewsLimitRespected(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
func TestNewsSendsUserAgent(t *testing.T) {
var gotUA string
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotUA = r.Header.Get("User-Agent")
_, _ = w.Write([]byte(mockRSS))
}))
defer srv.Close()

c := newTestClient(srv)
arts, err := c.News(context.Background(), 3)
defer ts.Close()
c := newTestClient(ts)
_, err := c.News(context.Background(), 0)
if err != nil {
t.Fatal(err)
}
if len(arts) != 3 {
t.Fatalf("got %d articles, want 3", len(arts))
if gotUA == "" {
t.Error("User-Agent header not sent")
}
}

func TestNewsRetriesOn503(t *testing.T) {
var hits int
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits++
if hits < 3 {
w.WriteHeader(http.StatusServiceUnavailable)
var calls int32
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
n := atomic.AddInt32(&calls, 1)
if n < 3 {
w.WriteHeader(503)
return
}
_, _ = w.Write([]byte(mockRSS))
}))
defer srv.Close()

cfg := kr36.DefaultConfig()
cfg.BaseURL = srv.URL
cfg.Rate = 0
cfg.Retries = 5
c := kr36.NewClient(cfg)
defer ts.Close()
c := kr36.NewClient(kr36.Config{
BaseURL: ts.URL, UserAgent: "test", Rate: 0, Timeout: 5 * time.Second, Retries: 3,
})
articles, err := c.News(context.Background(), 0)
if err != nil {
t.Fatal(err)
}
if len(articles) == 0 {
t.Error("expected articles after retries")
}
if atomic.LoadInt32(&calls) < 3 {
t.Errorf("expected at least 3 calls, got %d", calls)
}
}

start := time.Now()
_, err := c.News(context.Background(), 5)
func TestNewsRanksAssigned(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(mockRSS))
}))
defer ts.Close()
c := newTestClient(ts)
articles, err := c.News(context.Background(), 0)
if err != nil {
t.Fatal(err)
}
if hits != 3 {
t.Errorf("server saw %d hits, want 3", hits)
for i, a := range articles {
if a.Rank != i+1 {
t.Errorf("articles[%d].Rank = %d, want %d", i, a.Rank, i+1)
}
}
if time.Since(start) < 500*time.Millisecond {
t.Error("retries did not back off")
}

func TestNewsEmptyFeed(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(emptyRSS))
}))
defer ts.Close()
c := newTestClient(ts)
articles, err := c.News(context.Background(), 0)
if err != nil {
t.Fatal(err)
}
if len(articles) != 0 {
t.Fatalf("want 0 articles, got %d", len(articles))
}
}

func TestNewsHTMLStripped(t *testing.T) {
body := `<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0"><channel><title>36氪</title><link>http://36kr.com</link>
<item>
<title>Test</title>
<link><![CDATA[https://36kr.com/p/test]]></link>
<pubDate>2026-06-14 10:00:00 +0800</pubDate>
<description><![CDATA[<b>bold</b> text]]></description>
</item>
</channel></rss>`
func TestNewsContextCancelled(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(10 * time.Millisecond)
_, _ = w.Write([]byte(mockRSS))
}))
defer ts.Close()
ctx, cancel := context.WithCancel(context.Background())
cancel() // cancel immediately
c := newTestClient(ts)
_, err := c.News(ctx, 0)
if err == nil {
t.Fatal("expected error when context is cancelled, got nil")
}
}

srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(body))
func TestNewsBadXML(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`{"not":"xml"}`))
}))
defer srv.Close()
defer ts.Close()
c := newTestClient(ts)
_, err := c.News(context.Background(), 0)
if err == nil {
t.Fatal("expected error for bad XML, got nil")
}
if !strings.Contains(err.Error(), "parse") {
t.Errorf("error should mention parse failure, got: %v", err)
}
}

c := newTestClient(srv)
arts, err := c.News(context.Background(), 1)
func TestNewsLimitZero(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(mockRSS))
}))
defer ts.Close()
c := newTestClient(ts)
articles, err := c.News(context.Background(), 0)
if err != nil {
t.Fatal(err)
}
if len(arts) != 1 {
t.Fatalf("got %d articles, want 1", len(arts))
}
if arts[0].Summary != "bold text" {
t.Errorf("summary = %q, want %q", arts[0].Summary, "bold text")
if len(articles) != 3 {
t.Fatalf("limit 0 should return all 3 articles, got %d", len(articles))
}
}
Loading
Loading