3667c678e4
with '#' will be ignored, and an empty message aborts the commit. On branch main Initial commit Changes to be committed: new file: .DS_Store new file: .env new file: .gitignore new file: ai-worker/Dockerfile new file: ai-worker/requirements.txt new file: ai-worker/worker.py new file: background-worker/Dockerfile new file: background-worker/go.mod new file: background-worker/go.sum new file: background-worker/main.go new file: background-worker/market.go new file: background-worker/rmv.go new file: background-worker/rss.go new file: background-worker/sql_work.go new file: db/Dockerfile new file: db/init.sql new file: docker-compose.yml new file: server-app/dockerfile new file: server-app/go.mod new file: server-app/go.sum new file: server-app/main.go new file: volumes/.DS_Store new file: volumes/db-init/.DS_Store new file: volumes/db-init/data/news_rss_feeds.csv new file: volumes/web/.DS_Store new file: volumes/web/static/css/blog.css new file: volumes/web/static/css/index-lite.css new file: volumes/web/static/css/index.css new file: volumes/web/static/css/mandelbrot.css new file: volumes/web/static/img/minecraft.png new file: volumes/web/static/js/blog.js new file: volumes/web/static/js/index-lite.js new file: volumes/web/static/js/index.js new file: volumes/web/static/js/mandelbrot.js new file: volumes/web/static/media/cantina.mp3 new file: volumes/web/static/media/countdowns.json new file: volumes/web/static/media/gong.mp4 new file: volumes/web/template/blog.html new file: volumes/web/template/index-lite.html new file: volumes/web/template/index.html new file: volumes/web/template/mandelbrot.html
67 lines
1.3 KiB
Go
67 lines
1.3 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"log"
|
|
"time"
|
|
|
|
"github.com/mmcdole/gofeed"
|
|
)
|
|
|
|
func fetchRSSFeeds() {
|
|
ctx := context.Background()
|
|
|
|
feeds, err := fetchPendingFeeds(ctx)
|
|
if err != nil {
|
|
log.Printf("fetchPendingFeeds failed: %v", err)
|
|
return
|
|
}
|
|
|
|
if len(feeds) == 0 {
|
|
log.Println("no feeds due for refresh")
|
|
return
|
|
}
|
|
|
|
parser := gofeed.NewParser()
|
|
|
|
for _, feed := range feeds {
|
|
fp, err := parser.ParseURL(feed.URL)
|
|
if err != nil {
|
|
log.Printf("parse feed %s: %v", feed.URL, err)
|
|
if err := setFeedAccess(ctx, feed.ID, false); err != nil {
|
|
log.Printf("disable feed %d failed: %v", feed.ID, err)
|
|
}
|
|
continue
|
|
}
|
|
|
|
var articles []Article
|
|
for _, item := range fp.Items {
|
|
var published *time.Time
|
|
if item.PublishedParsed != nil {
|
|
published = item.PublishedParsed
|
|
}
|
|
|
|
summary := item.Description
|
|
if summary == "" {
|
|
summary = item.Content
|
|
}
|
|
|
|
articles = append(articles, Article{
|
|
ArticleID: articleIDFromLink(item.Link),
|
|
FeedID: feed.ID,
|
|
Title: item.Title,
|
|
Link: item.Link,
|
|
Summary: summary,
|
|
PublishedAt: published,
|
|
})
|
|
}
|
|
|
|
if err := saveArticles(ctx, feed.ID, articles); err != nil {
|
|
log.Printf("save articles for feed %d: %v", feed.ID, err)
|
|
continue
|
|
}
|
|
|
|
log.Printf("processed feed %d (%s) with %d items", feed.ID, feed.URL, len(articles))
|
|
}
|
|
}
|