Please enter the commit message for your changes. Lines starting
with '#' will be ignored, and an empty message aborts the commit. On branch main Initial commit Changes to be committed: new file: .DS_Store new file: .env new file: .gitignore new file: ai-worker/Dockerfile new file: ai-worker/requirements.txt new file: ai-worker/worker.py new file: background-worker/Dockerfile new file: background-worker/go.mod new file: background-worker/go.sum new file: background-worker/main.go new file: background-worker/market.go new file: background-worker/rmv.go new file: background-worker/rss.go new file: background-worker/sql_work.go new file: db/Dockerfile new file: db/init.sql new file: docker-compose.yml new file: server-app/dockerfile new file: server-app/go.mod new file: server-app/go.sum new file: server-app/main.go new file: volumes/.DS_Store new file: volumes/db-init/.DS_Store new file: volumes/db-init/data/news_rss_feeds.csv new file: volumes/web/.DS_Store new file: volumes/web/static/css/blog.css new file: volumes/web/static/css/index-lite.css new file: volumes/web/static/css/index.css new file: volumes/web/static/css/mandelbrot.css new file: volumes/web/static/img/minecraft.png new file: volumes/web/static/js/blog.js new file: volumes/web/static/js/index-lite.js new file: volumes/web/static/js/index.js new file: volumes/web/static/js/mandelbrot.js new file: volumes/web/static/media/cantina.mp3 new file: volumes/web/static/media/countdowns.json new file: volumes/web/static/media/gong.mp4 new file: volumes/web/template/blog.html new file: volumes/web/template/index-lite.html new file: volumes/web/template/index.html new file: volumes/web/template/mandelbrot.html
This commit is contained in:
hubble_dubble
9
background-worker/Dockerfile
Normal file
9
background-worker/Dockerfile
Normal file
@@ -0,0 +1,9 @@
|
||||
FROM golang:1.24
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
COPY . .
|
||||
CMD ["go", "run", "."]
|
||||
23
background-worker/go.mod
Normal file
23
background-worker/go.mod
Normal file
@@ -0,0 +1,23 @@
|
||||
module background-worker
|
||||
|
||||
go 1.24.0
|
||||
|
||||
toolchain go1.24.12
|
||||
|
||||
require (
|
||||
github.com/jackc/pgx/v5 v5.8.0
|
||||
github.com/mmcdole/gofeed v1.3.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.8.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.1 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
golang.org/x/net v0.4.0 // indirect
|
||||
golang.org/x/text v0.29.0 // indirect
|
||||
)
|
||||
50
background-worker/go.sum
Normal file
50
background-worker/go.sum
Normal file
@@ -0,0 +1,50 @@
|
||||
github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
|
||||
github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
|
||||
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
|
||||
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.8.0 h1:TYPDoleBBme0xGSAX3/+NujXXtpZn9HBONkQC7IEZSo=
|
||||
github.com/jackc/pgx/v5 v5.8.0/go.mod h1:QVeDInX2m9VyzvNeiCJVjCkNFqzsNb43204HshNSZKw=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4=
|
||||
github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE=
|
||||
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk=
|
||||
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU=
|
||||
golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
|
||||
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
|
||||
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
|
||||
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
33
background-worker/main.go
Normal file
33
background-worker/main.go
Normal file
@@ -0,0 +1,33 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
log.Println("[background-worker] started")
|
||||
|
||||
tickerRMV := time.NewTicker(5 * time.Minute)
|
||||
tickerRSS := time.NewTicker(1 * time.Minute)
|
||||
tickerMarket := time.NewTicker(10 * time.Second)
|
||||
defer tickerRMV.Stop()
|
||||
defer tickerRSS.Stop()
|
||||
defer tickerMarket.Stop()
|
||||
|
||||
rmv_request()
|
||||
fetchMarketData()
|
||||
fetchRSSFeeds()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-tickerRMV.C:
|
||||
go rmv_request()
|
||||
case <-tickerRSS.C:
|
||||
go fetchRSSFeeds()
|
||||
case <-tickerMarket.C:
|
||||
go fetchMarketData()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
87
background-worker/market.go
Normal file
87
background-worker/market.go
Normal file
@@ -0,0 +1,87 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
type marketQuote struct {
|
||||
Instrument string
|
||||
Bid float64
|
||||
QuotedAt time.Time
|
||||
}
|
||||
|
||||
type sqResponse struct {
|
||||
SpreadProfilePrices []struct {
|
||||
Bid float64 `json:"bid"`
|
||||
} `json:"spreadProfilePrices"`
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
}
|
||||
|
||||
func fetchMarketData() {
|
||||
ctx := context.Background()
|
||||
client := http.Client{Timeout: 10 * time.Second}
|
||||
|
||||
type endpoint struct {
|
||||
Instrument string
|
||||
URL string
|
||||
}
|
||||
endpoints := []endpoint{
|
||||
{Instrument: "USD/JPY", URL: "https://forex-data-feed.swissquote.com/public-quotes/bboquotes/instrument/USD/JPY"},
|
||||
{Instrument: "OIL/USD", URL: "https://forex-data-feed.swissquote.com/public-quotes/bboquotes/instrument/OIL/USD"},
|
||||
{Instrument: "XAU/USD", URL: "https://forex-data-feed.swissquote.com/public-quotes/bboquotes/instrument/XAU/USD"},
|
||||
{Instrument: "EUR/USD", URL: "https://forex-data-feed.swissquote.com/public-quotes/bboquotes/instrument/EUR/USD"},
|
||||
{Instrument: "USD/CHF", URL: "https://forex-data-feed.swissquote.com/public-quotes/bboquotes/instrument/USD/CHF"},
|
||||
{Instrument: "GBP/USD", URL: "https://forex-data-feed.swissquote.com/public-quotes/bboquotes/instrument/GBP/USD"},
|
||||
}
|
||||
|
||||
var quotes []MarketQuote
|
||||
|
||||
for _, ep := range endpoints {
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, ep.URL, nil)
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
log.Printf("market fetch %s failed: %v", ep.Instrument, err)
|
||||
continue
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
resp.Body.Close()
|
||||
log.Printf("market fetch %s bad status: %s", ep.Instrument, resp.Status)
|
||||
continue
|
||||
}
|
||||
|
||||
var payload []sqResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil {
|
||||
resp.Body.Close()
|
||||
log.Printf("decode %s failed: %v", ep.Instrument, err)
|
||||
continue
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
if len(payload) == 0 || len(payload[0].SpreadProfilePrices) == 0 {
|
||||
log.Printf("market fetch %s: empty payload", ep.Instrument)
|
||||
continue
|
||||
}
|
||||
|
||||
bid := payload[0].SpreadProfilePrices[0].Bid
|
||||
quotedAt := time.UnixMilli(payload[0].Timestamp)
|
||||
quotes = append(quotes, MarketQuote{
|
||||
Instrument: ep.Instrument,
|
||||
Bid: bid,
|
||||
QuotedAt: quotedAt,
|
||||
})
|
||||
}
|
||||
|
||||
if len(quotes) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
if err := saveMarketQuotes(ctx, quotes); err != nil {
|
||||
log.Printf("saveMarketQuotes failed: %v", err)
|
||||
}
|
||||
|
||||
log.Printf("market quotes updated: %d instruments", len(quotes))
|
||||
}
|
||||
108
background-worker/rmv.go
Normal file
108
background-worker/rmv.go
Normal file
@@ -0,0 +1,108 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
)
|
||||
|
||||
type RMVResponse struct {
|
||||
Trip []Trip `json:"Trip"`
|
||||
}
|
||||
|
||||
type Trip struct {
|
||||
LegList LegList `json:"LegList"`
|
||||
}
|
||||
|
||||
type LegList struct {
|
||||
Leg []Leg `json:"Leg"`
|
||||
}
|
||||
|
||||
type Leg struct {
|
||||
Type string `json:"type"`
|
||||
Name string `json:"name"`
|
||||
Origin Stop `json:"Origin"`
|
||||
Destination Stop `json:"Destination"`
|
||||
}
|
||||
|
||||
type Stop struct {
|
||||
Name string `json:"name"`
|
||||
Time string `json:"time"`
|
||||
RtTime string `json:"rtTime"`
|
||||
}
|
||||
|
||||
type Verbindung struct {
|
||||
Linie string
|
||||
Abfahrt string
|
||||
Ankunft string
|
||||
Von string
|
||||
Nach string
|
||||
}
|
||||
|
||||
func rmv_request() {
|
||||
u, _ := url.Parse("https://www.rmv.de/hapi/trip")
|
||||
q := u.Query()
|
||||
q.Set("originId", "3018009")
|
||||
q.Set("destId", "3029164")
|
||||
q.Set("format", "json")
|
||||
q.Set("accessId", os.Getenv("RMV_API_KEY"))
|
||||
u.RawQuery = q.Encode()
|
||||
|
||||
resp, err := http.Get(u.String())
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var respData RMVResponse
|
||||
|
||||
dec := json.NewDecoder(resp.Body)
|
||||
if err := dec.Decode(&respData); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
var verbindungen [][]Verbindung
|
||||
|
||||
for _, trip := range respData.Trip {
|
||||
var tripInfo []Verbindung
|
||||
|
||||
for _, leg := range trip.LegList.Leg {
|
||||
if leg.Type == "WALK" {
|
||||
continue
|
||||
}
|
||||
|
||||
abfahrt := leg.Origin.RtTime
|
||||
if abfahrt == "" {
|
||||
abfahrt = leg.Origin.Time
|
||||
}
|
||||
|
||||
ankunft := leg.Destination.RtTime
|
||||
if ankunft == "" {
|
||||
ankunft = leg.Destination.Time
|
||||
}
|
||||
|
||||
tripInfo = append(tripInfo, Verbindung{
|
||||
Linie: leg.Name,
|
||||
Abfahrt: abfahrt,
|
||||
Ankunft: ankunft,
|
||||
Von: leg.Origin.Name,
|
||||
Nach: leg.Destination.Name,
|
||||
})
|
||||
}
|
||||
|
||||
if len(tripInfo) > 0 {
|
||||
verbindungen = append(verbindungen, tripInfo)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
writeVerbindungToSQL(verbindungen)
|
||||
|
||||
log.Println("RMV data updated")
|
||||
|
||||
if err := writeVerbindungToSQL(verbindungen); err != nil {
|
||||
log.Printf("failed to persist RMV data: %v", err)
|
||||
}
|
||||
}
|
||||
66
background-worker/rss.go
Normal file
66
background-worker/rss.go
Normal file
@@ -0,0 +1,66 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/mmcdole/gofeed"
|
||||
)
|
||||
|
||||
func fetchRSSFeeds() {
|
||||
ctx := context.Background()
|
||||
|
||||
feeds, err := fetchPendingFeeds(ctx)
|
||||
if err != nil {
|
||||
log.Printf("fetchPendingFeeds failed: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(feeds) == 0 {
|
||||
log.Println("no feeds due for refresh")
|
||||
return
|
||||
}
|
||||
|
||||
parser := gofeed.NewParser()
|
||||
|
||||
for _, feed := range feeds {
|
||||
fp, err := parser.ParseURL(feed.URL)
|
||||
if err != nil {
|
||||
log.Printf("parse feed %s: %v", feed.URL, err)
|
||||
if err := setFeedAccess(ctx, feed.ID, false); err != nil {
|
||||
log.Printf("disable feed %d failed: %v", feed.ID, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
var articles []Article
|
||||
for _, item := range fp.Items {
|
||||
var published *time.Time
|
||||
if item.PublishedParsed != nil {
|
||||
published = item.PublishedParsed
|
||||
}
|
||||
|
||||
summary := item.Description
|
||||
if summary == "" {
|
||||
summary = item.Content
|
||||
}
|
||||
|
||||
articles = append(articles, Article{
|
||||
ArticleID: articleIDFromLink(item.Link),
|
||||
FeedID: feed.ID,
|
||||
Title: item.Title,
|
||||
Link: item.Link,
|
||||
Summary: summary,
|
||||
PublishedAt: published,
|
||||
})
|
||||
}
|
||||
|
||||
if err := saveArticles(ctx, feed.ID, articles); err != nil {
|
||||
log.Printf("save articles for feed %d: %v", feed.ID, err)
|
||||
continue
|
||||
}
|
||||
|
||||
log.Printf("processed feed %d (%s) with %d items", feed.ID, feed.URL, len(articles))
|
||||
}
|
||||
}
|
||||
217
background-worker/sql_work.go
Normal file
217
background-worker/sql_work.go
Normal file
@@ -0,0 +1,217 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
func getenv(k, fallback string) string {
|
||||
if v := os.Getenv(k); v != "" {
|
||||
return v
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func buildDSN() string {
|
||||
if url := os.Getenv("DATABASE_URL"); url != "" {
|
||||
return url
|
||||
}
|
||||
|
||||
host := getenv("POSTGRES_HOST", "database")
|
||||
port := getenv("POSTGRES_PORT", "5432")
|
||||
user := getenv("POSTGRES_USER", "app")
|
||||
pass := getenv("POSTGRES_PASSWORD", "appsecret")
|
||||
name := getenv("POSTGRES_DB", "appdb")
|
||||
|
||||
return fmt.Sprintf("postgres://%s:%s@%s:%s/%s?sslmode=disable", user, pass, host, port, name)
|
||||
}
|
||||
|
||||
type FeedMeta struct {
|
||||
ID int
|
||||
URL string
|
||||
}
|
||||
|
||||
type Article struct {
|
||||
ArticleID string
|
||||
FeedID int
|
||||
Title string
|
||||
Link string
|
||||
Summary string
|
||||
PublishedAt *time.Time
|
||||
}
|
||||
|
||||
type MarketQuote struct {
|
||||
Instrument string
|
||||
Bid float64
|
||||
QuotedAt time.Time
|
||||
}
|
||||
|
||||
func writeVerbindungToSQL(verbindungen [][]Verbindung) error {
|
||||
ctx := context.Background()
|
||||
dsn := buildDSN()
|
||||
|
||||
conn, err := pgx.Connect(ctx, dsn)
|
||||
if err != nil {
|
||||
return fmt.Errorf("connect db: %w", err)
|
||||
}
|
||||
defer conn.Close(ctx)
|
||||
|
||||
tx, err := conn.Begin(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin tx: %w", err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
if _, err := tx.Exec(ctx, "DELETE FROM rmv_data"); err != nil {
|
||||
return fmt.Errorf("clear rmv_data: %w", err)
|
||||
}
|
||||
|
||||
stmt := `INSERT INTO rmv_data (trip_index, leg_index, linie, abfahrt, ankunft, von, nach) VALUES ($1,$2,$3,$4,$5,$6,$7)`
|
||||
|
||||
for tripIdx, trip := range verbindungen {
|
||||
for legIdx, leg := range trip {
|
||||
if _, err := tx.Exec(ctx, stmt, tripIdx, legIdx, leg.Linie, leg.Abfahrt, leg.Ankunft, leg.Von, leg.Nach); err != nil {
|
||||
return fmt.Errorf("insert trip %d leg %d: %w", tripIdx, legIdx, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("commit tx: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func fetchPendingFeeds(ctx context.Context) ([]FeedMeta, error) {
|
||||
dsn := buildDSN()
|
||||
conn, err := pgx.Connect(ctx, dsn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("connect db: %w", err)
|
||||
}
|
||||
defer conn.Close(ctx)
|
||||
|
||||
rows, err := conn.Query(ctx, `
|
||||
SELECT id, url
|
||||
FROM feeds
|
||||
WHERE access = TRUE
|
||||
AND (last_checked IS NULL OR last_checked < now() - interval '5 minutes')
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query feeds: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var feeds []FeedMeta
|
||||
for rows.Next() {
|
||||
var f FeedMeta
|
||||
if err := rows.Scan(&f.ID, &f.URL); err != nil {
|
||||
return nil, fmt.Errorf("scan feed: %w", err)
|
||||
}
|
||||
feeds = append(feeds, f)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate feeds: %w", err)
|
||||
}
|
||||
|
||||
return feeds, nil
|
||||
}
|
||||
|
||||
func articleIDFromLink(link string) string {
|
||||
h := sha1.Sum([]byte(link))
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
|
||||
func saveArticles(ctx context.Context, feedID int, articles []Article) error {
|
||||
dsn := buildDSN()
|
||||
conn, err := pgx.Connect(ctx, dsn)
|
||||
if err != nil {
|
||||
return fmt.Errorf("connect db: %w", err)
|
||||
}
|
||||
defer conn.Close(ctx)
|
||||
|
||||
tx, err := conn.Begin(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin tx: %w", err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
stmt := `
|
||||
INSERT INTO articles (article_id, feed_id, title, link, summary, published_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
ON CONFLICT (article_id) DO NOTHING
|
||||
`
|
||||
|
||||
for _, a := range articles {
|
||||
aid := a.ArticleID
|
||||
if aid == "" {
|
||||
aid = articleIDFromLink(a.Link)
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(ctx, stmt, aid, feedID, a.Title, a.Link, a.Summary, a.PublishedAt); err != nil {
|
||||
return fmt.Errorf("insert article for feed %d: %w", feedID, err)
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(ctx, `UPDATE feeds SET last_checked = now() WHERE id = $1`, feedID); err != nil {
|
||||
return fmt.Errorf("update feed timestamp: %w", err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("commit tx: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func setFeedAccess(ctx context.Context, feedID int, access bool) error {
|
||||
dsn := buildDSN()
|
||||
conn, err := pgx.Connect(ctx, dsn)
|
||||
if err != nil {
|
||||
return fmt.Errorf("connect db: %w", err)
|
||||
}
|
||||
defer conn.Close(ctx)
|
||||
|
||||
if _, err := conn.Exec(ctx, `UPDATE feeds SET access = $1 WHERE id = $2`, access, feedID); err != nil {
|
||||
return fmt.Errorf("update feed access: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func saveMarketQuotes(ctx context.Context, quotes []MarketQuote) error {
|
||||
dsn := buildDSN()
|
||||
conn, err := pgx.Connect(ctx, dsn)
|
||||
if err != nil {
|
||||
return fmt.Errorf("connect db: %w", err)
|
||||
}
|
||||
defer conn.Close(ctx)
|
||||
|
||||
batch := &pgx.Batch{}
|
||||
for _, q := range quotes {
|
||||
batch.Queue(
|
||||
`INSERT INTO market_quotes (instrument, bid, quoted_at)
|
||||
VALUES ($1, $2, $3)
|
||||
ON CONFLICT (instrument) DO UPDATE
|
||||
SET bid = EXCLUDED.bid, quoted_at = EXCLUDED.quoted_at`,
|
||||
q.Instrument, q.Bid, q.QuotedAt,
|
||||
)
|
||||
}
|
||||
|
||||
br := conn.SendBatch(ctx, batch)
|
||||
defer br.Close()
|
||||
|
||||
for range quotes {
|
||||
if _, err := br.Exec(); err != nil {
|
||||
return fmt.Errorf("insert market quote: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user