Changes to the compse yml and the way and amout of artikeltexte are delivered and taht not duplicates are displayed

This commit is contained in:
hubble_dubble
2026-01-26 00:59:03 +00:00
parent 08a1f54a40
commit 71ff9344a1
4 changed files with 89 additions and 13 deletions

View File

@@ -3,6 +3,8 @@ services:
build: ./db build: ./db
image: bunker-database:latest image: bunker-database:latest
restart: unless-stopped restart: unless-stopped
networks:
- backend
environment: environment:
POSTGRES_USER: ${POSTGRES_USER} POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
@@ -15,10 +17,32 @@ services:
- ./volumes/db-init/data:/docker-entrypoint-initdb.d/data:ro - ./volumes/db-init/data:/docker-entrypoint-initdb.d/data:ro
- ./volumes/data:/data - ./volumes/data:/data
server-app: server-app-blue:
build: ./server-app build: ./server-app
image: bunker-server-app:latest image: bunker-server-app:latest
container_name: server-app-blue
restart: unless-stopped restart: unless-stopped
networks:
- proxy
- backend
environment:
APP_PORT: ${APP_PORT}
DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@database:5432/${POSTGRES_DB}?sslmode=disable
expose:
- "8080"
volumes:
- ./volumes/web:/app/web
depends_on:
- database
server-app-green:
build: ./server-app
image: bunker-server-app:latest
container_name: server-app-green
restart: unless-stopped
networks:
- proxy
- backend
environment: environment:
APP_PORT: ${APP_PORT} APP_PORT: ${APP_PORT}
DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@database:5432/${POSTGRES_DB}?sslmode=disable DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@database:5432/${POSTGRES_DB}?sslmode=disable
@@ -33,6 +57,8 @@ services:
build: ./ai-worker build: ./ai-worker
image: bunker-ai-worker:latest image: bunker-ai-worker:latest
restart: unless-stopped restart: unless-stopped
networks:
- backend
environment: environment:
DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@database:5432/${POSTGRES_DB}?sslmode=disable DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@database:5432/${POSTGRES_DB}?sslmode=disable
SD_MODEL_PATH: /app/models/sd15 SD_MODEL_PATH: /app/models/sd15
@@ -47,12 +73,16 @@ services:
build: ./background-worker build: ./background-worker
image: bunker-background-worker:latest image: bunker-background-worker:latest
restart: unless-stopped restart: unless-stopped
networks:
- backend
environment: environment:
DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@database:5432/${POSTGRES_DB}?sslmode=disable DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@database:5432/${POSTGRES_DB}?sslmode=disable
RMV_API_KEY: ${RMV_API_KEY} RMV_API_KEY: ${RMV_API_KEY}
depends_on: depends_on:
- database - database
volumes: networks:
caddy_data: proxy:
caddy_config: external: true
backend:
driver: bridge

View File

@@ -206,7 +206,7 @@ func main() {
if err := withConn(r, func(conn *pgx.Conn) error { if err := withConn(r, func(conn *pgx.Conn) error {
rows, err := conn.Query(r.Context(), ` rows, err := conn.Query(r.Context(), `
SELECT title, link, summary, image, published_at SELECT article_id, title, link, summary, image, published_at
FROM articles FROM articles
WHERE image IS NOT NULL WHERE image IS NOT NULL
ORDER BY COALESCE(published_at, created_at) DESC ORDER BY COALESCE(published_at, created_at) DESC
@@ -238,16 +238,40 @@ func main() {
w.Write([]byte("\n")) w.Write([]byte("\n"))
flusher.Flush() flusher.Flush()
seen := make(map[string]struct{})
rowCount := 0
for rows.Next() { for rows.Next() {
rowCount++
var articleID *string
var title, link, summary string var title, link, summary string
var image *string var image *string
var published *time.Time var published *time.Time
if err := rows.Scan(&title, &link, &summary, &image, &published); err != nil { if err := rows.Scan(&articleID, &title, &link, &summary, &image, &published); err != nil {
return err return err
} }
id := ""
if articleID != nil {
id = strings.TrimSpace(*articleID)
}
if id == "" {
if link != "" {
id = link
} else if title != "" {
id = title
} else {
id = fmt.Sprintf("row:%d", rowCount)
}
}
if _, exists := seen[id]; exists {
continue
}
seen[id] = struct{}{}
item := map[string]interface{}{ item := map[string]interface{}{
"id": id,
"title": title, "title": title,
"link": link, "link": link,
"text": summary, "text": summary,

View File

@@ -38,10 +38,16 @@ async function loadNews() {
const lines = text.split("\n").map((line) => line.trim()).filter(Boolean); const lines = text.split("\n").map((line) => line.trim()).filter(Boolean);
const items = []; const items = [];
const seen = new Set();
for (const line of lines) { for (const line of lines) {
try { try {
const parsed = JSON.parse(line); const parsed = JSON.parse(line);
const fallbackTitle = stripTags(parsed.title);
const id = parsed.id || parsed.link || fallbackTitle || `idx:${items.length}`;
if (seen.has(id)) continue;
seen.add(id);
items.push({ items.push({
id,
title: stripTags(parsed.title), title: stripTags(parsed.title),
text: stripTags(parsed.text), text: stripTags(parsed.text),
link: parsed.link, link: parsed.link,
@@ -49,7 +55,7 @@ async function loadNews() {
} catch (err) { } catch (err) {
console.warn("NDJSON parse error", err); console.warn("NDJSON parse error", err);
} }
if (items.length >= 60) break; if (items.length >= 20) break;
} }
newsContainer.innerHTML = ""; newsContainer.innerHTML = "";

View File

@@ -32,9 +32,14 @@ async function ladeNews() {
const loadingEl = null; const loadingEl = null;
container.innerHTML = "";
const newsIndex = new Map(); const newsIndex = new Map();
const maxItems = 20;
let renderedCount = 0;
const getKey = (nachricht) => { const getKey = (nachricht) => {
if (nachricht?.id) return `id:${nachricht.id}`;
if (nachricht?.link) return `link:${nachricht.link}`; if (nachricht?.link) return `link:${nachricht.link}`;
if (nachricht?.title) return `title:${stripTags(nachricht.title)}`; if (nachricht?.title) return `title:${stripTags(nachricht.title)}`;
if (nachricht?.publishedAt) return `ts:${nachricht.publishedAt}`; if (nachricht?.publishedAt) return `ts:${nachricht.publishedAt}`;
@@ -109,13 +114,15 @@ async function ladeNews() {
const existing = newsIndex.get(key); const existing = newsIndex.get(key);
if (existing) { if (existing) {
updateFields(existing, nachricht); updateFields(existing, nachricht);
return; return false;
} }
const state = buildItem(nachricht); const state = buildItem(nachricht);
updateFields(state, nachricht); updateFields(state, nachricht);
newsIndex.set(key, state); newsIndex.set(key, state);
container.appendChild(state.item); container.appendChild(state.item);
renderedCount += 1;
return true;
}; };
try { try {
@@ -136,6 +143,7 @@ async function ladeNews() {
const decoder = new TextDecoder("utf-8"); const decoder = new TextDecoder("utf-8");
let buffer = ""; let buffer = "";
let stop = false;
while (true) { while (true) {
const { value, done } = await reader.read(); const { value, done } = await reader.read();
if (done) break; if (done) break;
@@ -148,19 +156,27 @@ async function ladeNews() {
const s = line.trim(); const s = line.trim();
if (!s) continue; if (!s) continue;
try { try {
renderItem(JSON.parse(s)); const added = renderItem(JSON.parse(s));
foundAny = true; if (added) foundAny = true;
if (renderedCount >= maxItems) {
stop = true;
break;
}
} catch (e) { } catch (e) {
console.warn("NDJSON parse error:", s.slice(0, 200)); console.warn("NDJSON parse error:", s.slice(0, 200));
} }
} }
if (stop) break;
}
if (stop) {
await reader.cancel();
} }
const last = buffer.trim(); const last = buffer.trim();
if (last) { if (last && renderedCount < maxItems) {
try { try {
renderItem(JSON.parse(last)); const added = renderItem(JSON.parse(last));
foundAny = true; if (added) foundAny = true;
} catch {} } catch {}
} }