Browse Source

bug-fixes

imwald
Silberengel 1 month ago
parent
commit
d41cb716a9
  1. 21
      src/components/RssFeedItem/index.tsx
  2. 28
      src/components/RssFeedList/index.tsx
  3. 120
      src/components/StandardRssFeedUrlRow/index.tsx
  4. 9
      src/i18n/locales/en.ts
  5. 50
      src/lib/rss-web-feed.ts
  6. 122
      src/lib/standard-rss-feed-url.ts
  7. 17
      src/lib/vite-proxy-url.ts
  8. 26
      src/pages/secondary/RssFeedSettingsPage/index.tsx
  9. 396
      src/services/rss-feed.service.ts
  10. 15
      src/services/web.service.ts

21
src/components/RssFeedItem/index.tsx

@ -16,6 +16,7 @@ import MediaPlayer from '@/components/MediaPlayer' @@ -16,6 +16,7 @@ import MediaPlayer from '@/components/MediaPlayer'
import { Drawer, DrawerContent, DrawerHeader, DrawerTitle } from '@/components/ui/drawer'
import { useScreenSize } from '@/providers/ScreenSizeProvider'
import { useSmartRssArticleNavigation } from '@/PageManager'
import { getStandardRssFeedProfile } from '@/lib/standard-rss-feed-url'
/**
* Convert HTML to plain text by extracting text content and cleaning up whitespace
@ -401,16 +402,26 @@ export default function RssFeedItem({ @@ -401,16 +402,26 @@ export default function RssFeedItem({
setSelectedText('')
}
// Format feed source name from URL
const standardFeedProfile = useMemo(
() => (isWebFaux ? null : getStandardRssFeedProfile(item.feedUrl)),
[item.feedUrl, isWebFaux]
)
// Format feed source name from URL (known shapes get a translated label)
const feedSourceName = useMemo(() => {
if (isWebFaux) return ''
if (standardFeedProfile) {
return t(standardFeedProfile.labelKey, {
defaultValue: standardFeedProfile.defaultLabel
})
}
try {
const url = new URL(item.feedUrl)
return url.hostname.replace(/^www\./, '')
} catch {
return item.feedTitle || 'RSS Feed'
}
}, [item.feedUrl, item.feedTitle, isWebFaux])
}, [item.feedUrl, item.feedTitle, isWebFaux, standardFeedProfile, t])
// Clean and parse HTML description safely
// Decode HTML entities and remove any XML artifacts that might have leaked through
@ -586,6 +597,12 @@ export default function RssFeedItem({ @@ -586,6 +597,12 @@ export default function RssFeedItem({
<h3 className="font-semibold text-sm truncate">
{isWebFaux ? t('Web page') : item.feedTitle || feedSourceName}
</h3>
{!isWebFaux && standardFeedProfile && item.feedTitle ? (
<p className="text-xs text-muted-foreground mt-0.5 truncate">
{feedSourceName}
{standardFeedProfile.detail ? ` · ${standardFeedProfile.detail}` : ''}
</p>
) : null}
{item.feedDescription && (
<p className="text-xs text-muted-foreground line-clamp-1 mt-0.5">
{item.feedDescription}

28
src/components/RssFeedList/index.tsx

@ -43,6 +43,11 @@ import { @@ -43,6 +43,11 @@ import {
import { useScreenSize } from '@/providers/ScreenSizeProvider'
import { Check, ChevronDown } from 'lucide-react'
import { normalizeHttpArticleUrl } from '@/lib/rss-article'
import {
getRssFeedUrlHostname,
getStandardRssFeedProfile
} from '@/lib/standard-rss-feed-url'
import { StandardRssFeedUrlInline } from '@/components/StandardRssFeedUrlRow'
function ManualRssUrlAddRow({
className,
@ -436,15 +441,22 @@ export default function RssFeedList() { @@ -436,15 +441,22 @@ export default function RssFeedList() {
// Normalize URLs to prevent duplicates (e.g., with/without trailing slash)
const availableFeeds = useMemo(() => {
const feedMap = new Map<string, { url: string; title: string }>()
items.forEach(item => {
items.forEach((item) => {
const normalizedUrl = normalizeFeedUrl(item.feedUrl)
if (!feedMap.has(normalizedUrl)) {
feedMap.set(normalizedUrl, { url: normalizedUrl, title: item.feedTitle || item.feedUrl })
const profile = getStandardRssFeedProfile(normalizedUrl)
const fallback = profile
? t(profile.labelKey, { defaultValue: profile.defaultLabel })
: getRssFeedUrlHostname(normalizedUrl)
feedMap.set(normalizedUrl, {
url: normalizedUrl,
title: item.feedTitle?.trim() || fallback
})
}
})
return Array.from(feedMap.values())
}, [items])
}, [items, t])
// Helper function to truncate text
const truncateText = (text: string, maxLength: number): string => {
@ -882,8 +894,12 @@ export default function RssFeedList() { @@ -882,8 +894,12 @@ export default function RssFeedList() {
<div className="flex items-center justify-center w-4 h-4 border border-border rounded">
{isChecked && <Check className="w-3 h-3" />}
</div>
<label className="text-sm cursor-pointer flex-1 truncate" title={feed.title}>
{truncateText(feed.title, 50)}
<label className="text-sm cursor-pointer flex-1 min-w-0" title={feed.title}>
<StandardRssFeedUrlInline
feedUrl={feed.url}
title={feed.title}
maxLength={50}
/>
</label>
</div>
)

120
src/components/StandardRssFeedUrlRow/index.tsx

@ -0,0 +1,120 @@ @@ -0,0 +1,120 @@
import {
getRssFeedUrlHostname,
getStandardRssFeedProfile,
type StandardRssFeedIcon,
type StandardRssFeedProfile
} from '@/lib/standard-rss-feed-url'
import { cn } from '@/lib/utils'
import { BookOpen, Flame, Mail, Music2, Newspaper, Rss, Youtube } from 'lucide-react'
import type { LucideIcon } from 'lucide-react'
import type { ReactNode } from 'react'
import { useTranslation } from 'react-i18next'
const ICONS: Record<StandardRssFeedIcon, LucideIcon> = {
music: Music2,
youtube: Youtube,
feedburner: Flame,
reddit: Newspaper,
substack: Mail,
medium: BookOpen,
rss: Rss
}
function ProfileIcon({ profile }: { profile: StandardRssFeedProfile | null }) {
const Icon = profile ? ICONS[profile.icon] : Rss
return (
<div
className="flex size-9 shrink-0 items-center justify-center rounded-md border border-border/80 bg-muted/40 text-muted-foreground"
aria-hidden
>
<Icon className="size-4" strokeWidth={2} />
</div>
)
}
type Props = {
feedUrl: string
className?: string
/** Trailing actions (e.g. remove button) */
actions?: ReactNode
}
/**
* Settings-style row: icon, friendly name for known feed URLs, optional id line, full URL link.
*/
export default function StandardRssFeedUrlRow({ feedUrl, className, actions }: Props) {
const { t } = useTranslation()
const profile = getStandardRssFeedProfile(feedUrl)
const host = getRssFeedUrlHostname(feedUrl)
const title = profile
? t(profile.labelKey, { defaultValue: profile.defaultLabel })
: host
return (
<div
className={cn(
'flex items-start justify-between gap-3 rounded-lg border p-3',
className
)}
>
<div className="flex min-w-0 flex-1 gap-3">
<ProfileIcon profile={profile} />
<div className="min-w-0 flex-1 space-y-1">
<p className="text-sm font-medium leading-tight">{title}</p>
{profile?.detail ? (
<p className="text-xs text-muted-foreground font-mono">{profile.detail}</p>
) : null}
<a
href={feedUrl}
target="_blank"
rel="noopener noreferrer"
className="block break-all text-xs text-primary hover:underline"
onClick={(e) => e.stopPropagation()}
>
{feedUrl}
</a>
</div>
</div>
{actions ? <div className="flex shrink-0 flex-col items-end gap-1">{actions}</div> : null}
</div>
)
}
type InlineProps = {
feedUrl: string
/** Prefer RSS channel title when we have it */
title?: string
/** Truncate display label (full string still in `title` tooltip) */
maxLength?: number
className?: string
}
const INLINE_ICONS: Record<StandardRssFeedIcon, LucideIcon> = ICONS
/**
* Compact icon + label for filter menus (single line, truncated).
*/
export function StandardRssFeedUrlInline({ feedUrl, title, maxLength, className }: InlineProps) {
const { t } = useTranslation()
const profile = getStandardRssFeedProfile(feedUrl)
const host = getRssFeedUrlHostname(feedUrl)
const label =
title?.trim() ||
(profile
? t(profile.labelKey, { defaultValue: profile.defaultLabel })
: host)
const display =
maxLength !== undefined && label.length > maxLength
? `${label.slice(0, maxLength)}`
: label
const Icon = profile ? INLINE_ICONS[profile.icon] : Rss
return (
<span className={cn('inline-flex min-w-0 max-w-full items-center gap-2', className)}>
<Icon className="size-3.5 shrink-0 text-muted-foreground" strokeWidth={2} aria-hidden />
<span className="truncate" title={label}>
{display}
</span>
</span>
)
}

9
src/i18n/locales/en.ts

@ -1273,7 +1273,16 @@ export default { @@ -1273,7 +1273,16 @@ export default {
'No highlights yet': 'No highlights yet',
'Showing {{filtered}} of {{total}} entries':
'Showing {{filtered}} of {{total}} entries',
standardRssFeed_spotifeed: 'Spotify playlist (Spotifeed)',
standardRssFeed_youtube: 'YouTube feed',
standardRssFeed_youtubeChannel: 'YouTube channel feed',
standardRssFeed_youtubePlaylist: 'YouTube playlist feed',
standardRssFeed_feedburner: 'FeedBurner',
standardRssFeed_reddit: 'Reddit RSS',
standardRssFeed_substack: 'Substack',
standardRssFeed_medium: 'Medium',
'RSS Feed Settings': 'RSS Feed Settings',
'Remove feed': 'Remove feed',
'RSS Feeds': 'RSS Feeds',
'RSS feeds exported to OPML file': 'RSS feeds exported to OPML file',
'RSS feeds saved': 'RSS feeds saved',

50
src/lib/rss-web-feed.ts

@ -37,7 +37,7 @@ function trimManualRssWebUrlsToLimit(entries: ManualRssWebUrlEntry[]): ManualRss @@ -37,7 +37,7 @@ function trimManualRssWebUrlsToLimit(entries: ManualRssWebUrlEntry[]): ManualRss
/** Cap how many pubkeys we scan (self + follows) per discovery pass. */
const MAX_WEB_DISCOVERY_AUTHORS = 400
const WEB_DISCOVERY_AUTHORS_CHUNK = 20
const WEB_DISCOVERY_AUTHORS_CHUNK = 10
const WEB_DISCOVERY_EVENTS_LIMIT = 400
export async function loadManualRssWebUrls(): Promise<ManualRssWebUrlEntry[]> {
@ -102,7 +102,8 @@ export async function mergeDiscoveredRssWebUrls(discovered: ManualRssWebUrlEntry @@ -102,7 +102,8 @@ export async function mergeDiscoveredRssWebUrls(discovered: ManualRssWebUrlEntry
return true
}
const URL_CHUNK = 14
/** Small chunks keep each Nostr filter JSON under relay limits ("filter item too large"). */
const URL_CHUNK = 5
/** Dispatched after publishing a kind 17 web URL reaction so RSS+Web can refetch. */
export const WEB_EXTERNAL_REACTION_PUBLISHED_EVENT = 'jumble:webExternalReactionPublished'
@ -262,29 +263,38 @@ export async function fetchNostrWebActivityForUrls(urls: string[]): Promise<Nost @@ -262,29 +263,38 @@ export async function fetchNostrWebActivityForUrls(urls: string[]): Promise<Nost
const highlightById = new Map<string, Event>()
const externalReactionById = new Map<string, Event>()
const webActivityOpts = {
onevent: (evt: Event) => {
if (evt.kind === ExtendedKind.COMMENT) {
commentById.set(evt.id, evt)
} else if (evt.kind === ExtendedKind.EXTERNAL_REACTION) {
externalReactionById.set(evt.id, evt)
} else if (evt.kind === kinds.Highlights) {
highlightById.set(evt.id, evt)
}
},
eoseTimeout: 4000,
globalTimeout: 12000
}
for (let i = 0; i < httpUrls.length; i += URL_CHUNK) {
const chunk = httpUrls.slice(i, i + URL_CHUNK)
try {
// One filter per REQ — multiple large #i/#r arrays in one subscription hit relay size limits.
await queryService.fetchEvents(
relayUrls,
[
{ kinds: [ExtendedKind.COMMENT], '#i': chunk, limit: 120 },
{ kinds: [ExtendedKind.EXTERNAL_REACTION], '#i': chunk, limit: 120 },
{ kinds: [kinds.Highlights], '#r': chunk, limit: 120 }
],
{
onevent: (evt: Event) => {
if (evt.kind === ExtendedKind.COMMENT) {
commentById.set(evt.id, evt)
} else if (evt.kind === ExtendedKind.EXTERNAL_REACTION) {
externalReactionById.set(evt.id, evt)
} else if (evt.kind === kinds.Highlights) {
highlightById.set(evt.id, evt)
}
},
eoseTimeout: 4000,
globalTimeout: 12000
}
[{ kinds: [ExtendedKind.COMMENT], '#i': chunk, limit: 120 }],
webActivityOpts
)
await queryService.fetchEvents(
relayUrls,
[{ kinds: [ExtendedKind.EXTERNAL_REACTION], '#i': chunk, limit: 120 }],
webActivityOpts
)
await queryService.fetchEvents(
relayUrls,
[{ kinds: [kinds.Highlights], '#r': chunk, limit: 120 }],
webActivityOpts
)
} catch {
/* ignore chunk */

122
src/lib/standard-rss-feed-url.ts

@ -0,0 +1,122 @@ @@ -0,0 +1,122 @@
/**
* Recognize well-known RSS / Atom feed URL shapes (Spotifeed, YouTube, FeedBurner, etc.)
* for friendlier labels in the UI. Not exhaustive unknown URLs return null.
*/
export type StandardRssFeedIcon = 'music' | 'youtube' | 'feedburner' | 'reddit' | 'substack' | 'medium' | 'rss'
export type StandardRssFeedProfile = {
icon: StandardRssFeedIcon
/** i18n key under translation */
labelKey: string
/** English fallback if catalog missing */
defaultLabel: string
/** Short secondary line (e.g. truncated id) */
detail?: string
}
function truncateId(id: string, max = 14): string {
const t = id.trim()
if (t.length <= max) return t
return `${t.slice(0, max)}`
}
/**
* Returns a profile when `feedUrl` matches a known pattern; otherwise null.
*/
export function getStandardRssFeedProfile(feedUrl: string): StandardRssFeedProfile | null {
let u: URL
try {
u = new URL(feedUrl.trim())
} catch {
return null
}
const host = u.hostname.replace(/^www\./, '').toLowerCase()
if (host === 'spotifeed.timdorr.com') {
const id = u.pathname.replace(/^\//, '').split('/').filter(Boolean)[0] ?? ''
return {
icon: 'music',
labelKey: 'standardRssFeed_spotifeed',
defaultLabel: 'Spotify playlist (Spotifeed)',
detail: id ? truncateId(id, 22) : undefined
}
}
if (host === 'youtube.com' || host === 'm.youtube.com' || host === 'music.youtube.com') {
if (u.pathname.includes('/feeds/videos.xml')) {
const ch = u.searchParams.get('channel_id')
const pl = u.searchParams.get('playlist_id')
if (ch) {
return {
icon: 'youtube',
labelKey: 'standardRssFeed_youtubeChannel',
defaultLabel: 'YouTube channel feed',
detail: truncateId(ch, 18)
}
}
if (pl) {
return {
icon: 'youtube',
labelKey: 'standardRssFeed_youtubePlaylist',
defaultLabel: 'YouTube playlist feed',
detail: truncateId(pl, 18)
}
}
return {
icon: 'youtube',
labelKey: 'standardRssFeed_youtube',
defaultLabel: 'YouTube feed'
}
}
}
if (host.endsWith('feedburner.com') || host.endsWith('feedburner.google.com')) {
return {
icon: 'feedburner',
labelKey: 'standardRssFeed_feedburner',
defaultLabel: 'FeedBurner'
}
}
if (host === 'reddit.com' || host.endsWith('.reddit.com')) {
const p = u.pathname
if (p.endsWith('.rss') || p.includes('/.rss') || p.endsWith('/rss') || p.includes('/rss/')) {
return {
icon: 'reddit',
labelKey: 'standardRssFeed_reddit',
defaultLabel: 'Reddit RSS'
}
}
}
if (host.endsWith('.substack.com') && u.pathname.startsWith('/feed')) {
return {
icon: 'substack',
labelKey: 'standardRssFeed_substack',
defaultLabel: 'Substack'
}
}
if (host === 'medium.com' || host.endsWith('.medium.com')) {
if (u.pathname.startsWith('/feed') || u.pathname.startsWith('/@')) {
return {
icon: 'medium',
labelKey: 'standardRssFeed_medium',
defaultLabel: 'Medium'
}
}
}
return null
}
/** Hostname for display when there is no known profile. */
export function getRssFeedUrlHostname(feedUrl: string): string {
try {
return new URL(feedUrl.trim()).hostname.replace(/^www\./, '')
} catch {
return feedUrl.trim().slice(0, 80)
}
}

17
src/lib/vite-proxy-url.ts

@ -0,0 +1,17 @@ @@ -0,0 +1,17 @@
/**
* Builds the browser fetch URL for Jumble's server-side fetch proxy (`VITE_PROXY_SERVER`).
* Shared by OG/HTML fetches and RSS so both hit the same proxy contract.
*/
export function buildViteProxySitesFetchUrl(originalUrl: string, proxyServer: string): string {
const base = proxyServer.trim()
if (base.startsWith('http://') || base.startsWith('https://')) {
const withSlash = base.endsWith('/') ? base : `${base}/`
return `${withSlash}sites/?url=${encodeURIComponent(originalUrl)}`
}
const basePath = base.endsWith('/') ? base : `${base}/`
return `${basePath}?url=${encodeURIComponent(originalUrl)}`
}
export function urlLooksLikeViteProxyRequest(url: string): boolean {
return url.includes('/sites/') || url.includes('/sites/?url=')
}

26
src/pages/secondary/RssFeedSettingsPage/index.tsx

@ -22,6 +22,7 @@ import rssFeedService from '@/services/rss-feed.service' @@ -22,6 +22,7 @@ import rssFeedService from '@/services/rss-feed.service'
import { parseOpml, generateOpml, downloadFile } from '@/lib/opml'
import { toast } from 'sonner'
import { normalizeHttpUrl } from '@/lib/url'
import StandardRssFeedUrlRow from '@/components/StandardRssFeedUrlRow'
// Helper function to normalize and deduplicate feed URLs
const normalizeAndDeduplicateUrls = (urls: string[]): string[] => {
@ -617,17 +618,20 @@ const RssFeedSettingsPage = forwardRef(({ index, hideTitlebar = false }: { index @@ -617,17 +618,20 @@ const RssFeedSettingsPage = forwardRef(({ index, hideTitlebar = false }: { index
</div>
) : (
feedUrls.map((url) => (
<div key={url} className="flex items-center justify-between p-3 border rounded-lg">
<span className="text-sm break-all flex-1 mr-2">{url}</span>
<Button
onClick={() => handleRemoveFeed(url)}
size="icon"
variant="ghost"
className="flex-shrink-0"
>
<Trash2 className="h-4 w-4" />
</Button>
</div>
<StandardRssFeedUrlRow
key={url}
feedUrl={url}
actions={
<Button
onClick={() => handleRemoveFeed(url)}
size="icon"
variant="ghost"
aria-label={t('Remove feed')}
>
<Trash2 className="h-4 w-4" />
</Button>
}
/>
))
)}
</div>

396
src/services/rss-feed.service.ts

@ -2,6 +2,7 @@ import { DEFAULT_RSS_FEEDS } from '@/constants' @@ -2,6 +2,7 @@ import { DEFAULT_RSS_FEEDS } from '@/constants'
import { canonicalizeRssArticleUrl } from '@/lib/rss-article'
import { cleanUrl } from '@/lib/url'
import logger from '@/lib/logger'
import { buildViteProxySitesFetchUrl, urlLooksLikeViteProxyRequest } from '@/lib/vite-proxy-url'
import indexedDb from '@/services/indexed-db.service'
export interface RssFeedItemMedia {
@ -74,6 +75,8 @@ export function createWebOnlyRssFeedItem(articleUrl: string): RssFeedItem { @@ -74,6 +75,8 @@ export function createWebOnlyRssFeedItem(articleUrl: string): RssFeedItem {
}
}
const RSS_FEED_FETCH_ATTEMPTED_KEYS_SETTING = 'rssFeedFetchAttemptedKeys'
class RssFeedService {
static instance: RssFeedService
private feedCache: Map<string, { feed: RssFeed; timestamp: number }> = new Map()
@ -83,6 +86,19 @@ class RssFeedService { @@ -83,6 +86,19 @@ class RssFeedService {
private activeFetchPromises: Map<string, Promise<RssFeed>> = new Map() // Track active fetches by URL
/** Global RSS item cap in IndexedDB; oldest by pubDate are removed when exceeded. */
private readonly MAX_CACHED_RSS_ITEMS = 5000
/**
* Feed URLs we already tried to hydrate (success or hard failure). Without this, a feed that never
* yields items (CORS, dead host) stays "missing" forever and blocks every load / retriggers refresh.
* Persisted so a full reload does not repeat a 30s wait for the same dead URL.
*/
private rssFeedAttemptedKeys = new Set<string>()
private rssFeedAttemptedKeysLoaded = false
/** Same feed list + overlapping time: one network refresh (Strict Mode / remount / HMR). */
private rssMultiFeedRefreshInFlight = new Map<string, Promise<void>>()
private rssMultiFeedRefreshKey(feedUrls: string[]): string {
return [...feedUrls].map((u) => this.normalizeRssFeedKeyUrl(u)).sort().join('\u0001')
}
constructor() {
if (!RssFeedService.instance) {
@ -95,6 +111,41 @@ class RssFeedService { @@ -95,6 +111,41 @@ class RssFeedService {
return url.trim().replace(/\/$/, '')
}
private async ensureRssFeedAttemptedKeysLoaded(): Promise<void> {
if (this.rssFeedAttemptedKeysLoaded) return
this.rssFeedAttemptedKeysLoaded = true
try {
const raw = await indexedDb.getSetting(RSS_FEED_FETCH_ATTEMPTED_KEYS_SETTING)
if (!raw) return
const parsed = JSON.parse(raw) as unknown
if (!Array.isArray(parsed)) return
for (const x of parsed) {
if (typeof x === 'string' && x.trim()) {
this.rssFeedAttemptedKeys.add(this.normalizeRssFeedKeyUrl(x))
}
}
} catch (e) {
logger.warn('[RssFeedService] Failed to load attempted feed URL keys', { error: e })
}
}
private async persistRssFeedAttemptedKeys(): Promise<void> {
try {
await indexedDb.setSetting(
RSS_FEED_FETCH_ATTEMPTED_KEYS_SETTING,
JSON.stringify([...this.rssFeedAttemptedKeys])
)
} catch (e) {
logger.warn('[RssFeedService] Failed to persist attempted feed URL keys', { error: e })
}
}
private markFeedKeysAttempted(urls: string[]): void {
for (const u of urls) {
this.rssFeedAttemptedKeys.add(this.normalizeRssFeedKeyUrl(u))
}
}
private parseItemPubDate(item: RssFeedItem): Date | null {
if (!item.pubDate) return null
if (item.pubDate instanceof Date) return item.pubDate
@ -231,12 +282,12 @@ class RssFeedService { @@ -231,12 +282,12 @@ class RssFeedService {
private getFetchStrategies(url: string): Array<{ name: string; getUrl: (url: string) => string }> {
const strategies: Array<{ name: string; getUrl: (url: string) => string }> = []
// Strategy 1: Use configured proxy server (if available)
const proxyServer = import.meta.env.VITE_PROXY_SERVER
if (proxyServer && !url.includes('/sites/')) {
// Strategy 1: Same `VITE_PROXY_SERVER` contract as OG/link preview (`sites/?url=…`), not path-encoded `/sites/{url}`.
const proxyServer = import.meta.env.VITE_PROXY_SERVER?.trim()
if (proxyServer && !urlLooksLikeViteProxyRequest(url)) {
strategies.push({
name: 'configured-proxy',
getUrl: (url) => `${proxyServer}/sites/${encodeURIComponent(url)}`
getUrl: (u) => buildViteProxySitesFetchUrl(u, proxyServer)
})
}
@ -1074,13 +1125,20 @@ class RssFeedService { @@ -1074,13 +1125,20 @@ class RssFeedService {
// Add both the full foreign month name and its lowercase version
if (foreignMonth && englishMonth) {
const trimmed = foreignMonth.trim()
// Locales may emit numeric or odd tokens; never map those (would match "12:01:00" as \b12\b).
if (/^\d+$/.test(trimmed)) {
continue
}
monthMap[foreignMonth] = englishMonth
monthMap[foreignMonth.toLowerCase()] = englishMonth
// Also handle common variations (first 3 letters)
if (foreignMonth.length >= 3) {
const abbrev = foreignMonth.substring(0, 3)
monthMap[abbrev] = englishMonth
monthMap[abbrev.toLowerCase()] = englishMonth
const abbrev = foreignMonth.substring(0, 3).trim()
if (abbrev.length >= 2 && !/^\d/.test(abbrev)) {
monthMap[abbrev] = englishMonth
monthMap[abbrev.toLowerCase()] = englishMonth
}
}
}
}
@ -1128,10 +1186,14 @@ class RssFeedService { @@ -1128,10 +1186,14 @@ class RssFeedService {
})
}
// Replace foreign month names with English equivalents
// Replace foreign month names with English equivalents (longest key first so "September" beats "Sep";
// skip pure-numeric keys so "12:01:00" is never touched by a spurious "12" → "Dec" map entry).
let monthReplaced = false
for (const [foreign, english] of Object.entries(this.monthMapCache)) {
// Match month name (case-insensitive, word boundary)
const monthEntries = Object.entries(this.monthMapCache)
.filter(([foreign]) => !/^\d+$/.test(foreign.trim()))
.sort((a, b) => b[0].length - a[0].length)
for (const [foreign, english] of monthEntries) {
const regex = new RegExp(`\\b${this.escapeRegex(foreign)}\\b`, 'i')
if (regex.test(dateToParse)) {
dateToParse = dateToParse.replace(regex, english)
@ -1212,6 +1274,8 @@ class RssFeedService { @@ -1212,6 +1274,8 @@ class RssFeedService {
throw new DOMException('The operation was aborted.', 'AbortError')
}
await this.ensureRssFeedAttemptedKeysLoaded()
// Step 1: Read from IndexedDB cache first (cache-first strategy)
let cachedItems: RssFeedItem[] = []
try {
@ -1221,12 +1285,10 @@ class RssFeedService { @@ -1221,12 +1285,10 @@ class RssFeedService {
})
// Filter to only items from the requested feeds
// Normalize URLs for comparison (remove trailing slashes, ensure consistent format)
const normalizeUrl = (url: string) => url.trim().replace(/\/$/, '')
const normalizedRequestedUrls = new Set(feedUrls.map(normalizeUrl))
const normalizedRequestedUrls = new Set(feedUrls.map((u) => this.normalizeRssFeedKeyUrl(u)))
cachedItems = allCachedItems.filter(item => {
const normalizedItemUrl = normalizeUrl(item.feedUrl)
const normalizedItemUrl = this.normalizeRssFeedKeyUrl(item.feedUrl)
const matches = normalizedRequestedUrls.has(normalizedItemUrl)
if (!matches && allCachedItems.length > 0 && allCachedItems.length < 10) {
// Only log for small sets to avoid spam
@ -1279,10 +1341,13 @@ class RssFeedService { @@ -1279,10 +1341,13 @@ class RssFeedService {
const cacheWasEmpty = cachedItems.length === 0
// Check which feeds are missing from cache
const normalizeUrl = (url: string) => url.trim().replace(/\/$/, '')
const cachedFeedUrls = new Set(cachedItems.map(item => normalizeUrl(item.feedUrl)))
const missingFeeds = feedUrls.filter(url => !cachedFeedUrls.has(normalizeUrl(url)))
// Missing = no cached rows for this feed URL and we have not yet completed a fetch pass for it
const cachedFeedUrls = new Set(cachedItems.map((item) => this.normalizeRssFeedKeyUrl(item.feedUrl)))
const missingFeeds = feedUrls.filter(
(url) =>
!cachedFeedUrls.has(this.normalizeRssFeedKeyUrl(url)) &&
!this.rssFeedAttemptedKeys.has(this.normalizeRssFeedKeyUrl(url))
)
if (missingFeeds.length > 0) {
logger.info('[RssFeedService] Some feeds are missing from cache, will fetch them', {
@ -1292,148 +1357,165 @@ class RssFeedService { @@ -1292,148 +1357,165 @@ class RssFeedService {
})
}
// Step 2: Background refresh to merge new items
// If cache is empty, we'll wait a bit for the refresh to complete
const backgroundRefresh = async (refreshSignal?: AbortSignal) => {
// Use the provided signal, or fall back to the original signal
const activeSignal = refreshSignal || signal
if (activeSignal?.aborted) {
return
}
logger.info('[RssFeedService] Starting background refresh', {
feedCount: feedUrls.length,
feedUrls,
cacheWasEmpty,
cachedItemCount: cachedItems.length
})
// Step 2: Background refresh — never tied to React's AbortSignal (Strict Mode / HMR / remount would cancel network).
const refreshAc = new AbortController()
const refreshSignal = refreshAc.signal
// Check if already aborted before starting
if (activeSignal?.aborted) {
logger.warn('[RssFeedService] Background refresh aborted before starting', {
feedCount: feedUrls.length
})
const backgroundRefresh = async (): Promise<void> => {
const dedupeKey = this.rssMultiFeedRefreshKey(feedUrls)
const inflight = this.rssMultiFeedRefreshInFlight.get(dedupeKey)
if (inflight) {
await inflight
return
}
try {
logger.info('[RssFeedService] Starting to fetch feeds', {
const run = async (): Promise<void> => {
if (refreshSignal.aborted) {
return
}
logger.info('[RssFeedService] Starting background refresh', {
feedCount: feedUrls.length,
feedUrls,
signalAborted: activeSignal?.aborted
cacheWasEmpty,
cachedItemCount: cachedItems.length
})
const results = await Promise.allSettled(
feedUrls.map(url => {
if (activeSignal?.aborted) {
logger.warn('[RssFeedService] Signal aborted before fetching feed', { url })
return Promise.reject(new DOMException('The operation was aborted.', 'AbortError'))
}
logger.debug('[RssFeedService] Fetching feed', { url, signalAborted: activeSignal?.aborted })
return this.fetchFeed(url, activeSignal)
})
)
if (activeSignal?.aborted) {
logger.warn('[RssFeedService] Signal aborted after fetching feeds', {
feedCount: feedUrls.length
if (refreshSignal.aborted) {
logger.warn('[RssFeedService] Background refresh aborted before starting', {
feedCount: feedUrls.length
})
return
}
const newItems: RssFeedItem[] = []
let successCount = 0
let failureCount = 0
let abortCount = 0
results.forEach((result, index) => {
if (result.status === 'fulfilled') {
newItems.push(...result.value.items)
successCount++
logger.info('[RssFeedService] Successfully fetched feed', {
url: feedUrls[index],
itemCount: result.value.items.length,
feedTitle: result.value.title
try {
logger.info('[RssFeedService] Starting to fetch feeds', {
feedCount: feedUrls.length,
feedUrls,
signalAborted: refreshSignal.aborted
})
const results = await Promise.allSettled(
feedUrls.map((url) => {
if (refreshSignal.aborted) {
logger.warn('[RssFeedService] Signal aborted before fetching feed', { url })
return Promise.reject(new DOMException('The operation was aborted.', 'AbortError'))
}
logger.debug('[RssFeedService] Fetching feed', { url, signalAborted: refreshSignal.aborted })
return this.fetchFeed(url, refreshSignal)
})
} else {
failureCount++
const error = result.reason
if (error instanceof DOMException && error.name === 'AbortError') {
abortCount++
logger.warn('[RssFeedService] Feed fetch was aborted', {
url: feedUrls[index],
reason: error.message || 'AbortError'
})
return
}
const errorMessage = error instanceof Error ? error.message : String(error)
logger.warn('[RssFeedService] Failed to fetch feed after trying all strategies', {
url: feedUrls[index],
error: errorMessage,
errorStack: error instanceof Error ? error.stack : undefined,
errorType: error?.constructor?.name
)
if (refreshSignal.aborted) {
logger.warn('[RssFeedService] Signal aborted after fetching feeds', {
feedCount: feedUrls.length
})
return
}
})
logger.info('[RssFeedService] Background refresh completed', {
successCount,
failureCount,
abortCount,
newItemCount: newItems.length,
totalFeeds: feedUrls.length
})
if (!activeSignal?.aborted && successCount > 0) {
// Merge new items with cached items (deduplicate by feedUrl:guid)
const itemMap = new Map<string, RssFeedItem>()
// Add cached items first
cachedItems.forEach(item => {
const key = `${item.feedUrl}:${item.guid}`
itemMap.set(key, item)
})
// Add/update with new items (newer items replace older ones)
newItems.forEach(item => {
const key = `${item.feedUrl}:${item.guid}`
const existing = itemMap.get(key)
// Keep the newer item, or add if it doesn't exist
if (!existing || (item.pubDate && existing.pubDate && item.pubDate > existing.pubDate)) {
itemMap.set(key, item)
const newItems: RssFeedItem[] = []
let successCount = 0
let failureCount = 0
let abortCount = 0
results.forEach((result, index) => {
if (result.status === 'fulfilled') {
newItems.push(...result.value.items)
successCount++
logger.info('[RssFeedService] Successfully fetched feed', {
url: feedUrls[index],
itemCount: result.value.items.length,
feedTitle: result.value.title
})
} else {
failureCount++
const error = result.reason
if (error instanceof DOMException && error.name === 'AbortError') {
abortCount++
logger.warn('[RssFeedService] Feed fetch was aborted', {
url: feedUrls[index],
reason: error.message || 'AbortError'
})
return
}
const errorMessage = error instanceof Error ? error.message : String(error)
logger.warn('[RssFeedService] Failed to fetch feed after trying all strategies', {
url: feedUrls[index],
error: errorMessage,
errorStack: error instanceof Error ? error.stack : undefined,
errorType: error?.constructor?.name
})
}
})
const mergedItems = Array.from(itemMap.values())
// Sort by publication date (newest first) before global merge + trim
mergedItems.sort((a, b) => {
const dateA = a.pubDate?.getTime() || 0
const dateB = b.pubDate?.getTime() || 0
return dateB - dateA
logger.info('[RssFeedService] Background refresh completed', {
successCount,
failureCount,
abortCount,
newItemCount: newItems.length,
totalFeeds: feedUrls.length
})
try {
await this.persistGlobalRssCacheAfterMerge(mergedItems, feedUrls)
logger.info('[RssFeedService] Updated IndexedDB cache with merged items', {
mergedFromThisRefresh: mergedItems.length,
newItems: newItems.length,
cachedItems: cachedItems.length
if (!refreshSignal.aborted) {
this.markFeedKeysAttempted(feedUrls)
await this.persistRssFeedAttemptedKeys()
}
if (!refreshSignal.aborted && successCount > 0) {
const itemMap = new Map<string, RssFeedItem>()
cachedItems.forEach((item) => {
const key = `${item.feedUrl}:${item.guid}`
itemMap.set(key, item)
})
} catch (error) {
logger.error('[RssFeedService] Failed to update IndexedDB cache', { error })
newItems.forEach((item) => {
const key = `${item.feedUrl}:${item.guid}`
const existing = itemMap.get(key)
if (!existing || (item.pubDate && existing.pubDate && item.pubDate > existing.pubDate)) {
itemMap.set(key, item)
}
})
const mergedItems = Array.from(itemMap.values())
mergedItems.sort((a, b) => {
const dateA = a.pubDate?.getTime() || 0
const dateB = b.pubDate?.getTime() || 0
return dateB - dateA
})
try {
await this.persistGlobalRssCacheAfterMerge(mergedItems, feedUrls)
logger.info('[RssFeedService] Updated IndexedDB cache with merged items', {
mergedFromThisRefresh: mergedItems.length,
newItems: newItems.length,
cachedItems: cachedItems.length
})
} catch (error) {
logger.error('[RssFeedService] Failed to update IndexedDB cache', { error })
}
}
} catch (error) {
if (!(error instanceof DOMException && error.name === 'AbortError')) {
logger.error('[RssFeedService] Background refresh failed', { error })
}
}
} catch (error) {
if (!(error instanceof DOMException && error.name === 'AbortError')) {
logger.error('[RssFeedService] Background refresh failed', { error })
}
const p = run()
this.rssMultiFeedRefreshInFlight.set(dedupeKey, p)
try {
await p
} finally {
if (this.rssMultiFeedRefreshInFlight.get(dedupeKey) === p) {
this.rssMultiFeedRefreshInFlight.delete(dedupeKey)
}
}
}
// If cache is empty OR some feeds are missing, wait a bit for background refresh
const shouldWaitForRefresh = cacheWasEmpty || missingFeeds.length > 0
// Wait only while some requested feeds are still unknown (no cache rows and no completed fetch pass)
const shouldWaitForRefresh = missingFeeds.length > 0
if (shouldWaitForRefresh) {
logger.info('[RssFeedService] Waiting for background refresh to complete', {
@ -1443,30 +1525,28 @@ class RssFeedService { @@ -1443,30 +1525,28 @@ class RssFeedService {
missingFeeds
})
try {
// For missing feeds, create a separate abort controller that won't be aborted by component cleanup
// This allows the fetch to complete even if the component re-renders
const backgroundAbortController = new AbortController()
const backgroundSignal = signal ? (() => {
// Combine signals: abort if either the external signal OR our background controller aborts
const combined = new AbortController()
const abort = () => combined.abort()
signal.addEventListener('abort', abort, { once: true })
backgroundAbortController.signal.addEventListener('abort', abort, { once: true })
return combined.signal
})() : backgroundAbortController.signal
// Wait up to 30 seconds for background refresh to complete (longer for missing feeds)
const callerGone = signal
? new Promise<void>((resolve) => {
if (signal.aborted) resolve()
else signal.addEventListener('abort', () => resolve(), { once: true })
})
: new Promise<void>(() => {
/* never */
})
// Caller abort ends the wait early; refresh keeps running on refreshAc.signal
await Promise.race([
backgroundRefresh(backgroundSignal),
new Promise(resolve => setTimeout(resolve, 30000))
backgroundRefresh(),
new Promise<void>((resolve) => setTimeout(() => resolve(), 30000)),
callerGone
])
// Re-read from cache after background refresh
try {
const refreshedItems = await indexedDb.getRssFeedItems()
const feedUrlSet = new Set(feedUrls.map(normalizeUrl))
const feedUrlSet = new Set(feedUrls.map((u) => this.normalizeRssFeedKeyUrl(u)))
cachedItems = refreshedItems
.filter(item => feedUrlSet.has(normalizeUrl(item.feedUrl)))
.filter((item) => feedUrlSet.has(this.normalizeRssFeedKeyUrl(item.feedUrl)))
.map(item => ({
...item,
pubDate: item.pubDate ? new Date(item.pubDate) : null
@ -1487,7 +1567,7 @@ class RssFeedService { @@ -1487,7 +1567,7 @@ class RssFeedService {
} else {
// Cache has all requested feeds, start background refresh in background (don't wait)
logger.debug('[RssFeedService] All feeds in cache, starting background refresh without waiting')
backgroundRefresh().catch(err => {
void backgroundRefresh().catch((err) => {
if (!(err instanceof DOMException && err.name === 'AbortError')) {
logger.error('[RssFeedService] Background refresh error', { error: err })
}
@ -1515,12 +1595,19 @@ class RssFeedService { @@ -1515,12 +1595,19 @@ class RssFeedService {
return
}
await this.ensureRssFeedAttemptedKeysLoaded()
for (const u of feedUrls) {
this.rssFeedAttemptedKeys.delete(this.normalizeRssFeedKeyUrl(u))
}
await this.persistRssFeedAttemptedKeys()
// Abort any existing background refresh
if (this.backgroundRefreshController) {
logger.info('[RssFeedService] Aborting existing background refresh before starting new one')
this.backgroundRefreshController.abort()
this.backgroundRefreshController = null
}
this.rssMultiFeedRefreshInFlight.clear()
// Create a new AbortController for this refresh
const controller = new AbortController()
@ -1571,6 +1658,11 @@ class RssFeedService { @@ -1571,6 +1658,11 @@ class RssFeedService {
}
})
if (!combinedSignal.aborted && !controller.signal.aborted) {
this.markFeedKeysAttempted(feedUrls)
await this.persistRssFeedAttemptedKeys()
}
if (!combinedSignal.aborted && !controller.signal.aborted && successCount > 0) {
// Get existing cached items
let cachedItems: RssFeedItem[] = []
@ -1641,6 +1733,8 @@ class RssFeedService { @@ -1641,6 +1733,8 @@ class RssFeedService {
clearCache(url?: string) {
if (url) {
this.feedCache.delete(url)
this.rssFeedAttemptedKeys.delete(this.normalizeRssFeedKeyUrl(url))
void this.persistRssFeedAttemptedKeys()
// Also clear from IndexedDB (filter by feedUrl)
indexedDb.getRssFeedItems().then(items => {
const filtered = items.filter(item => item.feedUrl !== url)
@ -1652,6 +1746,8 @@ class RssFeedService { @@ -1652,6 +1746,8 @@ class RssFeedService {
})
} else {
this.feedCache.clear()
this.rssFeedAttemptedKeys.clear()
void this.persistRssFeedAttemptedKeys()
// Clear all from IndexedDB
indexedDb.clearRssFeedItems().catch(err => {
logger.error('[RssFeedService] Failed to clear IndexedDB cache', { error: err })

15
src/services/web.service.ts

@ -1,3 +1,4 @@ @@ -1,3 +1,4 @@
import { buildViteProxySitesFetchUrl, urlLooksLikeViteProxyRequest } from '@/lib/vite-proxy-url'
import { TWebMetadata } from '@/types'
import DataLoader from 'dataloader'
import logger from '@/lib/logger'
@ -40,21 +41,11 @@ async function tryFetchHtml(fetchUrl: string, timeoutMs: number): Promise<string @@ -40,21 +41,11 @@ async function tryFetchHtml(fetchUrl: string, timeoutMs: number): Promise<string
}
}
function buildOgProxyFetchUrl(originalUrl: string, proxyServer: string): string {
if (proxyServer.startsWith('http://') || proxyServer.startsWith('https://')) {
const base = proxyServer.endsWith('/') ? proxyServer : `${proxyServer}/`
return `${base}sites/?url=${encodeURIComponent(originalUrl)}`
}
const basePath = proxyServer.endsWith('/') ? proxyServer : `${proxyServer}/`
return `${basePath}?url=${encodeURIComponent(originalUrl)}`
}
/**
* OG HTML: always use `VITE_PROXY_SERVER` first when set; if that fails or is unset, fetch the page directly.
*/
async function fetchHtmlForOpenGraph(originalUrl: string): Promise<{ html: string; via: string } | null> {
const isAlreadyProxyRequest =
originalUrl.includes('/sites/') || originalUrl.includes('/sites/?url=')
const isAlreadyProxyRequest = urlLooksLikeViteProxyRequest(originalUrl)
if (isAlreadyProxyRequest) {
const html = await tryFetchHtml(originalUrl, 35_000)
@ -64,7 +55,7 @@ async function fetchHtmlForOpenGraph(originalUrl: string): Promise<{ html: strin @@ -64,7 +55,7 @@ async function fetchHtmlForOpenGraph(originalUrl: string): Promise<{ html: strin
const proxyServer = import.meta.env.VITE_PROXY_SERVER?.trim()
if (proxyServer) {
const proxyFetchUrl = buildOgProxyFetchUrl(originalUrl, proxyServer)
const proxyFetchUrl = buildViteProxySitesFetchUrl(originalUrl, proxyServer)
logger.debug('[WebService] OG fetch via VITE_PROXY_SERVER', { originalUrl, proxyFetchUrl })
let html = await tryFetchHtml(proxyFetchUrl, 35_000)
if (html) {

Loading…
Cancel
Save