Browse Source

bug-fixes

imwald
Silberengel 1 month ago
parent
commit
d41cb716a9
  1. 21
      src/components/RssFeedItem/index.tsx
  2. 26
      src/components/RssFeedList/index.tsx
  3. 120
      src/components/StandardRssFeedUrlRow/index.tsx
  4. 9
      src/i18n/locales/en.ts
  5. 50
      src/lib/rss-web-feed.ts
  6. 122
      src/lib/standard-rss-feed-url.ts
  7. 17
      src/lib/vite-proxy-url.ts
  8. 26
      src/pages/secondary/RssFeedSettingsPage/index.tsx
  9. 386
      src/services/rss-feed.service.ts
  10. 15
      src/services/web.service.ts

21
src/components/RssFeedItem/index.tsx

@ -16,6 +16,7 @@ import MediaPlayer from '@/components/MediaPlayer'
import { Drawer, DrawerContent, DrawerHeader, DrawerTitle } from '@/components/ui/drawer' import { Drawer, DrawerContent, DrawerHeader, DrawerTitle } from '@/components/ui/drawer'
import { useScreenSize } from '@/providers/ScreenSizeProvider' import { useScreenSize } from '@/providers/ScreenSizeProvider'
import { useSmartRssArticleNavigation } from '@/PageManager' import { useSmartRssArticleNavigation } from '@/PageManager'
import { getStandardRssFeedProfile } from '@/lib/standard-rss-feed-url'
/** /**
* Convert HTML to plain text by extracting text content and cleaning up whitespace * Convert HTML to plain text by extracting text content and cleaning up whitespace
@ -401,16 +402,26 @@ export default function RssFeedItem({
setSelectedText('') setSelectedText('')
} }
// Format feed source name from URL const standardFeedProfile = useMemo(
() => (isWebFaux ? null : getStandardRssFeedProfile(item.feedUrl)),
[item.feedUrl, isWebFaux]
)
// Format feed source name from URL (known shapes get a translated label)
const feedSourceName = useMemo(() => { const feedSourceName = useMemo(() => {
if (isWebFaux) return '' if (isWebFaux) return ''
if (standardFeedProfile) {
return t(standardFeedProfile.labelKey, {
defaultValue: standardFeedProfile.defaultLabel
})
}
try { try {
const url = new URL(item.feedUrl) const url = new URL(item.feedUrl)
return url.hostname.replace(/^www\./, '') return url.hostname.replace(/^www\./, '')
} catch { } catch {
return item.feedTitle || 'RSS Feed' return item.feedTitle || 'RSS Feed'
} }
}, [item.feedUrl, item.feedTitle, isWebFaux]) }, [item.feedUrl, item.feedTitle, isWebFaux, standardFeedProfile, t])
// Clean and parse HTML description safely // Clean and parse HTML description safely
// Decode HTML entities and remove any XML artifacts that might have leaked through // Decode HTML entities and remove any XML artifacts that might have leaked through
@ -586,6 +597,12 @@ export default function RssFeedItem({
<h3 className="font-semibold text-sm truncate"> <h3 className="font-semibold text-sm truncate">
{isWebFaux ? t('Web page') : item.feedTitle || feedSourceName} {isWebFaux ? t('Web page') : item.feedTitle || feedSourceName}
</h3> </h3>
{!isWebFaux && standardFeedProfile && item.feedTitle ? (
<p className="text-xs text-muted-foreground mt-0.5 truncate">
{feedSourceName}
{standardFeedProfile.detail ? ` · ${standardFeedProfile.detail}` : ''}
</p>
) : null}
{item.feedDescription && ( {item.feedDescription && (
<p className="text-xs text-muted-foreground line-clamp-1 mt-0.5"> <p className="text-xs text-muted-foreground line-clamp-1 mt-0.5">
{item.feedDescription} {item.feedDescription}

26
src/components/RssFeedList/index.tsx

@ -43,6 +43,11 @@ import {
import { useScreenSize } from '@/providers/ScreenSizeProvider' import { useScreenSize } from '@/providers/ScreenSizeProvider'
import { Check, ChevronDown } from 'lucide-react' import { Check, ChevronDown } from 'lucide-react'
import { normalizeHttpArticleUrl } from '@/lib/rss-article' import { normalizeHttpArticleUrl } from '@/lib/rss-article'
import {
getRssFeedUrlHostname,
getStandardRssFeedProfile
} from '@/lib/standard-rss-feed-url'
import { StandardRssFeedUrlInline } from '@/components/StandardRssFeedUrlRow'
function ManualRssUrlAddRow({ function ManualRssUrlAddRow({
className, className,
@ -437,14 +442,21 @@ export default function RssFeedList() {
const availableFeeds = useMemo(() => { const availableFeeds = useMemo(() => {
const feedMap = new Map<string, { url: string; title: string }>() const feedMap = new Map<string, { url: string; title: string }>()
items.forEach(item => { items.forEach((item) => {
const normalizedUrl = normalizeFeedUrl(item.feedUrl) const normalizedUrl = normalizeFeedUrl(item.feedUrl)
if (!feedMap.has(normalizedUrl)) { if (!feedMap.has(normalizedUrl)) {
feedMap.set(normalizedUrl, { url: normalizedUrl, title: item.feedTitle || item.feedUrl }) const profile = getStandardRssFeedProfile(normalizedUrl)
const fallback = profile
? t(profile.labelKey, { defaultValue: profile.defaultLabel })
: getRssFeedUrlHostname(normalizedUrl)
feedMap.set(normalizedUrl, {
url: normalizedUrl,
title: item.feedTitle?.trim() || fallback
})
} }
}) })
return Array.from(feedMap.values()) return Array.from(feedMap.values())
}, [items]) }, [items, t])
// Helper function to truncate text // Helper function to truncate text
const truncateText = (text: string, maxLength: number): string => { const truncateText = (text: string, maxLength: number): string => {
@ -882,8 +894,12 @@ export default function RssFeedList() {
<div className="flex items-center justify-center w-4 h-4 border border-border rounded"> <div className="flex items-center justify-center w-4 h-4 border border-border rounded">
{isChecked && <Check className="w-3 h-3" />} {isChecked && <Check className="w-3 h-3" />}
</div> </div>
<label className="text-sm cursor-pointer flex-1 truncate" title={feed.title}> <label className="text-sm cursor-pointer flex-1 min-w-0" title={feed.title}>
{truncateText(feed.title, 50)} <StandardRssFeedUrlInline
feedUrl={feed.url}
title={feed.title}
maxLength={50}
/>
</label> </label>
</div> </div>
) )

120
src/components/StandardRssFeedUrlRow/index.tsx

@ -0,0 +1,120 @@
import {
getRssFeedUrlHostname,
getStandardRssFeedProfile,
type StandardRssFeedIcon,
type StandardRssFeedProfile
} from '@/lib/standard-rss-feed-url'
import { cn } from '@/lib/utils'
import { BookOpen, Flame, Mail, Music2, Newspaper, Rss, Youtube } from 'lucide-react'
import type { LucideIcon } from 'lucide-react'
import type { ReactNode } from 'react'
import { useTranslation } from 'react-i18next'
const ICONS: Record<StandardRssFeedIcon, LucideIcon> = {
music: Music2,
youtube: Youtube,
feedburner: Flame,
reddit: Newspaper,
substack: Mail,
medium: BookOpen,
rss: Rss
}
function ProfileIcon({ profile }: { profile: StandardRssFeedProfile | null }) {
const Icon = profile ? ICONS[profile.icon] : Rss
return (
<div
className="flex size-9 shrink-0 items-center justify-center rounded-md border border-border/80 bg-muted/40 text-muted-foreground"
aria-hidden
>
<Icon className="size-4" strokeWidth={2} />
</div>
)
}
type Props = {
feedUrl: string
className?: string
/** Trailing actions (e.g. remove button) */
actions?: ReactNode
}
/**
* Settings-style row: icon, friendly name for known feed URLs, optional id line, full URL link.
*/
export default function StandardRssFeedUrlRow({ feedUrl, className, actions }: Props) {
const { t } = useTranslation()
const profile = getStandardRssFeedProfile(feedUrl)
const host = getRssFeedUrlHostname(feedUrl)
const title = profile
? t(profile.labelKey, { defaultValue: profile.defaultLabel })
: host
return (
<div
className={cn(
'flex items-start justify-between gap-3 rounded-lg border p-3',
className
)}
>
<div className="flex min-w-0 flex-1 gap-3">
<ProfileIcon profile={profile} />
<div className="min-w-0 flex-1 space-y-1">
<p className="text-sm font-medium leading-tight">{title}</p>
{profile?.detail ? (
<p className="text-xs text-muted-foreground font-mono">{profile.detail}</p>
) : null}
<a
href={feedUrl}
target="_blank"
rel="noopener noreferrer"
className="block break-all text-xs text-primary hover:underline"
onClick={(e) => e.stopPropagation()}
>
{feedUrl}
</a>
</div>
</div>
{actions ? <div className="flex shrink-0 flex-col items-end gap-1">{actions}</div> : null}
</div>
)
}
type InlineProps = {
feedUrl: string
/** Prefer RSS channel title when we have it */
title?: string
/** Truncate display label (full string still in `title` tooltip) */
maxLength?: number
className?: string
}
const INLINE_ICONS: Record<StandardRssFeedIcon, LucideIcon> = ICONS
/**
* Compact icon + label for filter menus (single line, truncated).
*/
export function StandardRssFeedUrlInline({ feedUrl, title, maxLength, className }: InlineProps) {
const { t } = useTranslation()
const profile = getStandardRssFeedProfile(feedUrl)
const host = getRssFeedUrlHostname(feedUrl)
const label =
title?.trim() ||
(profile
? t(profile.labelKey, { defaultValue: profile.defaultLabel })
: host)
const display =
maxLength !== undefined && label.length > maxLength
? `${label.slice(0, maxLength)}`
: label
const Icon = profile ? INLINE_ICONS[profile.icon] : Rss
return (
<span className={cn('inline-flex min-w-0 max-w-full items-center gap-2', className)}>
<Icon className="size-3.5 shrink-0 text-muted-foreground" strokeWidth={2} aria-hidden />
<span className="truncate" title={label}>
{display}
</span>
</span>
)
}

9
src/i18n/locales/en.ts

@ -1273,7 +1273,16 @@ export default {
'No highlights yet': 'No highlights yet', 'No highlights yet': 'No highlights yet',
'Showing {{filtered}} of {{total}} entries': 'Showing {{filtered}} of {{total}} entries':
'Showing {{filtered}} of {{total}} entries', 'Showing {{filtered}} of {{total}} entries',
standardRssFeed_spotifeed: 'Spotify playlist (Spotifeed)',
standardRssFeed_youtube: 'YouTube feed',
standardRssFeed_youtubeChannel: 'YouTube channel feed',
standardRssFeed_youtubePlaylist: 'YouTube playlist feed',
standardRssFeed_feedburner: 'FeedBurner',
standardRssFeed_reddit: 'Reddit RSS',
standardRssFeed_substack: 'Substack',
standardRssFeed_medium: 'Medium',
'RSS Feed Settings': 'RSS Feed Settings', 'RSS Feed Settings': 'RSS Feed Settings',
'Remove feed': 'Remove feed',
'RSS Feeds': 'RSS Feeds', 'RSS Feeds': 'RSS Feeds',
'RSS feeds exported to OPML file': 'RSS feeds exported to OPML file', 'RSS feeds exported to OPML file': 'RSS feeds exported to OPML file',
'RSS feeds saved': 'RSS feeds saved', 'RSS feeds saved': 'RSS feeds saved',

50
src/lib/rss-web-feed.ts

@ -37,7 +37,7 @@ function trimManualRssWebUrlsToLimit(entries: ManualRssWebUrlEntry[]): ManualRss
/** Cap how many pubkeys we scan (self + follows) per discovery pass. */ /** Cap how many pubkeys we scan (self + follows) per discovery pass. */
const MAX_WEB_DISCOVERY_AUTHORS = 400 const MAX_WEB_DISCOVERY_AUTHORS = 400
const WEB_DISCOVERY_AUTHORS_CHUNK = 20 const WEB_DISCOVERY_AUTHORS_CHUNK = 10
const WEB_DISCOVERY_EVENTS_LIMIT = 400 const WEB_DISCOVERY_EVENTS_LIMIT = 400
export async function loadManualRssWebUrls(): Promise<ManualRssWebUrlEntry[]> { export async function loadManualRssWebUrls(): Promise<ManualRssWebUrlEntry[]> {
@ -102,7 +102,8 @@ export async function mergeDiscoveredRssWebUrls(discovered: ManualRssWebUrlEntry
return true return true
} }
const URL_CHUNK = 14 /** Small chunks keep each Nostr filter JSON under relay limits ("filter item too large"). */
const URL_CHUNK = 5
/** Dispatched after publishing a kind 17 web URL reaction so RSS+Web can refetch. */ /** Dispatched after publishing a kind 17 web URL reaction so RSS+Web can refetch. */
export const WEB_EXTERNAL_REACTION_PUBLISHED_EVENT = 'jumble:webExternalReactionPublished' export const WEB_EXTERNAL_REACTION_PUBLISHED_EVENT = 'jumble:webExternalReactionPublished'
@ -262,29 +263,38 @@ export async function fetchNostrWebActivityForUrls(urls: string[]): Promise<Nost
const highlightById = new Map<string, Event>() const highlightById = new Map<string, Event>()
const externalReactionById = new Map<string, Event>() const externalReactionById = new Map<string, Event>()
const webActivityOpts = {
onevent: (evt: Event) => {
if (evt.kind === ExtendedKind.COMMENT) {
commentById.set(evt.id, evt)
} else if (evt.kind === ExtendedKind.EXTERNAL_REACTION) {
externalReactionById.set(evt.id, evt)
} else if (evt.kind === kinds.Highlights) {
highlightById.set(evt.id, evt)
}
},
eoseTimeout: 4000,
globalTimeout: 12000
}
for (let i = 0; i < httpUrls.length; i += URL_CHUNK) { for (let i = 0; i < httpUrls.length; i += URL_CHUNK) {
const chunk = httpUrls.slice(i, i + URL_CHUNK) const chunk = httpUrls.slice(i, i + URL_CHUNK)
try { try {
// One filter per REQ — multiple large #i/#r arrays in one subscription hit relay size limits.
await queryService.fetchEvents( await queryService.fetchEvents(
relayUrls, relayUrls,
[ [{ kinds: [ExtendedKind.COMMENT], '#i': chunk, limit: 120 }],
{ kinds: [ExtendedKind.COMMENT], '#i': chunk, limit: 120 }, webActivityOpts
{ kinds: [ExtendedKind.EXTERNAL_REACTION], '#i': chunk, limit: 120 }, )
{ kinds: [kinds.Highlights], '#r': chunk, limit: 120 } await queryService.fetchEvents(
], relayUrls,
{ [{ kinds: [ExtendedKind.EXTERNAL_REACTION], '#i': chunk, limit: 120 }],
onevent: (evt: Event) => { webActivityOpts
if (evt.kind === ExtendedKind.COMMENT) { )
commentById.set(evt.id, evt) await queryService.fetchEvents(
} else if (evt.kind === ExtendedKind.EXTERNAL_REACTION) { relayUrls,
externalReactionById.set(evt.id, evt) [{ kinds: [kinds.Highlights], '#r': chunk, limit: 120 }],
} else if (evt.kind === kinds.Highlights) { webActivityOpts
highlightById.set(evt.id, evt)
}
},
eoseTimeout: 4000,
globalTimeout: 12000
}
) )
} catch { } catch {
/* ignore chunk */ /* ignore chunk */

122
src/lib/standard-rss-feed-url.ts

@ -0,0 +1,122 @@
/**
* Recognize well-known RSS / Atom feed URL shapes (Spotifeed, YouTube, FeedBurner, etc.)
* for friendlier labels in the UI. Not exhaustive unknown URLs return null.
*/
export type StandardRssFeedIcon = 'music' | 'youtube' | 'feedburner' | 'reddit' | 'substack' | 'medium' | 'rss'
export type StandardRssFeedProfile = {
icon: StandardRssFeedIcon
/** i18n key under translation */
labelKey: string
/** English fallback if catalog missing */
defaultLabel: string
/** Short secondary line (e.g. truncated id) */
detail?: string
}
function truncateId(id: string, max = 14): string {
const t = id.trim()
if (t.length <= max) return t
return `${t.slice(0, max)}`
}
/**
* Returns a profile when `feedUrl` matches a known pattern; otherwise null.
*/
export function getStandardRssFeedProfile(feedUrl: string): StandardRssFeedProfile | null {
let u: URL
try {
u = new URL(feedUrl.trim())
} catch {
return null
}
const host = u.hostname.replace(/^www\./, '').toLowerCase()
if (host === 'spotifeed.timdorr.com') {
const id = u.pathname.replace(/^\//, '').split('/').filter(Boolean)[0] ?? ''
return {
icon: 'music',
labelKey: 'standardRssFeed_spotifeed',
defaultLabel: 'Spotify playlist (Spotifeed)',
detail: id ? truncateId(id, 22) : undefined
}
}
if (host === 'youtube.com' || host === 'm.youtube.com' || host === 'music.youtube.com') {
if (u.pathname.includes('/feeds/videos.xml')) {
const ch = u.searchParams.get('channel_id')
const pl = u.searchParams.get('playlist_id')
if (ch) {
return {
icon: 'youtube',
labelKey: 'standardRssFeed_youtubeChannel',
defaultLabel: 'YouTube channel feed',
detail: truncateId(ch, 18)
}
}
if (pl) {
return {
icon: 'youtube',
labelKey: 'standardRssFeed_youtubePlaylist',
defaultLabel: 'YouTube playlist feed',
detail: truncateId(pl, 18)
}
}
return {
icon: 'youtube',
labelKey: 'standardRssFeed_youtube',
defaultLabel: 'YouTube feed'
}
}
}
if (host.endsWith('feedburner.com') || host.endsWith('feedburner.google.com')) {
return {
icon: 'feedburner',
labelKey: 'standardRssFeed_feedburner',
defaultLabel: 'FeedBurner'
}
}
if (host === 'reddit.com' || host.endsWith('.reddit.com')) {
const p = u.pathname
if (p.endsWith('.rss') || p.includes('/.rss') || p.endsWith('/rss') || p.includes('/rss/')) {
return {
icon: 'reddit',
labelKey: 'standardRssFeed_reddit',
defaultLabel: 'Reddit RSS'
}
}
}
if (host.endsWith('.substack.com') && u.pathname.startsWith('/feed')) {
return {
icon: 'substack',
labelKey: 'standardRssFeed_substack',
defaultLabel: 'Substack'
}
}
if (host === 'medium.com' || host.endsWith('.medium.com')) {
if (u.pathname.startsWith('/feed') || u.pathname.startsWith('/@')) {
return {
icon: 'medium',
labelKey: 'standardRssFeed_medium',
defaultLabel: 'Medium'
}
}
}
return null
}
/** Hostname for display when there is no known profile. */
export function getRssFeedUrlHostname(feedUrl: string): string {
try {
return new URL(feedUrl.trim()).hostname.replace(/^www\./, '')
} catch {
return feedUrl.trim().slice(0, 80)
}
}

17
src/lib/vite-proxy-url.ts

@ -0,0 +1,17 @@
/**
* Builds the browser fetch URL for Jumble's server-side fetch proxy (`VITE_PROXY_SERVER`).
* Shared by OG/HTML fetches and RSS so both hit the same proxy contract.
*/
export function buildViteProxySitesFetchUrl(originalUrl: string, proxyServer: string): string {
const base = proxyServer.trim()
if (base.startsWith('http://') || base.startsWith('https://')) {
const withSlash = base.endsWith('/') ? base : `${base}/`
return `${withSlash}sites/?url=${encodeURIComponent(originalUrl)}`
}
const basePath = base.endsWith('/') ? base : `${base}/`
return `${basePath}?url=${encodeURIComponent(originalUrl)}`
}
export function urlLooksLikeViteProxyRequest(url: string): boolean {
return url.includes('/sites/') || url.includes('/sites/?url=')
}

26
src/pages/secondary/RssFeedSettingsPage/index.tsx

@ -22,6 +22,7 @@ import rssFeedService from '@/services/rss-feed.service'
import { parseOpml, generateOpml, downloadFile } from '@/lib/opml' import { parseOpml, generateOpml, downloadFile } from '@/lib/opml'
import { toast } from 'sonner' import { toast } from 'sonner'
import { normalizeHttpUrl } from '@/lib/url' import { normalizeHttpUrl } from '@/lib/url'
import StandardRssFeedUrlRow from '@/components/StandardRssFeedUrlRow'
// Helper function to normalize and deduplicate feed URLs // Helper function to normalize and deduplicate feed URLs
const normalizeAndDeduplicateUrls = (urls: string[]): string[] => { const normalizeAndDeduplicateUrls = (urls: string[]): string[] => {
@ -617,17 +618,20 @@ const RssFeedSettingsPage = forwardRef(({ index, hideTitlebar = false }: { index
</div> </div>
) : ( ) : (
feedUrls.map((url) => ( feedUrls.map((url) => (
<div key={url} className="flex items-center justify-between p-3 border rounded-lg"> <StandardRssFeedUrlRow
<span className="text-sm break-all flex-1 mr-2">{url}</span> key={url}
<Button feedUrl={url}
onClick={() => handleRemoveFeed(url)} actions={
size="icon" <Button
variant="ghost" onClick={() => handleRemoveFeed(url)}
className="flex-shrink-0" size="icon"
> variant="ghost"
<Trash2 className="h-4 w-4" /> aria-label={t('Remove feed')}
</Button> >
</div> <Trash2 className="h-4 w-4" />
</Button>
}
/>
)) ))
)} )}
</div> </div>

386
src/services/rss-feed.service.ts

@ -2,6 +2,7 @@ import { DEFAULT_RSS_FEEDS } from '@/constants'
import { canonicalizeRssArticleUrl } from '@/lib/rss-article' import { canonicalizeRssArticleUrl } from '@/lib/rss-article'
import { cleanUrl } from '@/lib/url' import { cleanUrl } from '@/lib/url'
import logger from '@/lib/logger' import logger from '@/lib/logger'
import { buildViteProxySitesFetchUrl, urlLooksLikeViteProxyRequest } from '@/lib/vite-proxy-url'
import indexedDb from '@/services/indexed-db.service' import indexedDb from '@/services/indexed-db.service'
export interface RssFeedItemMedia { export interface RssFeedItemMedia {
@ -74,6 +75,8 @@ export function createWebOnlyRssFeedItem(articleUrl: string): RssFeedItem {
} }
} }
const RSS_FEED_FETCH_ATTEMPTED_KEYS_SETTING = 'rssFeedFetchAttemptedKeys'
class RssFeedService { class RssFeedService {
static instance: RssFeedService static instance: RssFeedService
private feedCache: Map<string, { feed: RssFeed; timestamp: number }> = new Map() private feedCache: Map<string, { feed: RssFeed; timestamp: number }> = new Map()
@ -83,6 +86,19 @@ class RssFeedService {
private activeFetchPromises: Map<string, Promise<RssFeed>> = new Map() // Track active fetches by URL private activeFetchPromises: Map<string, Promise<RssFeed>> = new Map() // Track active fetches by URL
/** Global RSS item cap in IndexedDB; oldest by pubDate are removed when exceeded. */ /** Global RSS item cap in IndexedDB; oldest by pubDate are removed when exceeded. */
private readonly MAX_CACHED_RSS_ITEMS = 5000 private readonly MAX_CACHED_RSS_ITEMS = 5000
/**
* Feed URLs we already tried to hydrate (success or hard failure). Without this, a feed that never
* yields items (CORS, dead host) stays "missing" forever and blocks every load / retriggers refresh.
* Persisted so a full reload does not repeat a 30s wait for the same dead URL.
*/
private rssFeedAttemptedKeys = new Set<string>()
private rssFeedAttemptedKeysLoaded = false
/** Same feed list + overlapping time: one network refresh (Strict Mode / remount / HMR). */
private rssMultiFeedRefreshInFlight = new Map<string, Promise<void>>()
private rssMultiFeedRefreshKey(feedUrls: string[]): string {
return [...feedUrls].map((u) => this.normalizeRssFeedKeyUrl(u)).sort().join('\u0001')
}
constructor() { constructor() {
if (!RssFeedService.instance) { if (!RssFeedService.instance) {
@ -95,6 +111,41 @@ class RssFeedService {
return url.trim().replace(/\/$/, '') return url.trim().replace(/\/$/, '')
} }
private async ensureRssFeedAttemptedKeysLoaded(): Promise<void> {
if (this.rssFeedAttemptedKeysLoaded) return
this.rssFeedAttemptedKeysLoaded = true
try {
const raw = await indexedDb.getSetting(RSS_FEED_FETCH_ATTEMPTED_KEYS_SETTING)
if (!raw) return
const parsed = JSON.parse(raw) as unknown
if (!Array.isArray(parsed)) return
for (const x of parsed) {
if (typeof x === 'string' && x.trim()) {
this.rssFeedAttemptedKeys.add(this.normalizeRssFeedKeyUrl(x))
}
}
} catch (e) {
logger.warn('[RssFeedService] Failed to load attempted feed URL keys', { error: e })
}
}
private async persistRssFeedAttemptedKeys(): Promise<void> {
try {
await indexedDb.setSetting(
RSS_FEED_FETCH_ATTEMPTED_KEYS_SETTING,
JSON.stringify([...this.rssFeedAttemptedKeys])
)
} catch (e) {
logger.warn('[RssFeedService] Failed to persist attempted feed URL keys', { error: e })
}
}
private markFeedKeysAttempted(urls: string[]): void {
for (const u of urls) {
this.rssFeedAttemptedKeys.add(this.normalizeRssFeedKeyUrl(u))
}
}
private parseItemPubDate(item: RssFeedItem): Date | null { private parseItemPubDate(item: RssFeedItem): Date | null {
if (!item.pubDate) return null if (!item.pubDate) return null
if (item.pubDate instanceof Date) return item.pubDate if (item.pubDate instanceof Date) return item.pubDate
@ -231,12 +282,12 @@ class RssFeedService {
private getFetchStrategies(url: string): Array<{ name: string; getUrl: (url: string) => string }> { private getFetchStrategies(url: string): Array<{ name: string; getUrl: (url: string) => string }> {
const strategies: Array<{ name: string; getUrl: (url: string) => string }> = [] const strategies: Array<{ name: string; getUrl: (url: string) => string }> = []
// Strategy 1: Use configured proxy server (if available) // Strategy 1: Same `VITE_PROXY_SERVER` contract as OG/link preview (`sites/?url=…`), not path-encoded `/sites/{url}`.
const proxyServer = import.meta.env.VITE_PROXY_SERVER const proxyServer = import.meta.env.VITE_PROXY_SERVER?.trim()
if (proxyServer && !url.includes('/sites/')) { if (proxyServer && !urlLooksLikeViteProxyRequest(url)) {
strategies.push({ strategies.push({
name: 'configured-proxy', name: 'configured-proxy',
getUrl: (url) => `${proxyServer}/sites/${encodeURIComponent(url)}` getUrl: (u) => buildViteProxySitesFetchUrl(u, proxyServer)
}) })
} }
@ -1074,13 +1125,20 @@ class RssFeedService {
// Add both the full foreign month name and its lowercase version // Add both the full foreign month name and its lowercase version
if (foreignMonth && englishMonth) { if (foreignMonth && englishMonth) {
const trimmed = foreignMonth.trim()
// Locales may emit numeric or odd tokens; never map those (would match "12:01:00" as \b12\b).
if (/^\d+$/.test(trimmed)) {
continue
}
monthMap[foreignMonth] = englishMonth monthMap[foreignMonth] = englishMonth
monthMap[foreignMonth.toLowerCase()] = englishMonth monthMap[foreignMonth.toLowerCase()] = englishMonth
// Also handle common variations (first 3 letters) // Also handle common variations (first 3 letters)
if (foreignMonth.length >= 3) { if (foreignMonth.length >= 3) {
const abbrev = foreignMonth.substring(0, 3) const abbrev = foreignMonth.substring(0, 3).trim()
monthMap[abbrev] = englishMonth if (abbrev.length >= 2 && !/^\d/.test(abbrev)) {
monthMap[abbrev.toLowerCase()] = englishMonth monthMap[abbrev] = englishMonth
monthMap[abbrev.toLowerCase()] = englishMonth
}
} }
} }
} }
@ -1128,10 +1186,14 @@ class RssFeedService {
}) })
} }
// Replace foreign month names with English equivalents // Replace foreign month names with English equivalents (longest key first so "September" beats "Sep";
// skip pure-numeric keys so "12:01:00" is never touched by a spurious "12" → "Dec" map entry).
let monthReplaced = false let monthReplaced = false
for (const [foreign, english] of Object.entries(this.monthMapCache)) { const monthEntries = Object.entries(this.monthMapCache)
// Match month name (case-insensitive, word boundary) .filter(([foreign]) => !/^\d+$/.test(foreign.trim()))
.sort((a, b) => b[0].length - a[0].length)
for (const [foreign, english] of monthEntries) {
const regex = new RegExp(`\\b${this.escapeRegex(foreign)}\\b`, 'i') const regex = new RegExp(`\\b${this.escapeRegex(foreign)}\\b`, 'i')
if (regex.test(dateToParse)) { if (regex.test(dateToParse)) {
dateToParse = dateToParse.replace(regex, english) dateToParse = dateToParse.replace(regex, english)
@ -1212,6 +1274,8 @@ class RssFeedService {
throw new DOMException('The operation was aborted.', 'AbortError') throw new DOMException('The operation was aborted.', 'AbortError')
} }
await this.ensureRssFeedAttemptedKeysLoaded()
// Step 1: Read from IndexedDB cache first (cache-first strategy) // Step 1: Read from IndexedDB cache first (cache-first strategy)
let cachedItems: RssFeedItem[] = [] let cachedItems: RssFeedItem[] = []
try { try {
@ -1221,12 +1285,10 @@ class RssFeedService {
}) })
// Filter to only items from the requested feeds // Filter to only items from the requested feeds
// Normalize URLs for comparison (remove trailing slashes, ensure consistent format) const normalizedRequestedUrls = new Set(feedUrls.map((u) => this.normalizeRssFeedKeyUrl(u)))
const normalizeUrl = (url: string) => url.trim().replace(/\/$/, '')
const normalizedRequestedUrls = new Set(feedUrls.map(normalizeUrl))
cachedItems = allCachedItems.filter(item => { cachedItems = allCachedItems.filter(item => {
const normalizedItemUrl = normalizeUrl(item.feedUrl) const normalizedItemUrl = this.normalizeRssFeedKeyUrl(item.feedUrl)
const matches = normalizedRequestedUrls.has(normalizedItemUrl) const matches = normalizedRequestedUrls.has(normalizedItemUrl)
if (!matches && allCachedItems.length > 0 && allCachedItems.length < 10) { if (!matches && allCachedItems.length > 0 && allCachedItems.length < 10) {
// Only log for small sets to avoid spam // Only log for small sets to avoid spam
@ -1279,10 +1341,13 @@ class RssFeedService {
const cacheWasEmpty = cachedItems.length === 0 const cacheWasEmpty = cachedItems.length === 0
// Check which feeds are missing from cache // Missing = no cached rows for this feed URL and we have not yet completed a fetch pass for it
const normalizeUrl = (url: string) => url.trim().replace(/\/$/, '') const cachedFeedUrls = new Set(cachedItems.map((item) => this.normalizeRssFeedKeyUrl(item.feedUrl)))
const cachedFeedUrls = new Set(cachedItems.map(item => normalizeUrl(item.feedUrl))) const missingFeeds = feedUrls.filter(
const missingFeeds = feedUrls.filter(url => !cachedFeedUrls.has(normalizeUrl(url))) (url) =>
!cachedFeedUrls.has(this.normalizeRssFeedKeyUrl(url)) &&
!this.rssFeedAttemptedKeys.has(this.normalizeRssFeedKeyUrl(url))
)
if (missingFeeds.length > 0) { if (missingFeeds.length > 0) {
logger.info('[RssFeedService] Some feeds are missing from cache, will fetch them', { logger.info('[RssFeedService] Some feeds are missing from cache, will fetch them', {
@ -1292,148 +1357,165 @@ class RssFeedService {
}) })
} }
// Step 2: Background refresh to merge new items // Step 2: Background refresh — never tied to React's AbortSignal (Strict Mode / HMR / remount would cancel network).
// If cache is empty, we'll wait a bit for the refresh to complete const refreshAc = new AbortController()
const backgroundRefresh = async (refreshSignal?: AbortSignal) => { const refreshSignal = refreshAc.signal
// Use the provided signal, or fall back to the original signal
const activeSignal = refreshSignal || signal
if (activeSignal?.aborted) { const backgroundRefresh = async (): Promise<void> => {
const dedupeKey = this.rssMultiFeedRefreshKey(feedUrls)
const inflight = this.rssMultiFeedRefreshInFlight.get(dedupeKey)
if (inflight) {
await inflight
return return
} }
logger.info('[RssFeedService] Starting background refresh', { const run = async (): Promise<void> => {
feedCount: feedUrls.length, if (refreshSignal.aborted) {
feedUrls, return
cacheWasEmpty, }
cachedItemCount: cachedItems.length
})
// Check if already aborted before starting
if (activeSignal?.aborted) {
logger.warn('[RssFeedService] Background refresh aborted before starting', {
feedCount: feedUrls.length
})
return
}
try { logger.info('[RssFeedService] Starting background refresh', {
logger.info('[RssFeedService] Starting to fetch feeds', {
feedCount: feedUrls.length, feedCount: feedUrls.length,
feedUrls, feedUrls,
signalAborted: activeSignal?.aborted cacheWasEmpty,
cachedItemCount: cachedItems.length
}) })
const results = await Promise.allSettled( if (refreshSignal.aborted) {
feedUrls.map(url => { logger.warn('[RssFeedService] Background refresh aborted before starting', {
if (activeSignal?.aborted) {
logger.warn('[RssFeedService] Signal aborted before fetching feed', { url })
return Promise.reject(new DOMException('The operation was aborted.', 'AbortError'))
}
logger.debug('[RssFeedService] Fetching feed', { url, signalAborted: activeSignal?.aborted })
return this.fetchFeed(url, activeSignal)
})
)
if (activeSignal?.aborted) {
logger.warn('[RssFeedService] Signal aborted after fetching feeds', {
feedCount: feedUrls.length feedCount: feedUrls.length
}) })
return return
} }
const newItems: RssFeedItem[] = [] try {
let successCount = 0 logger.info('[RssFeedService] Starting to fetch feeds', {
let failureCount = 0 feedCount: feedUrls.length,
let abortCount = 0 feedUrls,
signalAborted: refreshSignal.aborted
results.forEach((result, index) => { })
if (result.status === 'fulfilled') {
newItems.push(...result.value.items) const results = await Promise.allSettled(
successCount++ feedUrls.map((url) => {
logger.info('[RssFeedService] Successfully fetched feed', { if (refreshSignal.aborted) {
url: feedUrls[index], logger.warn('[RssFeedService] Signal aborted before fetching feed', { url })
itemCount: result.value.items.length, return Promise.reject(new DOMException('The operation was aborted.', 'AbortError'))
feedTitle: result.value.title }
logger.debug('[RssFeedService] Fetching feed', { url, signalAborted: refreshSignal.aborted })
return this.fetchFeed(url, refreshSignal)
}) })
} else { )
failureCount++
const error = result.reason if (refreshSignal.aborted) {
if (error instanceof DOMException && error.name === 'AbortError') { logger.warn('[RssFeedService] Signal aborted after fetching feeds', {
abortCount++ feedCount: feedUrls.length
logger.warn('[RssFeedService] Feed fetch was aborted', {
url: feedUrls[index],
reason: error.message || 'AbortError'
})
return
}
const errorMessage = error instanceof Error ? error.message : String(error)
logger.warn('[RssFeedService] Failed to fetch feed after trying all strategies', {
url: feedUrls[index],
error: errorMessage,
errorStack: error instanceof Error ? error.stack : undefined,
errorType: error?.constructor?.name
}) })
return
} }
})
logger.info('[RssFeedService] Background refresh completed', { const newItems: RssFeedItem[] = []
successCount, let successCount = 0
failureCount, let failureCount = 0
abortCount, let abortCount = 0
newItemCount: newItems.length,
totalFeeds: feedUrls.length
})
if (!activeSignal?.aborted && successCount > 0) { results.forEach((result, index) => {
// Merge new items with cached items (deduplicate by feedUrl:guid) if (result.status === 'fulfilled') {
const itemMap = new Map<string, RssFeedItem>() newItems.push(...result.value.items)
successCount++
logger.info('[RssFeedService] Successfully fetched feed', {
url: feedUrls[index],
itemCount: result.value.items.length,
feedTitle: result.value.title
})
} else {
failureCount++
const error = result.reason
if (error instanceof DOMException && error.name === 'AbortError') {
abortCount++
logger.warn('[RssFeedService] Feed fetch was aborted', {
url: feedUrls[index],
reason: error.message || 'AbortError'
})
return
}
const errorMessage = error instanceof Error ? error.message : String(error)
logger.warn('[RssFeedService] Failed to fetch feed after trying all strategies', {
url: feedUrls[index],
error: errorMessage,
errorStack: error instanceof Error ? error.stack : undefined,
errorType: error?.constructor?.name
})
}
})
// Add cached items first logger.info('[RssFeedService] Background refresh completed', {
cachedItems.forEach(item => { successCount,
const key = `${item.feedUrl}:${item.guid}` failureCount,
itemMap.set(key, item) abortCount,
newItemCount: newItems.length,
totalFeeds: feedUrls.length
}) })
// Add/update with new items (newer items replace older ones) if (!refreshSignal.aborted) {
newItems.forEach(item => { this.markFeedKeysAttempted(feedUrls)
const key = `${item.feedUrl}:${item.guid}` await this.persistRssFeedAttemptedKeys()
const existing = itemMap.get(key) }
// Keep the newer item, or add if it doesn't exist
if (!existing || (item.pubDate && existing.pubDate && item.pubDate > existing.pubDate)) { if (!refreshSignal.aborted && successCount > 0) {
const itemMap = new Map<string, RssFeedItem>()
cachedItems.forEach((item) => {
const key = `${item.feedUrl}:${item.guid}`
itemMap.set(key, item) itemMap.set(key, item)
} })
})
const mergedItems = Array.from(itemMap.values()) newItems.forEach((item) => {
const key = `${item.feedUrl}:${item.guid}`
const existing = itemMap.get(key)
if (!existing || (item.pubDate && existing.pubDate && item.pubDate > existing.pubDate)) {
itemMap.set(key, item)
}
})
// Sort by publication date (newest first) before global merge + trim const mergedItems = Array.from(itemMap.values())
mergedItems.sort((a, b) => {
const dateA = a.pubDate?.getTime() || 0
const dateB = b.pubDate?.getTime() || 0
return dateB - dateA
})
try { mergedItems.sort((a, b) => {
await this.persistGlobalRssCacheAfterMerge(mergedItems, feedUrls) const dateA = a.pubDate?.getTime() || 0
logger.info('[RssFeedService] Updated IndexedDB cache with merged items', { const dateB = b.pubDate?.getTime() || 0
mergedFromThisRefresh: mergedItems.length, return dateB - dateA
newItems: newItems.length,
cachedItems: cachedItems.length
}) })
} catch (error) {
logger.error('[RssFeedService] Failed to update IndexedDB cache', { error }) try {
await this.persistGlobalRssCacheAfterMerge(mergedItems, feedUrls)
logger.info('[RssFeedService] Updated IndexedDB cache with merged items', {
mergedFromThisRefresh: mergedItems.length,
newItems: newItems.length,
cachedItems: cachedItems.length
})
} catch (error) {
logger.error('[RssFeedService] Failed to update IndexedDB cache', { error })
}
}
} catch (error) {
if (!(error instanceof DOMException && error.name === 'AbortError')) {
logger.error('[RssFeedService] Background refresh failed', { error })
} }
} }
} catch (error) { }
if (!(error instanceof DOMException && error.name === 'AbortError')) {
logger.error('[RssFeedService] Background refresh failed', { error }) const p = run()
this.rssMultiFeedRefreshInFlight.set(dedupeKey, p)
try {
await p
} finally {
if (this.rssMultiFeedRefreshInFlight.get(dedupeKey) === p) {
this.rssMultiFeedRefreshInFlight.delete(dedupeKey)
} }
} }
} }
// If cache is empty OR some feeds are missing, wait a bit for background refresh // Wait only while some requested feeds are still unknown (no cache rows and no completed fetch pass)
const shouldWaitForRefresh = cacheWasEmpty || missingFeeds.length > 0 const shouldWaitForRefresh = missingFeeds.length > 0
if (shouldWaitForRefresh) { if (shouldWaitForRefresh) {
logger.info('[RssFeedService] Waiting for background refresh to complete', { logger.info('[RssFeedService] Waiting for background refresh to complete', {
@ -1443,30 +1525,28 @@ class RssFeedService {
missingFeeds missingFeeds
}) })
try { try {
// For missing feeds, create a separate abort controller that won't be aborted by component cleanup const callerGone = signal
// This allows the fetch to complete even if the component re-renders ? new Promise<void>((resolve) => {
const backgroundAbortController = new AbortController() if (signal.aborted) resolve()
const backgroundSignal = signal ? (() => { else signal.addEventListener('abort', () => resolve(), { once: true })
// Combine signals: abort if either the external signal OR our background controller aborts })
const combined = new AbortController() : new Promise<void>(() => {
const abort = () => combined.abort() /* never */
signal.addEventListener('abort', abort, { once: true }) })
backgroundAbortController.signal.addEventListener('abort', abort, { once: true })
return combined.signal // Caller abort ends the wait early; refresh keeps running on refreshAc.signal
})() : backgroundAbortController.signal
// Wait up to 30 seconds for background refresh to complete (longer for missing feeds)
await Promise.race([ await Promise.race([
backgroundRefresh(backgroundSignal), backgroundRefresh(),
new Promise(resolve => setTimeout(resolve, 30000)) new Promise<void>((resolve) => setTimeout(() => resolve(), 30000)),
callerGone
]) ])
// Re-read from cache after background refresh // Re-read from cache after background refresh
try { try {
const refreshedItems = await indexedDb.getRssFeedItems() const refreshedItems = await indexedDb.getRssFeedItems()
const feedUrlSet = new Set(feedUrls.map(normalizeUrl)) const feedUrlSet = new Set(feedUrls.map((u) => this.normalizeRssFeedKeyUrl(u)))
cachedItems = refreshedItems cachedItems = refreshedItems
.filter(item => feedUrlSet.has(normalizeUrl(item.feedUrl))) .filter((item) => feedUrlSet.has(this.normalizeRssFeedKeyUrl(item.feedUrl)))
.map(item => ({ .map(item => ({
...item, ...item,
pubDate: item.pubDate ? new Date(item.pubDate) : null pubDate: item.pubDate ? new Date(item.pubDate) : null
@ -1487,7 +1567,7 @@ class RssFeedService {
} else { } else {
// Cache has all requested feeds, start background refresh in background (don't wait) // Cache has all requested feeds, start background refresh in background (don't wait)
logger.debug('[RssFeedService] All feeds in cache, starting background refresh without waiting') logger.debug('[RssFeedService] All feeds in cache, starting background refresh without waiting')
backgroundRefresh().catch(err => { void backgroundRefresh().catch((err) => {
if (!(err instanceof DOMException && err.name === 'AbortError')) { if (!(err instanceof DOMException && err.name === 'AbortError')) {
logger.error('[RssFeedService] Background refresh error', { error: err }) logger.error('[RssFeedService] Background refresh error', { error: err })
} }
@ -1515,12 +1595,19 @@ class RssFeedService {
return return
} }
await this.ensureRssFeedAttemptedKeysLoaded()
for (const u of feedUrls) {
this.rssFeedAttemptedKeys.delete(this.normalizeRssFeedKeyUrl(u))
}
await this.persistRssFeedAttemptedKeys()
// Abort any existing background refresh // Abort any existing background refresh
if (this.backgroundRefreshController) { if (this.backgroundRefreshController) {
logger.info('[RssFeedService] Aborting existing background refresh before starting new one') logger.info('[RssFeedService] Aborting existing background refresh before starting new one')
this.backgroundRefreshController.abort() this.backgroundRefreshController.abort()
this.backgroundRefreshController = null this.backgroundRefreshController = null
} }
this.rssMultiFeedRefreshInFlight.clear()
// Create a new AbortController for this refresh // Create a new AbortController for this refresh
const controller = new AbortController() const controller = new AbortController()
@ -1571,6 +1658,11 @@ class RssFeedService {
} }
}) })
if (!combinedSignal.aborted && !controller.signal.aborted) {
this.markFeedKeysAttempted(feedUrls)
await this.persistRssFeedAttemptedKeys()
}
if (!combinedSignal.aborted && !controller.signal.aborted && successCount > 0) { if (!combinedSignal.aborted && !controller.signal.aborted && successCount > 0) {
// Get existing cached items // Get existing cached items
let cachedItems: RssFeedItem[] = [] let cachedItems: RssFeedItem[] = []
@ -1641,6 +1733,8 @@ class RssFeedService {
clearCache(url?: string) { clearCache(url?: string) {
if (url) { if (url) {
this.feedCache.delete(url) this.feedCache.delete(url)
this.rssFeedAttemptedKeys.delete(this.normalizeRssFeedKeyUrl(url))
void this.persistRssFeedAttemptedKeys()
// Also clear from IndexedDB (filter by feedUrl) // Also clear from IndexedDB (filter by feedUrl)
indexedDb.getRssFeedItems().then(items => { indexedDb.getRssFeedItems().then(items => {
const filtered = items.filter(item => item.feedUrl !== url) const filtered = items.filter(item => item.feedUrl !== url)
@ -1652,6 +1746,8 @@ class RssFeedService {
}) })
} else { } else {
this.feedCache.clear() this.feedCache.clear()
this.rssFeedAttemptedKeys.clear()
void this.persistRssFeedAttemptedKeys()
// Clear all from IndexedDB // Clear all from IndexedDB
indexedDb.clearRssFeedItems().catch(err => { indexedDb.clearRssFeedItems().catch(err => {
logger.error('[RssFeedService] Failed to clear IndexedDB cache', { error: err }) logger.error('[RssFeedService] Failed to clear IndexedDB cache', { error: err })

15
src/services/web.service.ts

@ -1,3 +1,4 @@
import { buildViteProxySitesFetchUrl, urlLooksLikeViteProxyRequest } from '@/lib/vite-proxy-url'
import { TWebMetadata } from '@/types' import { TWebMetadata } from '@/types'
import DataLoader from 'dataloader' import DataLoader from 'dataloader'
import logger from '@/lib/logger' import logger from '@/lib/logger'
@ -40,21 +41,11 @@ async function tryFetchHtml(fetchUrl: string, timeoutMs: number): Promise<string
} }
} }
function buildOgProxyFetchUrl(originalUrl: string, proxyServer: string): string {
if (proxyServer.startsWith('http://') || proxyServer.startsWith('https://')) {
const base = proxyServer.endsWith('/') ? proxyServer : `${proxyServer}/`
return `${base}sites/?url=${encodeURIComponent(originalUrl)}`
}
const basePath = proxyServer.endsWith('/') ? proxyServer : `${proxyServer}/`
return `${basePath}?url=${encodeURIComponent(originalUrl)}`
}
/** /**
* OG HTML: always use `VITE_PROXY_SERVER` first when set; if that fails or is unset, fetch the page directly. * OG HTML: always use `VITE_PROXY_SERVER` first when set; if that fails or is unset, fetch the page directly.
*/ */
async function fetchHtmlForOpenGraph(originalUrl: string): Promise<{ html: string; via: string } | null> { async function fetchHtmlForOpenGraph(originalUrl: string): Promise<{ html: string; via: string } | null> {
const isAlreadyProxyRequest = const isAlreadyProxyRequest = urlLooksLikeViteProxyRequest(originalUrl)
originalUrl.includes('/sites/') || originalUrl.includes('/sites/?url=')
if (isAlreadyProxyRequest) { if (isAlreadyProxyRequest) {
const html = await tryFetchHtml(originalUrl, 35_000) const html = await tryFetchHtml(originalUrl, 35_000)
@ -64,7 +55,7 @@ async function fetchHtmlForOpenGraph(originalUrl: string): Promise<{ html: strin
const proxyServer = import.meta.env.VITE_PROXY_SERVER?.trim() const proxyServer = import.meta.env.VITE_PROXY_SERVER?.trim()
if (proxyServer) { if (proxyServer) {
const proxyFetchUrl = buildOgProxyFetchUrl(originalUrl, proxyServer) const proxyFetchUrl = buildViteProxySitesFetchUrl(originalUrl, proxyServer)
logger.debug('[WebService] OG fetch via VITE_PROXY_SERVER', { originalUrl, proxyFetchUrl }) logger.debug('[WebService] OG fetch via VITE_PROXY_SERVER', { originalUrl, proxyFetchUrl })
let html = await tryFetchHtml(proxyFetchUrl, 35_000) let html = await tryFetchHtml(proxyFetchUrl, 35_000)
if (html) { if (html) {

Loading…
Cancel
Save