From 08c0abeaf843e57fd5242f8f6b75b24177f81cdc Mon Sep 17 00:00:00 2001 From: Ou Date: Sun, 3 Nov 2024 00:59:44 +0800 Subject: [PATCH] feat: refactor fetch and refetch logic --- server/api/proxy/index.ts | 14 ----- server/api/s/{[id].ts => [id].get.ts} | 0 server/api/s/entries.post.ts | 14 +++++ server/api/s/index.ts | 89 +++++++++++++++++++++++++++ server/database/cache.ts | 33 +++++++++- server/utils/fetch.ts | 2 +- shared/metadata.ts | 2 +- shared/types.ts | 4 +- src/atoms/index.ts | 15 ----- src/components/column/card.tsx | 49 ++++++--------- src/components/column/dnd.tsx | 22 +++++++ src/components/header/index.tsx | 12 +--- src/hooks/usePWA.ts | 6 +- src/hooks/useRefetch.ts | 34 ++++++++++ src/utils/cache.ts | 3 + src/utils/index.ts | 2 +- 16 files changed, 223 insertions(+), 78 deletions(-) delete mode 100644 server/api/proxy/index.ts rename server/api/s/{[id].ts => [id].get.ts} (100%) create mode 100644 server/api/s/entries.post.ts create mode 100644 server/api/s/index.ts create mode 100644 src/hooks/useRefetch.ts create mode 100644 src/utils/cache.ts diff --git a/server/api/proxy/index.ts b/server/api/proxy/index.ts deleted file mode 100644 index 633d81c..0000000 --- a/server/api/proxy/index.ts +++ /dev/null @@ -1,14 +0,0 @@ -export default defineEventHandler(async (event) => { - const img = getQuery(event).img - if (img) { - const url = decodeURIComponent(img as string) - return sendProxy(event, url, { - headers: { - "Access-Control-Allow-Origin": "*", - "Access-Control-Allow-Credentials": "*", - "Access-Control-Allow-Methods": "GET, HEAD, POST, PUT, OPTIONS", - "Access-Control-Allow-Headers": "*", - }, - }) - } -}) diff --git a/server/api/s/[id].ts b/server/api/s/[id].get.ts similarity index 100% rename from server/api/s/[id].ts rename to server/api/s/[id].get.ts diff --git a/server/api/s/entries.post.ts b/server/api/s/entries.post.ts new file mode 100644 index 0000000..ec2e479 --- /dev/null +++ b/server/api/s/entries.post.ts @@ -0,0 +1,14 @@ +import { getCacheTable } from "#/database/cache" + +export default defineEventHandler(async (event) => { + try { + const { sources } = await readBody(event) + const cacheTable = await getCacheTable() + if (sources && cacheTable) { + const data = await cacheTable.getEntries(sources) + return data + } + } catch { + // + } +}) diff --git a/server/api/s/index.ts b/server/api/s/index.ts new file mode 100644 index 0000000..657e924 --- /dev/null +++ b/server/api/s/index.ts @@ -0,0 +1,89 @@ +import type { SourceID, SourceResponse } from "@shared/types" +import { getters } from "#/getters" +import { getCacheTable } from "#/database/cache" +import type { CacheInfo } from "#/types" + +export default defineEventHandler(async (event): Promise => { + try { + const query = getQuery(event) + const latest = query.latest !== undefined && query.latest !== "false" + let id = query.id as SourceID + const isValid = (id: SourceID) => !id || !sources[id] || !getters[id] + + if (isValid(id)) { + const redirectID = sources?.[id]?.redirect + if (redirectID) id = redirectID + if (isValid(id)) throw new Error("Invalid source id") + } + + const cacheTable = await getCacheTable() + const now = Date.now() + let cache: CacheInfo + if (cacheTable) { + cache = await cacheTable.get(id) + if (cache) { + // interval 刷新间隔，对于缓存失效也要执行的。本质上表示本来内容更新就很慢，这个间隔内可能内容压根不会更新。 + // 默认 10 分钟，是低于 TTL 的，但部分 Source 的更新间隔会超过 TTL，甚至有的一天更新一次。 + const interval = sources[id].interval + if (now - cache.updated < interval) { + return { + status: "success", + id, + updatedTime: now, + items: cache.data, + } + } + + // 而 TTL 缓存失效时间，在时间范围内，就算内容更新了也要用这个缓存。 + // 复用缓存是不会更新时间的。 + if (now - cache.updated < TTL) { + // 有 latest + // 没有 latest，但服务器禁止登录 + + // 没有 latest + // 有 latest，服务器可以登录但没有登录 + if (!latest || (!event.context.disabledLogin && !event.context.user)) { + return { + status: "cache", + id, + updatedTime: cache.updated, + items: cache.data, + } + } + } + } + } + + try { + const newData = (await getters[id]()).slice(0, 30) + if (cacheTable && newData) { + if (event.context.waitUntil) event.context.waitUntil(cacheTable.set(id, newData)) + else await cacheTable.set(id, newData) + } + logger.success(`fetch ${id} latest`) + return { + status: "success", + id, + updatedTime: now, + items: newData, + } + } catch (e) { + if (cache!) { + return { + status: "cache", + id, + updatedTime: cache.updated, + items: cache.data, + } + } else { + throw e + } + } + } catch (e: any) { + logger.error(e) + throw createError({ + statusCode: 500, + message: e instanceof Error ? e.message : "Internal Server Error", + }) + } +}) diff --git a/server/database/cache.ts b/server/database/cache.ts index 36377e7..bae3f8f 100644 --- a/server/database/cache.ts +++ b/server/database/cache.ts @@ -40,6 +40,35 @@ export class Cache { return r } + async getEntries(keys: string[]) { + const keysStr = keys.map(k => `id = '${k}'`).join(" or ") + const res = await this.db.prepare(`SELECT id, data, updated FROM cache WHERE ${keysStr}`).all() as any + + const rows = (res.results ?? res) as { + id: SourceID + data: string + updated: number + }[] + + /** + * https://developers.cloudflare.com/d1/build-with-d1/d1-client-api/#return-object + * cloudflare d1 .all() will return + * { + * success: boolean + * meta: + * results: + * } + */ + if (rows?.length) { + logger.success(`get entries cache`) + return Object.fromEntries(rows.map(row => [row.id, { + id: row.id, + updatedTime: row.updated, + items: JSON.parse(row.data) as NewsItem[], + }])) + } + } + async delete(key: string) { return await this.db.prepare(`DELETE FROM cache WHERE id = ?`).run(key) } @@ -53,7 +82,7 @@ export async function getCacheTable() { const cacheTable = new Cache(db) if (process.env.INIT_TABLE !== "false") await cacheTable.init() return cacheTable - } catch (e) { - logger.error("failed to init database ", e) + } catch { + // logger.error("failed to init database ", e) } } diff --git a/server/utils/fetch.ts b/server/utils/fetch.ts index aa9c616..3cae957 100644 --- a/server/utils/fetch.ts +++ b/server/utils/fetch.ts @@ -4,6 +4,6 @@ export const myFetch = $fetch.create({ headers: { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36", }, - timeout: 5000, + timeout: 10000, retry: 3, }) diff --git a/shared/metadata.ts b/shared/metadata.ts index fb4dca8..346292d 100644 --- a/shared/metadata.ts +++ b/shared/metadata.ts @@ -27,7 +27,7 @@ export const columns = { } as const export const fixedColumnIds = ["focus", "hottest", "realtime"] as const satisfies Partial[] -export const hiddenColumns = Object.keys(sources).filter(id => !fixedColumnIds.includes(id as any)) as HiddenColumnID[] +export const hiddenColumns = Object.keys(columns).filter(id => !fixedColumnIds.includes(id as any)) as HiddenColumnID[] export const metadata: Metadata = typeSafeObjectFromEntries(typeSafeObjectEntries(columns).map(([k, v]) => { switch (k) { diff --git a/shared/types.ts b/shared/types.ts index 91f010a..29cf9e2 100644 --- a/shared/types.ts +++ b/shared/types.ts @@ -11,7 +11,7 @@ export type SourceID = { [Key in MainSourceID]: ConstSources[Key] extends { disable?: true } ? never : ConstSources[Key] extends { sub?: infer SubSource } ? { // @ts-expect-error >_< - [SubKey in keyof SubSource ]: SubSource[SubKey] extends { disable?: true } ? never : `${Key}-${SubKey}` + [SubKey in keyof SubSource]: SubSource[SubKey] extends { disable?: true } ? never : `${Key}-${SubKey}` }[keyof SubSource] | Key : Key; }[MainSourceID] @@ -108,3 +108,5 @@ export interface SourceResponse { updatedTime: number | string items: NewsItem[] } + +export type EntriesSourceResponse = Partial> diff --git a/src/atoms/index.ts b/src/atoms/index.ts index 594c47d..dcb4784 100644 --- a/src/atoms/index.ts +++ b/src/atoms/index.ts @@ -15,21 +15,6 @@ export const focusSourcesAtom = atom((get) => { }) }) -function initRefetchSources() { - let time = 0 - // useOnReload - // 没有放在 useOnReload 里面, 可以避免初始化后再修改 refetchSourceAtom，导致多次请求 API - const _ = localStorage.getItem("quitTime") - const now = Date.now() - const quitTime = _ ? Number(_) : 0 - if (!Number.isNaN(quitTime) && now - quitTime < 1000) { - time = now - } - return Object.fromEntries(Object.keys(sources).map(k => [k, time])) as Record -} - -export const refetchSourcesAtom = atom(initRefetchSources()) - export const currentColumnIDAtom = atom("focus") export const currentSourcesAtom = atom((get) => { diff --git a/src/components/column/card.tsx b/src/components/column/card.tsx index fde1279..8948142 100644 --- a/src/components/column/card.tsx +++ b/src/components/column/card.tsx @@ -1,12 +1,12 @@ import type { NewsItem, SourceID, SourceResponse } from "@shared/types" import { useQuery } from "@tanstack/react-query" -import { AnimatePresence, motion, useInView } from "framer-motion" +import { AnimatePresence, motion } from "framer-motion" import type { SyntheticListenerMap } from "@dnd-kit/core/dist/hooks/utilities" import { useWindowSize } from "react-use" import { forwardRef, useImperativeHandle } from "react" import { OverlayScrollbar } from "../common/overlay-scrollbar" -import { refetchSourcesAtom } from "~/atoms" import { safeParseString } from "~/utils" +import { cache } from "~/utils/cache" export interface ItemsProps extends React.HTMLAttributes { id: SourceID @@ -20,12 +20,10 @@ export interface ItemsProps extends React.HTMLAttributes { interface NewsCardProps { id: SourceID handleListeners?: SyntheticListenerMap - inView: boolean } export const CardWrapper = forwardRef(({ id, isDragged, handleListeners, style, ...props }, dndRef) => { const ref = useRef(null) - const inView = useInView(ref) useImperativeHandle(dndRef, () => ref.current!) @@ -44,33 +42,35 @@ export const CardWrapper = forwardRef(({ id, isDragg }} {...props} > - + ) }) -const prevSourceItems: Partial> = {} -function NewsCard({ id, inView, handleListeners }: NewsCardProps) { - const [refetchSource, setRefetchSource] = useAtom(refetchSourcesAtom) +function NewsCard({ id, handleListeners }: NewsCardProps) { + const { refresh, getRefreshId } = useRefetch() const { data, isFetching, isPlaceholderData, isError } = useQuery({ - queryKey: [id, refetchSource[id]], + queryKey: [id, getRefreshId(id)], queryFn: async ({ queryKey }) => { const [_id, _refetchTime] = queryKey as [SourceID, number] - let url = `/api/s/${_id}` + let url = `/api/s?id=${_id}` const headers: Record = {} if (Date.now() - _refetchTime < 1000) { - url = `/api/s/${_id}?latest` + url = `/api/s?id=${_id}&latest` const jwt = safeParseString(localStorage.getItem("jwt")) if (jwt) headers.Authorization = `Bearer ${jwt}` + } else if (cache.has(_id)) { + return cache.get(_id) } + const response: SourceResponse = await myFetch(url, { headers, }) try { - if (response.items && sources[_id].type === "hottest" && prevSourceItems[_id]) { + if (response.items && sources[_id].type === "hottest" && cache.has(_id)) { response.items.forEach((item, i) => { - const o = prevSourceItems[_id]!.findIndex(k => k.id === item.id) + const o = cache.get(_id)!.items.findIndex(k => k.id === item.id) item.extra = { ...item?.extra, diff: o === -1 ? undefined : o - i, @@ -81,27 +81,14 @@ function NewsCard({ id, inView, handleListeners }: NewsCardProps) { console.log(e) } + cache.set(_id, response) return response }, - // refetch 时显示原有的数据 - placeholderData: (prev) => { - if (prev?.id === id) { - if (prev?.items && sources[id].type === "hottest") prevSourceItems[id] = prev.items - return prev - } - }, + placeholderData: prev => prev, + staleTime: 1000 * 60 * 1, retry: false, - staleTime: 1000 * 60 * 5, - enabled: inView, }) - const manualRefetch = useCallback(() => { - setRefetchSource(prev => ({ - ...prev, - [id]: Date.now(), - })) - }, [setRefetchSource, id]) - const isFreshFetching = useMemo(() => isFetching && !isPlaceholderData, [isFetching, isPlaceholderData]) const { isFocused, toggleFocus } = useFocusWith(id) @@ -136,7 +123,7 @@ function NewsCard({ id, inView, handleListeners }: NewsCardProps) {