feat: refactor fetch and refetch logic

server/api/proxy/index.ts

@@ -1,14 +0,0 @@
-export default defineEventHandler(async (event) => {
-  const img = getQuery(event).img
-  if (img) {
-    const url = decodeURIComponent(img as string)
-    return sendProxy(event, url, {
-      headers: {
-        "Access-Control-Allow-Origin": "*",
-        "Access-Control-Allow-Credentials": "*",
-        "Access-Control-Allow-Methods": "GET, HEAD, POST, PUT, OPTIONS",
-        "Access-Control-Allow-Headers": "*",
-      },
-    })
-  }
-})

server/api/s/entries.post.ts

@@ -0,0 +1,14 @@
+import { getCacheTable } from "#/database/cache"
+
+export default defineEventHandler(async (event) => {
+  try {
+    const { sources } = await readBody(event)
+    const cacheTable = await getCacheTable()
+    if (sources && cacheTable) {
+      const data = await cacheTable.getEntries(sources)
+      return data
+    }
+  } catch {
+    //
+  }
+})

server/api/s/index.ts

@@ -0,0 +1,89 @@
+import type { SourceID, SourceResponse } from "@shared/types"
+import { getters } from "#/getters"
+import { getCacheTable } from "#/database/cache"
+import type { CacheInfo } from "#/types"
+
+export default defineEventHandler(async (event): Promise<SourceResponse> => {
+  try {
+    const query = getQuery(event)
+    const latest = query.latest !== undefined && query.latest !== "false"
+    let id = query.id as SourceID
+    const isValid = (id: SourceID) => !id || !sources[id] || !getters[id]
+
+    if (isValid(id)) {
+      const redirectID = sources?.[id]?.redirect
+      if (redirectID) id = redirectID
+      if (isValid(id)) throw new Error("Invalid source id")
+    }
+
+    const cacheTable = await getCacheTable()
+    const now = Date.now()
+    let cache: CacheInfo
+    if (cacheTable) {
+      cache = await cacheTable.get(id)
+      if (cache) {
+        // interval 刷新间隔，对于缓存失效也要执行的。本质上表示本来内容更新就很慢，这个间隔内可能内容压根不会更新。
+        // 默认 10 分钟，是低于 TTL 的，但部分 Source 的更新间隔会超过 TTL，甚至有的一天更新一次。
+        const interval = sources[id].interval
+        if (now - cache.updated < interval) {
+          return {
+            status: "success",
+            id,
+            updatedTime: now,
+            items: cache.data,
+          }
+        }
+
+        // 而 TTL 缓存失效时间，在时间范围内，就算内容更新了也要用这个缓存。
+        // 复用缓存是不会更新时间的。
+        if (now - cache.updated < TTL) {
+          // 有 latest
+          // 没有 latest，但服务器禁止登录
+
+          // 没有 latest
+          // 有 latest，服务器可以登录但没有登录
+          if (!latest || (!event.context.disabledLogin && !event.context.user)) {
+            return {
+              status: "cache",
+              id,
+              updatedTime: cache.updated,
+              items: cache.data,
+            }
+          }
+        }
+      }
+    }
+
+    try {
+      const newData = (await getters[id]()).slice(0, 30)
+      if (cacheTable && newData) {
+        if (event.context.waitUntil) event.context.waitUntil(cacheTable.set(id, newData))
+        else await cacheTable.set(id, newData)
+      }
+      logger.success(`fetch ${id} latest`)
+      return {
+        status: "success",
+        id,
+        updatedTime: now,
+        items: newData,
+      }
+    } catch (e) {
+      if (cache!) {
+        return {
+          status: "cache",
+          id,
+          updatedTime: cache.updated,
+          items: cache.data,
+        }
+      } else {
+        throw e
+      }
+    }
+  } catch (e: any) {
+    logger.error(e)
+    throw createError({
+      statusCode: 500,
+      message: e instanceof Error ? e.message : "Internal Server Error",
+    })
+  }
+})

server/database/cache.ts

@@ -40,6 +40,35 @@ export class Cache {
     return r
   }
 
+  async getEntries(keys: string[]) {
+    const keysStr = keys.map(k => `id = '${k}'`).join(" or ")
+    const res = await this.db.prepare(`SELECT id, data, updated FROM cache WHERE ${keysStr}`).all() as any
+
+    const rows = (res.results ?? res) as {
+      id: SourceID
+      data: string
+      updated: number
+    }[]
+
+    /**
+     * https://developers.cloudflare.com/d1/build-with-d1/d1-client-api/#return-object
+     * cloudflare d1 .all() will return
+     * {
+     *   success: boolean
+     *   meta:
+     *   results:
+     * }
+     */
+    if (rows?.length) {
+      logger.success(`get entries cache`)
+      return Object.fromEntries(rows.map(row => [row.id, {
+        id: row.id,
+        updatedTime: row.updated,
+        items: JSON.parse(row.data) as NewsItem[],
+      }]))
+    }
+  }
+
   async delete(key: string) {
     return await this.db.prepare(`DELETE FROM cache WHERE id = ?`).run(key)
   }
@@ -53,7 +82,7 @@ export async function getCacheTable() {
     const cacheTable = new Cache(db)
     if (process.env.INIT_TABLE !== "false") await cacheTable.init()
     return cacheTable
-  } catch (e) {
-    logger.error("failed to init database ", e)
+  } catch {
+    // logger.error("failed to init database ", e)
   }
 }

server/utils/fetch.ts

@@ -4,6 +4,6 @@ export const myFetch = $fetch.create({
   headers: {
     "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
   },
-  timeout: 5000,
+  timeout: 10000,
   retry: 3,
 })

shared/metadata.ts

@@ -27,7 +27,7 @@ export const columns = {
 } as const
 
 export const fixedColumnIds = ["focus", "hottest", "realtime"] as const satisfies Partial<ColumnID>[]
-export const hiddenColumns = Object.keys(sources).filter(id => !fixedColumnIds.includes(id as any)) as HiddenColumnID[]
+export const hiddenColumns = Object.keys(columns).filter(id => !fixedColumnIds.includes(id as any)) as HiddenColumnID[]
 
 export const metadata: Metadata = typeSafeObjectFromEntries(typeSafeObjectEntries(columns).map(([k, v]) => {
   switch (k) {

shared/types.ts

@@ -11,7 +11,7 @@ export type SourceID = {
   [Key in MainSourceID]: ConstSources[Key] extends { disable?: true } ? never :
     ConstSources[Key] extends { sub?: infer SubSource } ? {
     // @ts-expect-error >_<
-      [SubKey in keyof SubSource ]: SubSource[SubKey] extends { disable?: true } ? never : `${Key}-${SubKey}`
+      [SubKey in keyof SubSource]: SubSource[SubKey] extends { disable?: true } ? never : `${Key}-${SubKey}`
     }[keyof SubSource] | Key : Key;
 }[MainSourceID]
 
@@ -108,3 +108,5 @@ export interface SourceResponse {
   updatedTime: number | string
   items: NewsItem[]
 }
+
+export type EntriesSourceResponse = Partial<Record<SourceID, SourceResponse>>

src/atoms/index.ts

@@ -15,21 +15,6 @@ export const focusSourcesAtom = atom((get) => {
   })
 })
 
-function initRefetchSources() {
-  let time = 0
-  // useOnReload
-  // 没有放在 useOnReload 里面, 可以避免初始化后再修改 refetchSourceAtom，导致多次请求 API
-  const _ = localStorage.getItem("quitTime")
-  const now = Date.now()
-  const quitTime = _ ? Number(_) : 0
-  if (!Number.isNaN(quitTime) && now - quitTime < 1000) {
-    time = now
-  }
-  return Object.fromEntries(Object.keys(sources).map(k => [k, time])) as Record<SourceID, number>
-}
-
-export const refetchSourcesAtom = atom(initRefetchSources())
-
 export const currentColumnIDAtom = atom<FixedColumnID>("focus")
 
 export const currentSourcesAtom = atom((get) => {

src/components/column/card.tsx

@@ -1,12 +1,12 @@
 import type { NewsItem, SourceID, SourceResponse } from "@shared/types"
 import { useQuery } from "@tanstack/react-query"
-import { AnimatePresence, motion, useInView } from "framer-motion"
+import { AnimatePresence, motion } from "framer-motion"
 import type { SyntheticListenerMap } from "@dnd-kit/core/dist/hooks/utilities"
 import { useWindowSize } from "react-use"
 import { forwardRef, useImperativeHandle } from "react"
 import { OverlayScrollbar } from "../common/overlay-scrollbar"
-import { refetchSourcesAtom } from "~/atoms"
 import { safeParseString } from "~/utils"
+import { cache } from "~/utils/cache"
 
 export interface ItemsProps extends React.HTMLAttributes<HTMLDivElement> {
   id: SourceID
@@ -20,12 +20,10 @@ export interface ItemsProps extends React.HTMLAttributes<HTMLDivElement> {
 interface NewsCardProps {
   id: SourceID
   handleListeners?: SyntheticListenerMap
-  inView: boolean
 }
 
 export const CardWrapper = forwardRef<HTMLDivElement, ItemsProps>(({ id, isDragged, handleListeners, style, ...props }, dndRef) => {
   const ref = useRef<HTMLDivElement>(null)
-  const inView = useInView(ref)
 
   useImperativeHandle(dndRef, () => ref.current!)
 
@@ -44,33 +42,35 @@ export const CardWrapper = forwardRef<HTMLDivElement, ItemsProps>(({ id, isDragg
       }}
       {...props}
     >
-      <NewsCard id={id} inView={inView} handleListeners={handleListeners} />
+      <NewsCard id={id} handleListeners={handleListeners} />
     </div>
   )
 })
 
-const prevSourceItems: Partial<Record<SourceID, NewsItem[]>> = {}
-function NewsCard({ id, inView, handleListeners }: NewsCardProps) {
-  const [refetchSource, setRefetchSource] = useAtom(refetchSourcesAtom)
+function NewsCard({ id, handleListeners }: NewsCardProps) {
+  const { refresh, getRefreshId } = useRefetch()
   const { data, isFetching, isPlaceholderData, isError } = useQuery({
-    queryKey: [id, refetchSource[id]],
+    queryKey: [id, getRefreshId(id)],
     queryFn: async ({ queryKey }) => {
       const [_id, _refetchTime] = queryKey as [SourceID, number]
-      let url = `/api/s/${_id}`
+      let url = `/api/s?id=${_id}`
       const headers: Record<string, any> = {}
       if (Date.now() - _refetchTime < 1000) {
-        url = `/api/s/${_id}?latest`
+        url = `/api/s?id=${_id}&latest`
         const jwt = safeParseString(localStorage.getItem("jwt"))
         if (jwt) headers.Authorization = `Bearer ${jwt}`
+      } else if (cache.has(_id)) {
+        return cache.get(_id)
       }
+
       const response: SourceResponse = await myFetch(url, {
         headers,
       })
 
       try {
-        if (response.items && sources[_id].type === "hottest" && prevSourceItems[_id]) {
+        if (response.items && sources[_id].type === "hottest" && cache.has(_id)) {
           response.items.forEach((item, i) => {
-            const o = prevSourceItems[_id]!.findIndex(k => k.id === item.id)
+            const o = cache.get(_id)!.items.findIndex(k => k.id === item.id)
             item.extra = {
               ...item?.extra,
               diff: o === -1 ? undefined : o - i,
@@ -81,27 +81,14 @@ function NewsCard({ id, inView, handleListeners }: NewsCardProps) {
         console.log(e)
       }
 
+      cache.set(_id, response)
       return response
     },
-    // refetch 时显示原有的数据
-    placeholderData: (prev) => {
-      if (prev?.id === id) {
-        if (prev?.items && sources[id].type === "hottest") prevSourceItems[id] = prev.items
-        return prev
-      }
-    },
+    placeholderData: prev => prev,
+    staleTime: 1000 * 60 * 1,
     retry: false,
-    staleTime: 1000 * 60 * 5,
-    enabled: inView,
   })
 
-  const manualRefetch = useCallback(() => {
-    setRefetchSource(prev => ({
-      ...prev,
-      [id]: Date.now(),
-    }))
-  }, [setRefetchSource, id])
-
   const isFreshFetching = useMemo(() => isFetching && !isPlaceholderData, [isFetching, isPlaceholderData])
 
   const { isFocused, toggleFocus } = useFocusWith(id)
@@ -136,7 +123,7 @@ function NewsCard({ id, inView, handleListeners }: NewsCardProps) {
           <button
             type="button"
             className={$("btn i-ph:arrow-counter-clockwise-duotone", isFetching && "animate-spin i-ph:circle-dashed-duotone")}
-            onClick={manualRefetch}
+            onClick={() => refresh(id)}
           />
           <button
             type="button"
@@ -162,7 +149,7 @@ function NewsCard({ id, inView, handleListeners }: NewsCardProps) {
         options={{
           overflow: { x: "hidden" },
         }}
-        defer
+        defer={false}
       >
         <div className={$("transition-opacity-500", isFreshFetching && "op-20")}>
           {!!data?.items?.length && (sources[id].type === "hottest" ? <NewsListHot items={data.items} /> : <NewsListTimeLine items={data.items} />)}

src/components/column/dnd.tsx

@@ -17,12 +17,34 @@ import { SortableContext, arrayMove, defaultAnimateLayoutChanges, rectSortingStr
 import type { SourceID } from "@shared/types"
 import { CSS } from "@dnd-kit/utilities"
 import { motion } from "framer-motion"
+import { useQuery } from "@tanstack/react-query"
 import type { ItemsProps } from "./card"
 import { CardWrapper } from "./card"
 import { currentSourcesAtom } from "~/atoms"
 
 export function Dnd() {
   const [items, setItems] = useAtom(currentSourcesAtom)
+  useQuery({
+    queryKey: ["entries", items.sort()],
+    queryFn: async ({ queryKey }) => {
+      const sources = queryKey[1]
+      const res: EntriesSourceResponse = await myFetch("/api/s/entries", {
+        method: "POST",
+        body: {
+          sources,
+        },
+      })
+      if (res) {
+        for (const [k, v] of Object.entries(res)) {
+          cache.set(k as SourceID, v)
+        }
+        return res
+      }
+      return null
+    },
+    staleTime: 1000 * 60 * 5,
+  })
+
   return (
     <DndWrapper items={items} setItems={setItems}>
       <motion.ol

src/components/header/index.tsx

@@ -3,7 +3,7 @@ import { useIsFetching } from "@tanstack/react-query"
 import type { SourceID } from "@shared/types"
 import { NavBar } from "../navbar"
 import { Menu } from "./menu"
-import { currentSourcesAtom, goToTopAtom, refetchSourcesAtom } from "~/atoms"
+import { currentSourcesAtom, goToTopAtom } from "~/atoms"
 
 function GoTop() {
   const { ok, fn: goToTop } = useAtomValue(goToTopAtom)
@@ -19,14 +19,8 @@ function GoTop() {
 
 function Refresh() {
   const currentSources = useAtomValue(currentSourcesAtom)
-  const setRefetchSource = useSetAtom(refetchSourcesAtom)
-  const refreshAll = useCallback(() => {
-    const obj = Object.fromEntries(currentSources.map(id => [id, Date.now()]))
-    setRefetchSource(prev => ({
-      ...prev,
-      ...obj,
-    }))
-  }, [currentSources, setRefetchSource])
+  const { refresh } = useRefetch()
+  const refreshAll = useCallback(() => refresh(...currentSources), [refresh, currentSources])
 
   const isFetching = useIsFetching({
     predicate: (query) => {

src/hooks/usePWA.ts

@@ -16,9 +16,9 @@ export function usePWA() {
           label: "更新",
           onClick: () => updateServiceWorker(true),
         },
-        onDismiss: () => {
-          setNeedRefresh(false)
-        },
+        // onDismiss: () => {
+        //   setNeedRefresh(false)
+        // },
       })
     }
   }, [needRefresh, updateServiceWorker, setNeedRefresh, toaster])

src/hooks/useRefetch.ts

@@ -0,0 +1,34 @@
+import type { SourceID } from "@shared/types"
+
+function initRefetchSources() {
+  let time = 0
+  // useOnReload
+  // 没有放在 useOnReload 里面, 可以避免初始化后再修改 refetchSourceAtom，导致多次请求 API
+  const _ = localStorage.getItem("quitTime")
+  const now = Date.now()
+  const quitTime = _ ? Number(_) : 0
+  if (!Number.isNaN(quitTime) && now - quitTime < 1000) {
+    time = now
+  }
+  return Object.fromEntries(Object.keys(sources).map(k => [k, time])) as Record<SourceID, number>
+}
+
+const refetchSourcesAtom = atom(initRefetchSources())
+export function useRefetch() {
+  const [refetchSource, setRefetchSource] = useAtom(refetchSourcesAtom)
+
+  const refresh = useCallback((...sources: SourceID[]) => {
+    const obj = Object.fromEntries(sources.map(id => [id, Date.now()]))
+    setRefetchSource(prev => ({
+      ...prev,
+      ...obj,
+    }))
+  }, [setRefetchSource])
+
+  const getRefreshId = useCallback((id: SourceID) => refetchSource[id], [refetchSource])
+
+  return {
+    refresh,
+    getRefreshId,
+  }
+}

src/utils/cache.ts

@@ -0,0 +1,3 @@
+import type { SourceID, SourceResponse } from "@shared/types"
+
+export const cache: Map<SourceID, SourceResponse> = new Map()

src/utils/index.ts

@@ -38,6 +38,6 @@ export class Timer {
 }
 
 export const myFetch = $fetch.create({
-  timeout: 10000,
+  timeout: 15000,
   retry: 0,
 })