feat: add cache

.gitignore

@@ -3,4 +3,6 @@ dist/
 .vercel
 .output
 .vinxi
-.cache
+.cache
+.data
+.wrangler

package.json

@@ -22,17 +22,14 @@
     "@tanstack/react-router": "^1.58.9",
     "@tanstack/router-devtools": "^1.58.9",
     "@unocss/reset": "^0.62.4",
+    "better-sqlite3": "^9.4.3",
     "cheerio": "^1.0.0",
     "clsx": "^2.1.1",
     "dayjs": "^1.11.13",
     "fast-xml-parser": "^4.5.0",
     "favicons-scraper": "^1.3.2",
-    "flat-cache": "^6.1.0",
-    "h3": "^1.12.0",
     "iconv-lite": "^0.6.3",
     "jotai": "^2.10.0",
-    "node-fetch": "^3.3.2",
-    "ofetch": "^1.4.0",
     "overlayscrollbars": "^2.10.0",
     "overlayscrollbars-react": "^0.5.6",
     "react": "^18.3.1",
@@ -52,7 +49,9 @@
     "@vitejs/plugin-react-swc": "^3.7.0",
     "eslint": "^9.11.1",
     "eslint-plugin-react-hooks": "^5.1.0-rc-778e1ed2-20240926",
+    "ofetch": "^1.4.0",
     "eslint-plugin-react-refresh": "^0.4.12",
+    "nitro-cloudflare-dev": "^0.1.6",
     "nitropack": "^2.9.7",
     "tsx": "^4.19.1",
     "typescript": "^5.6.2",
@@ -61,6 +60,7 @@
     "vite": "^5.4.8",
     "vite-plugin-with-nitro": "0.0.0-beta.4",
     "vite-tsconfig-paths": "^5.0.1",
-    "vitest": "^2.1.1"
+    "vitest": "^2.1.1",
+    "wrangler": "^3.79.0"
   }
 }

schema.sql

@@ -0,0 +1,7 @@
+DROP TABLE IF EXISTS cache;
+CREATE TABLE IF NOT EXISTS cache (
+  id TEXT PRIMARY KEY,
+  data TEXT,
+  updated INTEGER,
+  expires INTEGER
+);

server/cache.ts

@@ -1,9 +1,38 @@
-import { FlatCache } from "flat-cache"
+import { TTL } from "@shared/consts"
+import type { CacheInfo } from "@shared/types"
 
-// init
+export class Cache {
-export const cache = new FlatCache({
+  private db
-  ttl: 60 * 60 * 1000, // 1 hour
+  constructor(db: any) {
-  lruSize: 10000, // 10,000 items
+    this.db = db
-  expirationInterval: 5 * 1000 * 60, // 5 minutes
+    this.db.exec(`
-  persistInterval: 5 * 1000 * 60, // 5 minutes
+      CREATE TABLE IF NOT EXISTS cache (
-})
+        id TEXT PRIMARY KEY,
+        data TEXT,
+        updated INTEGER,
+        expires INTEGER
+      );
+    `)
+  }
+
+  async set(key: string, value: any) {
+    const now = Date.now()
+    return await this.db.prepare(
+      `INSERT OR REPLACE INTO cache (id, data, updated, expires) VALUES (?, ?, ?, ?)`,
+    ).run(key, JSON.stringify(value), now, now + TTL)
+  }
+
+  async get(key: string): Promise<CacheInfo> {
+    const row = await this.db.prepare(`SELECT id, data, updated, expires FROM cache WHERE id = ?`).get(key)
+    return row
+      ? {
+          ...row,
+          data: JSON.parse(row.data),
+        }
+      : undefined
+  }
+
+  async delete(key: string) {
+    return await this.db.prepare(`DELETE FROM cache WHERE id = ?`).run(key)
+  }
+}

server/routes/[id].ts

@@ -1,24 +1,52 @@
-import { defineEventHandler, getRouterParam } from "h3"
 import { fallback, sources } from "#/sources"
-// import { cache } from "#/cache"
+import { Cache } from "#/cache"
 
 export default defineEventHandler(async (event) => {
-  const id = getRouterParam(event, "id") as keyof typeof sources
+  try {
-  // const { latest } = getQuery(event)
+    const id = getRouterParam(event, "id") as keyof typeof sources
-  // console.log(id, latest)
+    const query = getQuery(event)
-  if (!id) throw new Error("Invalid source id")
+    const latest = query.latest !== undefined && query.latest !== "false"
-  // if (!latest) {
-  //   const _ = cache.get(id)
-  //   if (_) return _
-  // }
 
-  if (!sources[id]) {
+    if (!id) throw new Error("Invalid source id")
-    const _ = await fallback(id)
+    const db = useDatabase()
-    // cache.set(id, _)
+    const cacheStore = db ? new Cache(db) : undefined
-    return _
+    if (cacheStore) {
-  } else {
+      const cache = await cacheStore.get(id)
-    const _ = await sources[id]()
+      if (cache) {
-    // cache.set(id, _)
+        if (!latest && cache.expires > Date.now()) {
-    return _
+          return {
+            status: "cache",
+            data: cache.data,
+          }
+        } else if (latest && Date.now() - cache.updated < 60 * 1000) {
+          return {
+            status: "success",
+            data: cache.data,
+          }
+        }
+      }
+    }
+
+    if (!sources[id]) {
+      const data = await fallback(id)
+      if (cacheStore) cacheStore.set(id, data)
+      return {
+        status: "success",
+        data,
+      }
+    } else {
+      const data = await sources[id]()
+      if (cacheStore) cacheStore.set(id, data)
+      return {
+        status: "success",
+        data,
+      }
+    }
+  } catch (e: any) {
+    console.error(e)
+    return {
+      status: "error",
+      message: e.message ?? e,
+    }
   }
 })

server/sources/fallback.ts

@@ -1,5 +1,4 @@
-import type { OResponse } from "@shared/types"
+import type { SourceInfo } from "@shared/types"
-import { $fetch } from "ofetch"
 
 export interface Res {
   code: number
@@ -18,24 +17,22 @@ export interface Res {
   }[]
 }
 
-export async function fallback(id: string): Promise<OResponse> {
+export async function fallback(id: string): Promise<SourceInfo> {
-  const res: Res = await $fetch(`https://smzdk.top/api/${id}/new`)
+  const url = `https://smzdk.top/api/${id}/new`
+  const res: Res = await $fetch(url)
   if (res.code !== 200 || !res.data) throw new Error(res.message)
   return {
-    status: "success",
+    name: res.title,
-    data: {
+    type: res.subtitle,
-      name: res.title,
+    updateTime: res.updateTime,
-      type: res.subtitle,
+    items: res.data.map(item => ({
-      updateTime: res.updateTime,
+      extra: {
-      items: res.data.map(item => ({
+        date: item.time,
-        extra: {
+      },
-          date: item.time,
+      id: item.url,
-        },
+      title: item.title,
-        id: item.url,
+      url: item.url,
-        title: item.title,
+      mobileUrl: item.mobileUrl,
-        url: item.url,
+    })),
-        mobileUrl: item.mobileUrl,
-      })),
-    },
   }
 }

server/sources/peopledaily.ts

@@ -1,20 +1,17 @@
-import type { OResponse, RSS2JSON } from "@shared/types"
+import type { RSS2JSON, SourceInfo } from "@shared/types"
 import { rss2json } from "#/utils/rss2json"
 
-export async function peopledaily(): Promise<OResponse> {
+export async function peopledaily(): Promise<SourceInfo> {
   const source = await rss2json("https://feedx.net/rss/people.xml")
   if (!source?.items.length) throw new Error("Cannot fetch data")
   return {
-    status: "success",
+    name: "人民日报",
-    data: {
+    type: "报纸",
-      name: "人民日报",
+    updateTime: Date.now(),
-      type: "报纸",
+    items: source.items.slice(0, 30).map((item: RSS2JSON) => ({
-      updateTime: Date.now(),
+      title: item.title,
-      items: source.items.slice(0, 30).map((item: RSS2JSON) => ({
+      url: item.link,
-        title: item.title,
+      id: item.link,
-        url: item.link,
+    })),
-        id: item.link,
-      })),
-    },
   }
 }

server/sources/weibo.ts

@@ -1,5 +1,4 @@
-import type { OResponse } from "@shared/types"
+import type { SourceInfo } from "@shared/types"
-import { $fetch } from "ofetch"
 
 interface Res {
   ok: number // 1 is ok
@@ -27,28 +26,25 @@ interface Res {
   }
 }
 
-export async function weibo(): Promise<OResponse> {
+export async function weibo(): Promise<SourceInfo> {
   const url = "https://weibo.com/ajax/side/hotSearch"
   const res: Res = await $fetch(url)
   if (!res.ok || res.data.realtime.length === 0) throw new Error("Cannot fetch data")
   return {
-    status: "success",
+    name: "微博热搜",
-    data: {
+    updateTime: Date.now(),
-      name: "微博热搜",
+    type: "热搜",
-      updateTime: Date.now(),
+    items: res.data.realtime.filter(k => !k.icon_desc || k.icon_desc !== "荐").map((k) => {
-      type: "热搜",
+      const keyword = k.word_scheme ? k.word_scheme : `#${k.word}#`
-      items: res.data.realtime.filter(k => !k.icon_desc || k.icon_desc !== "荐").map((k) => {
+      return {
-        const keyword = k.word_scheme ? k.word_scheme : `#${k.word}#`
+        id: k.num,
-        return {
+        title: k.word,
-          id: k.num,
+        extra: {
-          title: k.word,
+          icon: k.icon,
-          extra: {
+        },
-            icon: k.icon,
+        url: `https://s.weibo.com/weibo?q=${encodeURIComponent(keyword)}`,
-          },
+        mobileUrl: `https://m.weibo.cn/search?containerid=231522type%3D1%26q%3D${encodeURIComponent(keyword)}&_T_WM=16922097837&v_p=42`,
-          url: `https://s.weibo.com/weibo?q=${encodeURIComponent(keyword)}`,
+      }
-          mobileUrl: `https://m.weibo.cn/search?containerid=231522type%3D1%26q%3D${encodeURIComponent(keyword)}&_T_WM=16922097837&v_p=42`,
+    }),
-        }
-      }),
-    },
   }
 }

server/sources/zaobao.ts

@@ -1,8 +1,7 @@
 import { Buffer } from "node:buffer"
-import { $fetch } from "ofetch"
 import * as cheerio from "cheerio"
 import iconv from "iconv-lite"
-import type { NewsItem, OResponse } from "@shared/types"
+import type { NewsItem, OResponse, SourceInfo } from "@shared/types"
 import { tranformToUTC } from "#/utils/date"
 
 const columns = [
@@ -15,15 +14,16 @@ const columns = [
   "国际军事",
   "国际视野",
 ] as const
-// type: "中国聚焦" | "人物记事" | "观点评论"
+export async function zaobao(type: typeof columns[number] = "中国聚焦"): Promise<SourceInfo> {
-export async function zaobao(type: typeof columns[number] = "中国聚焦"): Promise<OResponse> {
   const response = await $fetch("https://www.kzaobao.com/top.html", {
     responseType: "arrayBuffer",
   })
   const base = "https://www.kzaobao.com"
   const utf8String = iconv.decode(Buffer.from(response), "gb2312")
   const $ = cheerio.load(utf8String)
-  const $main = $(`#cd0${columns.indexOf(type) + 1}`)
+  // const all = []
+  // columns.forEach((column, index) => {
+  const $main = $(`#cd0${2}`)
   const news: NewsItem[] = []
   $main.find("tr").each((_, el) => {
     const a = $(el).find("h3>a")
@@ -42,13 +42,17 @@ export async function zaobao(type: typeof columns[number] = "中国聚焦"): Pro
       })
     }
   })
+  // all.push({
+  //   type: column,
+  //   items: news,
+  // })
+  // })
+  // console.log(all)
   return {
-    status: "success",
+    name: `联合早报`,
-    data: {
+    type,
-      name: `联合早报`,
+    updateTime: Date.now(),
-      type,
+    // items: all[0].items,
-      updateTime: Date.now(),
+    items: news,
-      items: news,
-    },
   }
 }

server/types.ts

@@ -1,41 +0,0 @@
-import type { NewsItem, SourceInfo } from "@shared/types"
-
-// 榜单数据
-export interface ListItem extends NewsItem { }
-
-// 路由数据
-export interface RouterData extends SourceInfo { }
-
-// 请求类型
-export interface Get {
-  url: string
-  headers?: Record<string, string | string[]>
-  params?: Record<string, string | number>
-  timeout?: number
-  noCache?: boolean
-  ttl?: number
-  originaInfo?: boolean
-}
-
-export interface Post {
-  url: string
-  headers?: Record<string, string | string[]>
-  body?: string | object | import("node:buffer").Buffer | undefined
-  timeout?: number
-  noCache?: boolean
-  ttl?: number
-  originaInfo?: boolean
-}
-
-export interface Web {
-  url: string
-  timeout?: number
-  noCache?: boolean
-  ttl?: number
-  userAgent?: string
-}
-
-// 参数类型
-export interface Options {
-  [key: string]: string | number | undefined
-}

shared/consts.ts

@@ -0,0 +1 @@
+export const TTL = 15 * 60 * 1000

shared/types.ts

@@ -41,3 +41,10 @@ export interface RSS2JSON {
   published: number
   created: number
 }
+
+export interface CacheInfo {
+  id: SourceID
+  data: SourceInfo
+  updated: number
+  expires: number
+}

tsconfig.node.json

@@ -8,5 +8,5 @@
       "@shared/*": ["shared/*"]
     }
   },
-  "include": ["server", "*.config.*", "shared", "test", "scripts"]
+  "include": ["server", "*.config.*", "shared", "test", "scripts", "dist/.nitro/types"]
 }

vite.config.ts

@@ -1,5 +1,6 @@
 import process from "node:process"
 import { fileURLToPath } from "node:url"
+import nitroCloudflareBindings from "nitro-cloudflare-dev"
 import { defineConfig } from "vite"
 import react from "@vitejs/plugin-react-swc"
 import nitro from "vite-plugin-with-nitro"
@@ -20,14 +21,28 @@ export default defineConfig({
     react(),
     nitro({ ssr: false }, {
       srcDir: "server",
+      modules: [nitroCloudflareBindings],
+      experimental: {
+        database: true,
+      },
+      database: {
+        default: {
+          connector: "cloudflare-d1",
+          options: {
+            bindingName: "CACHE_DB",
+          },
+        },
+      },
+      devDatabase: {
+        default: {
+          connector: "sqlite",
+        },
+      },
       alias: {
         "@shared": fileURLToPath(new URL("shared", import.meta.url)),
         "#": fileURLToPath(new URL("server", import.meta.url)),
       },
-      runtimeConfig: {
+      preset: "cloudflare-pages",
-        // apiPrefix: "",
-      },
-      preset: process.env.VERCEL ? "vercel-edge" : "node-server",
     }),
   ],
 })

wrangler.toml

@@ -0,0 +1,4 @@
+[[d1_databases]]
+binding = "CACHE_DB" # i.e. available in your Worker on env.CACHE_DB
+database_name = "newsnow-cache"
+database_id = "1e091d20-2f57-48b4-b735-690ec65c725e"