pref: server logic

2025-01-19 03:09:14 +08:00 · 2024-10-04 15:36:03 +08:00 · 2024-10-04 15:36:03 +08:00 · 6a8e3ba1c5
commit 6a8e3ba1c5
parent f2711fd80f
15 changed files with 161 additions and 104 deletions
--- a/server/cache.ts
+++ b/server/cache.ts
@ -9,6 +9,7 @@ export class Cache {
  }

  async init() {
+    const last = performance.now()
    await this.db.prepare(`
      CREATE TABLE IF NOT EXISTS cache (
        id TEXT PRIMARY KEY,
@ -17,23 +18,29 @@ export class Cache {
        expires INTEGER
      );
    `).run()
+    console.log(`init: `, performance.now() - last)
  }

  async set(key: string, value: any) {
    const now = Date.now()
-    return await this.db.prepare(
+    const last = performance.now()
+    await this.db.prepare(
      `INSERT OR REPLACE INTO cache (id, data, updated, expires) VALUES (?, ?, ?, ?)`,
    ).run(key, JSON.stringify(value), now, now + TTL)
+    console.log(`set ${key}: `, performance.now() - last)
  }

  async get(key: string): Promise<CacheInfo> {
+    const last = performance.now()
    const row: any = await this.db.prepare(`SELECT id, data, updated, expires FROM cache WHERE id = ?`).get(key)
-    return row
+    const r = row
      ? {
          ...row,
          data: JSON.parse(row.data),
        }
      : undefined
+    console.log(`get ${key}: `, performance.now() - last)
+    return r
  }

  async delete(key: string) {
--- a/server/routes/[id].ts
+++ b/server/routes/[id].ts
@ -11,7 +11,7 @@ export default defineEventHandler(async (event) => {
    const db = useDatabase()
    const cacheStore = db ? new Cache(db) : undefined
    if (cacheStore) {
-      await cacheStore.init()
+      // await cacheStore.init()
      const cache = await cacheStore.get(id)
      if (cache) {
        if (!latest && cache.expires > Date.now()) {
@ -29,14 +29,18 @@ export default defineEventHandler(async (event) => {
    }

    if (!sources[id]) {
+      const last = performance.now()
      const data = await fallback(id)
+      console.log(`fetch: ${id}`, performance.now() - last)
      if (cacheStore) await cacheStore.set(id, data)
      return {
        status: "success",
        data,
      }
    } else {
-      const data = await sources[id]()
+      const last = performance.now()
+      const data = await (await sources[id])()
+      console.log(`fetch: ${id}`, performance.now() - last)
      if (cacheStore) await cacheStore.set(id, data)
      return {
        status: "success",
--- a/server/sources/36kr-quick.ts
+++ b/server/sources/36kr-quick.ts
@ -0,0 +1,3 @@
+import { defineRSSSource } from "#/utils"
+
+export default defineRSSSource("https://rsshub.rssforever.com/36kr/newsflashes")
--- a/server/sources/fallback.ts
+++ b/server/sources/fallback.ts
@ -22,9 +22,7 @@ export async function fallback(id: string): Promise<SourceInfo> {
  const res: Res = await $fetch(url)
  if (res.code !== 200 || !res.data) throw new Error(res.message)
  return {
-    name: res.title,
-    type: res.subtitle,
-    updateTime: res.updateTime,
+    updatedTime: res.updateTime,
    items: res.data.map(item => ({
      extra: {
        date: item.time,
--- a/server/sources/index.ts
+++ b/server/sources/index.ts
@ -1,11 +1,14 @@
-import { peopledaily } from "./peopledaily"
-import { weibo } from "./weibo"
-import { zaobao } from "./zaobao"
+import type { SourceID, SourceInfo } from "@shared/types"
+import peopledaily from "./peopledaily"
+import weibo from "./weibo"
+import zaobao from "./zaobao"
+import kr from "./36kr-quick"

 export { fallback } from "./fallback"

 export const sources = {
  peopledaily,
  weibo,
-  zaobao: () => zaobao("中国聚焦"),
-}
+  zaobao,
+  "36kr-quick": kr,
+} as Record<SourceID, () => Promise<SourceInfo>>
--- a/server/sources/peopledaily.ts
+++ b/server/sources/peopledaily.ts
@ -1,17 +1,3 @@
-import type { RSS2JSON, SourceInfo } from "@shared/types"
-import { rss2json } from "#/utils/rss2json"
+import { defineRSSSource } from "#/utils"

-export async function peopledaily(): Promise<SourceInfo> {
-  const source = await rss2json("https://feedx.net/rss/people.xml")
-  if (!source?.items.length) throw new Error("Cannot fetch data")
-  return {
-    name: "人民日报",
-    type: "报纸",
-    updateTime: Date.now(),
-    items: source.items.slice(0, 30).map((item: RSS2JSON) => ({
-      title: item.title,
-      url: item.link,
-      id: item.link,
-    })),
-  }
-}
+export default defineRSSSource("https://feedx.net/rss/people.xml")
--- a/server/sources/weibo.ts
+++ b/server/sources/weibo.ts
@ -1,4 +1,4 @@
-import type { SourceInfo } from "@shared/types"
+import { defineSource } from "#/utils"

 interface Res {
  ok: number // 1 is ok
@ -26,15 +26,14 @@ interface Res {
  }
 }

-export async function weibo(): Promise<SourceInfo> {
+export default defineSource(async () => {
  const url = "https://weibo.com/ajax/side/hotSearch"
  const res: Res = await $fetch(url)
  if (!res.ok || res.data.realtime.length === 0) throw new Error("Cannot fetch data")
-  return {
-    name: "微博热搜",
-    updateTime: Date.now(),
-    type: "热搜",
-    items: res.data.realtime.filter(k => !k.icon_desc || k.icon_desc !== "荐").map((k) => {
+  return res.data.realtime
+    .filter(k => !k.icon_desc || k.icon_desc !== "荐")
+    .slice(0, 20)
+    .map((k) => {
      const keyword = k.word_scheme ? k.word_scheme : `#${k.word}#`
      return {
        id: k.num,
@ -45,6 +44,5 @@ export async function weibo(): Promise<SourceInfo> {
        url: `https://s.weibo.com/weibo?q=${encodeURIComponent(keyword)}`,
        mobileUrl: `https://m.weibo.cn/search?containerid=231522type%3D1%26q%3D${encodeURIComponent(keyword)}&_T_WM=16922097837&v_p=42`,
      }
-    }),
-  }
-}
+    })
+})
--- a/server/sources/zaobao.ts
+++ b/server/sources/zaobao.ts
@ -1,58 +1,43 @@
 import { Buffer } from "node:buffer"
 import * as cheerio from "cheerio"
 import iconv from "iconv-lite"
-import type { NewsItem, OResponse, SourceInfo } from "@shared/types"
+import type { NewsItem } from "@shared/types"
+import { $fetch } from "ofetch"
 import { tranformToUTC } from "#/utils/date"
+import { defineSource } from "#/utils"

-const columns = [
-  "人物记事",
-  "观点评论",
-  "中国聚焦",
-  "香港澳门",
-  "台湾新闻",
-  "国际时政",
-  "国际军事",
-  "国际视野",
-] as const
-export async function zaobao(type: typeof columns[number] = "中国聚焦"): Promise<SourceInfo> {
+export default defineSource(async () => {
  const response: ArrayBuffer = await $fetch("https://www.kzaobao.com/top.html", {
    responseType: "arrayBuffer",
  })
  const base = "https://www.kzaobao.com"
  const utf8String = iconv.decode(Buffer.from(response), "gb2312")
  const $ = cheerio.load(utf8String)
-  // const all = []
-  // columns.forEach((column, index) => {
-  const $main = $(`#cd0${columns.indexOf(type) + 1}`)
+  const $main = $("div[id^='cd0'] tr")
  const news: NewsItem[] = []
-  $main.find("tr").each((_, el) => {
+  $main.each((_, el) => {
    const a = $(el).find("h3>a")
    // https://www.kzaobao.com/shiju/20241002/170659.html
    const url = a.attr("href")
    const title = a.text()
-    if (url && title) {
-      const date = $(el).find("td:nth-child(3)").text()
+    const date = $(el).find("td:nth-child(3)").text()
+    if (url && title && date) {
      news.push({
        url: base + url,
        title,
        id: url,
        extra: {
-          date: date && tranformToUTC(date),
+          origin: date,
        },
      })
    }
  })
-  // all.push({
-  //   type: column,
-  //   items: news,
-  // })
-  // })
-  // console.log(all)
-  return {
-    name: `联合早报`,
-    type,
-    updateTime: Date.now(),
-    // items: all[0].items,
-    items: news,
-  }
-}
+  return news.sort((m, n) => n.extra!.origin > m.extra!.origin ? 1 : -1)
+    .slice(0, 20)
+    .map(item => ({
+      ...item,
+      extra: {
+        date: tranformToUTC(item.extra!.origin),
+      },
+    }))
+})
--- a/server/utils/index.ts
+++ b/server/utils/index.ts
@ -1 +1,26 @@
-import type { SourceInfo } from "@shared/types"
+import type { NewsItem, SourceInfo } from "@shared/types"
+
+export function defineSource(source: () => Promise<NewsItem[]>): () => Promise<SourceInfo> {
+  return async () => ({
+    updatedTime: Date.now(),
+    items: await source(),
+  })
+}
+
+export function defineRSSSource(url: string): () => Promise<SourceInfo> {
+  return async () => {
+    const source = await rss2json(url)
+    if (!source?.items.length) throw new Error("Cannot fetch data")
+    return {
+      updatedTime: source.updatedTime ?? Date.now(),
+      items: source.items.slice(0, 20).map(item => ({
+        title: item.title,
+        url: item.link,
+        id: item.link,
+        extra: {
+          date: item.created,
+        },
+      })),
+    }
+  }
+}
--- a/server/utils/rss2json.ts
+++ b/server/utils/rss2json.ts
@ -1,8 +1,9 @@
+import type { RSSInfo } from "@shared/types"
 import { XMLParser } from "fast-xml-parser"
 import { $fetch } from "ofetch"

-export async function rss2json(url: string) {
-  if (!/^https?:\/\/[^\s$.?#].\S*/i.test(url)) return null
+export async function rss2json(url: string): Promise<RSSInfo | undefined> {
+  if (!/^https?:\/\/[^\s$.?#].\S*/i.test(url)) return

  const data = await $fetch(url)

@ -23,6 +24,7 @@ export async function rss2json(url: string) {
    link: channel.link && channel.link.href ? channel.link.href : channel.link,
    image: channel.image ? channel.image.url : channel["itunes:image"] ? channel["itunes:image"].href : "",
    category: channel.category || [],
+    updatedTime: channel.lastBuildDate ?? channel.updated,
    items: [],
  }

@ -39,8 +41,7 @@ export async function rss2json(url: string) {
      description: val.summary && val.summary.$text ? val.summary.$text : val.description,
      link: val.link && val.link.href ? val.link.href : val.link,
      author: val.author && val.author.name ? val.author.name : val["dc:creator"],
-      published: val.created ? Date.parse(val.created) : val.pubDate ? Date.parse(val.pubDate) : Date.now(),
-      created: val.updated ? Date.parse(val.updated) : val.pubDate ? Date.parse(val.pubDate) : val.created ? Date.parse(val.created) : Date.now(),
+      created: val.updated ?? val.pubDate ?? val.created,
      category: val.category || [],
      content: val.content && val.content.$text ? val.content.$text : val["content:encoded"],
      enclosures: val.enclosure ? (Array.isArray(val.enclosure) ? val.enclosure : [val.enclosure]) : [],
--- a/shared/consts.ts
+++ b/shared/consts.ts
@ -1 +1,5 @@
 export const TTL = 15 * 60 * 1000
+/**
+ * 默认刷新间隔，否则复用缓存
+ */
+export const Interval = 30 * 60 * 1000
--- a/shared/data.ts
+++ b/shared/data.ts
@ -5,10 +5,19 @@ export const sectionIds = ["focus", "social", "china", "world", "digital"] as co
 export const sources = {
  "36kr": {
    name: "36氪",
+    type: "人气榜",
+    interval: 10,
+    home: "https://36kr.com",
+  },
+  "36kr-quick": {
+    name: "36氪",
+    type: "快讯",
+    interval: 3,
    home: "https://36kr.com",
  },
  "douyin": {
    name: "抖音",
+    interval: 1,
    home: "https://www.douyin.com",
  },
  "hupu": {
@ -17,18 +26,24 @@ export const sources = {
  },
  "zhihu": {
    name: "知乎",
+    interval: 10,
    home: "https://www.zhihu.com",
  },
  "weibo": {
    name: "微博",
+    type: "实时热搜",
+    interval: 1,
    home: "https://weibo.com",
  },
  "tieba": {
    name: "百度贴吧",
+    interval: 2,
    home: "https://tieba.baidu.com",
  },
  "zaobao": {
    name: "联合早报",
+    type: "实时新闻",
+    interval: 10,
    home: "https://www.zaobao.com",
  },
  "thepaper": {
@ -37,6 +52,7 @@ export const sources = {
  },
  "toutiao": {
    name: "今日头条",
+    interval: 2,
    home: "https://www.toutiao.com",
  },
  "cankaoxiaoxi": {
@ -49,6 +65,15 @@ export const sources = {
  },
 } as const satisfies Record<string, {
  name: string
+  type?: string
+  /**
+   * 分钟，刷新的间隔时间，复用缓存
+   */
+  interval?: number
+  /**
+   * 每天刷新一次
+   */
+  once?: number
  home: string
 }>

@ -63,7 +88,7 @@ export const metadata: Metadata = {
  },
  china: {
    name: "国内",
-    sourceList: ["peopledaily", "36kr", "toutiao"],
+    sourceList: ["peopledaily", "36kr", "toutiao", "36kr-quick"],
  },
  world: {
    name: "国外",
--- a/shared/types.ts
+++ b/shared/types.ts
@ -19,9 +19,7 @@ export interface NewsItem {

 // 路由数据
 export interface SourceInfo {
-  name: string
-  type: string
-  updateTime: number | string
+  updatedTime: number | string
  items: NewsItem[]
 }

@ -33,13 +31,19 @@ export type OResponse = {
  message?: string
 }

-export interface RSS2JSON {
-  id?: string
+export interface RSSInfo {
  title: string
  description: string
  link: string
-  published: number
-  created: number
+  image: string
+  updatedTime: string
+  items: RSSItem[]
+}
+export interface RSSItem {
+  title: string
+  description: string
+  link: string
+  created?: string
 }

 export interface CacheInfo {
--- a/src/components/section/Card.tsx
+++ b/src/components/section/Card.tsx
@ -1,4 +1,4 @@
-import type { OResponse, SourceID, SourceInfo } from "@shared/types"
+import type { NewsItem, SourceID, SourceInfo } from "@shared/types"
 import { OverlayScrollbarsComponent } from "overlayscrollbars-react"
 import type { UseQueryResult } from "@tanstack/react-query"
 import { useQuery } from "@tanstack/react-query"
@ -6,7 +6,7 @@ import { relativeTime } from "@shared/utils"
 import clsx from "clsx"
 import { useInView } from "react-intersection-observer"
 import { useAtom } from "jotai"
-import { forwardRef, useCallback, useEffect, useImperativeHandle, useRef } from "react"
+import { forwardRef, useCallback, useImperativeHandle, useRef } from "react"
 import { sources } from "@shared/data"
 import type { SyntheticListenerMap } from "@dnd-kit/core/dist/hooks/utilities"
 import { focusSourcesAtom, refetchSourceAtom } from "~/atoms"
@ -45,7 +45,7 @@ export const CardWrapper = forwardRef<HTMLDivElement, ItemsProps>(({ id, isDragg
    <div
      ref={ref}
      className={clsx(
-        "flex flex-col bg-base border rounded-md h-450px border-gray-500/40",
+        "flex flex-col h-500px aspect-auto border border-gray-100 rounded-xl shadow-2xl shadow-gray-600/10 bg-base dark:( border-gray-700 shadow-none)",
        isDragged && "op-50",
        isOverlay ? "bg-glass" : "",
      )}
@ -106,12 +106,13 @@ export function NewsCard({ id, inView, isOverlay, handleListeners }: NewsCardPro
        ])}
      >
        <div className="flex items-center gap-2">
-          <img src={`/icons/${id}.png`} className="w-4 h-4 rounded" alt={id} onError={e => e.currentTarget.hidden = true} />
+          <img src={`/icons/${id.split("-")[0]}.png`} className="w-4 h-4 rounded" alt={id} onError={e => e.currentTarget.hidden = true} />
          <span className="text-md font-bold">
            {sources[id].name}
          </span>
        </div>
-        <SubTitle query={query} />
+        {/* @ts-expect-error -_- */}
+        <span className="text-xs">{sources[id]?.type}</span>
      </div>
      <OverlayScrollbarsComponent
        defer
@ -136,13 +137,8 @@ export function NewsCard({ id, inView, isOverlay, handleListeners }: NewsCardPro
  )
 }

-function SubTitle({ query }: Query) {
-  const subTitle = query.data?.type
-  if (subTitle) return <span className="text-xs">{subTitle}</span>
-}
-
 function UpdateTime({ query }: Query) {
-  const updateTime = query.data?.updateTime
+  const updateTime = query.data?.updatedTime
  if (updateTime) return <span>{`${relativeTime(updateTime)}更新`}</span>
  if (query.isError) return <span>获取失败</span>
  return <span className="skeleton w-20" />
@ -157,6 +153,24 @@ function Num({ num }: { num: number }) {
  )
 }

+function ExtraInfo({ item }: { item: NewsItem }) {
+  if (item?.extra?.date) {
+    return (
+      <span className="text-xs text-gray-4/80 self-center">
+        {relativeTime(item.extra.date)}
+      </span>
+    )
+  }
+
+  if (item?.extra?.icon) {
+    return (
+      <span className="text-xs text-gray-4/80 self-start">
+        <img src={item.extra.icon} className="w-2em" />
+      </span>
+    )
+  }
+}
+
 function NewsList({ query }: Query) {
  const items = query.data?.items
  if (items?.length) {
@ -165,15 +179,11 @@ function NewsList({ query }: Query) {
        {items.slice(0, 20).map((item, i) => (
          <div key={item.title} className="flex gap-2 items-center">
            <Num num={i + 1} />
-            <a href={item.url} target="_blank" className="my-1 w-full flex items-center justify-between flex-wrap">
-              <span className="flex-1 mr-2 hover:(underline underline-offset-4)">
+            <a href={item.url} target="_blank" className="my-1 w-full flex justify-between flex-wrap">
+              <span className="flex-1 mr-2">
                {item.title}
              </span>
-              {item?.extra?.date && (
-                <span className="text-xs text-gray-4/80">
-                  {relativeTime(item.extra.date)}
-                </span>
-              )}
+              <ExtraInfo item={item} />
            </a>
          </div>
        ))}
--- a/src/routes/__root.tsx
+++ b/src/routes/__root.tsx
@ -28,8 +28,12 @@ export function RootComponent() {
    >
      <Header />
      <Outlet />
-      <ReactQueryDevtools buttonPosition="bottom-left" />
-      <TanStackRouterDevtools position="bottom-right" />
+      { import.meta.env.DEV && (
+        <>
+          <ReactQueryDevtools buttonPosition="bottom-left" />
+          <TanStackRouterDevtools position="bottom-right" />
+        </>
+      )}
    </OverlayScrollbarsComponent>
  )
 }