diff --git a/server/cache.ts b/server/cache.ts index 73a939d..ef2d44e 100644 --- a/server/cache.ts +++ b/server/cache.ts @@ -9,6 +9,7 @@ export class Cache { } async init() { + const last = performance.now() await this.db.prepare(` CREATE TABLE IF NOT EXISTS cache ( id TEXT PRIMARY KEY, @@ -17,23 +18,29 @@ export class Cache { expires INTEGER ); `).run() + console.log(`init: `, performance.now() - last) } async set(key: string, value: any) { const now = Date.now() - return await this.db.prepare( + const last = performance.now() + await this.db.prepare( `INSERT OR REPLACE INTO cache (id, data, updated, expires) VALUES (?, ?, ?, ?)`, ).run(key, JSON.stringify(value), now, now + TTL) + console.log(`set ${key}: `, performance.now() - last) } async get(key: string): Promise { + const last = performance.now() const row: any = await this.db.prepare(`SELECT id, data, updated, expires FROM cache WHERE id = ?`).get(key) - return row + const r = row ? { ...row, data: JSON.parse(row.data), } : undefined + console.log(`get ${key}: `, performance.now() - last) + return r } async delete(key: string) { diff --git a/server/routes/[id].ts b/server/routes/[id].ts index 482feca..1bfa9dc 100644 --- a/server/routes/[id].ts +++ b/server/routes/[id].ts @@ -11,7 +11,7 @@ export default defineEventHandler(async (event) => { const db = useDatabase() const cacheStore = db ? new Cache(db) : undefined if (cacheStore) { - await cacheStore.init() + // await cacheStore.init() const cache = await cacheStore.get(id) if (cache) { if (!latest && cache.expires > Date.now()) { @@ -29,14 +29,18 @@ export default defineEventHandler(async (event) => { } if (!sources[id]) { + const last = performance.now() const data = await fallback(id) + console.log(`fetch: ${id}`, performance.now() - last) if (cacheStore) await cacheStore.set(id, data) return { status: "success", data, } } else { - const data = await sources[id]() + const last = performance.now() + const data = await (await sources[id])() + console.log(`fetch: ${id}`, performance.now() - last) if (cacheStore) await cacheStore.set(id, data) return { status: "success", diff --git a/server/sources/36kr-quick.ts b/server/sources/36kr-quick.ts new file mode 100644 index 0000000..9e4d3a9 --- /dev/null +++ b/server/sources/36kr-quick.ts @@ -0,0 +1,3 @@ +import { defineRSSSource } from "#/utils" + +export default defineRSSSource("https://rsshub.rssforever.com/36kr/newsflashes") diff --git a/server/sources/fallback.ts b/server/sources/fallback.ts index e6b8668..83a6721 100644 --- a/server/sources/fallback.ts +++ b/server/sources/fallback.ts @@ -22,9 +22,7 @@ export async function fallback(id: string): Promise { const res: Res = await $fetch(url) if (res.code !== 200 || !res.data) throw new Error(res.message) return { - name: res.title, - type: res.subtitle, - updateTime: res.updateTime, + updatedTime: res.updateTime, items: res.data.map(item => ({ extra: { date: item.time, diff --git a/server/sources/index.ts b/server/sources/index.ts index fedb83d..555d7ca 100644 --- a/server/sources/index.ts +++ b/server/sources/index.ts @@ -1,11 +1,14 @@ -import { peopledaily } from "./peopledaily" -import { weibo } from "./weibo" -import { zaobao } from "./zaobao" +import type { SourceID, SourceInfo } from "@shared/types" +import peopledaily from "./peopledaily" +import weibo from "./weibo" +import zaobao from "./zaobao" +import kr from "./36kr-quick" export { fallback } from "./fallback" export const sources = { peopledaily, weibo, - zaobao: () => zaobao("中国聚焦"), -} + zaobao, + "36kr-quick": kr, +} as Record Promise> diff --git a/server/sources/peopledaily.ts b/server/sources/peopledaily.ts index a0aeefa..378a8c1 100644 --- a/server/sources/peopledaily.ts +++ b/server/sources/peopledaily.ts @@ -1,17 +1,3 @@ -import type { RSS2JSON, SourceInfo } from "@shared/types" -import { rss2json } from "#/utils/rss2json" +import { defineRSSSource } from "#/utils" -export async function peopledaily(): Promise { - const source = await rss2json("https://feedx.net/rss/people.xml") - if (!source?.items.length) throw new Error("Cannot fetch data") - return { - name: "人民日报", - type: "报纸", - updateTime: Date.now(), - items: source.items.slice(0, 30).map((item: RSS2JSON) => ({ - title: item.title, - url: item.link, - id: item.link, - })), - } -} +export default defineRSSSource("https://feedx.net/rss/people.xml") diff --git a/server/sources/weibo.ts b/server/sources/weibo.ts index 28fc8e5..58a7ddd 100644 --- a/server/sources/weibo.ts +++ b/server/sources/weibo.ts @@ -1,4 +1,4 @@ -import type { SourceInfo } from "@shared/types" +import { defineSource } from "#/utils" interface Res { ok: number // 1 is ok @@ -26,15 +26,14 @@ interface Res { } } -export async function weibo(): Promise { +export default defineSource(async () => { const url = "https://weibo.com/ajax/side/hotSearch" const res: Res = await $fetch(url) if (!res.ok || res.data.realtime.length === 0) throw new Error("Cannot fetch data") - return { - name: "微博热搜", - updateTime: Date.now(), - type: "热搜", - items: res.data.realtime.filter(k => !k.icon_desc || k.icon_desc !== "荐").map((k) => { + return res.data.realtime + .filter(k => !k.icon_desc || k.icon_desc !== "荐") + .slice(0, 20) + .map((k) => { const keyword = k.word_scheme ? k.word_scheme : `#${k.word}#` return { id: k.num, @@ -45,6 +44,5 @@ export async function weibo(): Promise { url: `https://s.weibo.com/weibo?q=${encodeURIComponent(keyword)}`, mobileUrl: `https://m.weibo.cn/search?containerid=231522type%3D1%26q%3D${encodeURIComponent(keyword)}&_T_WM=16922097837&v_p=42`, } - }), - } -} + }) +}) diff --git a/server/sources/zaobao.ts b/server/sources/zaobao.ts index e692c43..60bddef 100644 --- a/server/sources/zaobao.ts +++ b/server/sources/zaobao.ts @@ -1,58 +1,43 @@ import { Buffer } from "node:buffer" import * as cheerio from "cheerio" import iconv from "iconv-lite" -import type { NewsItem, OResponse, SourceInfo } from "@shared/types" +import type { NewsItem } from "@shared/types" +import { $fetch } from "ofetch" import { tranformToUTC } from "#/utils/date" +import { defineSource } from "#/utils" -const columns = [ - "人物记事", - "观点评论", - "中国聚焦", - "香港澳门", - "台湾新闻", - "国际时政", - "国际军事", - "国际视野", -] as const -export async function zaobao(type: typeof columns[number] = "中国聚焦"): Promise { +export default defineSource(async () => { const response: ArrayBuffer = await $fetch("https://www.kzaobao.com/top.html", { responseType: "arrayBuffer", }) const base = "https://www.kzaobao.com" const utf8String = iconv.decode(Buffer.from(response), "gb2312") const $ = cheerio.load(utf8String) - // const all = [] - // columns.forEach((column, index) => { - const $main = $(`#cd0${columns.indexOf(type) + 1}`) + const $main = $("div[id^='cd0'] tr") const news: NewsItem[] = [] - $main.find("tr").each((_, el) => { + $main.each((_, el) => { const a = $(el).find("h3>a") // https://www.kzaobao.com/shiju/20241002/170659.html const url = a.attr("href") const title = a.text() - if (url && title) { - const date = $(el).find("td:nth-child(3)").text() + const date = $(el).find("td:nth-child(3)").text() + if (url && title && date) { news.push({ url: base + url, title, id: url, extra: { - date: date && tranformToUTC(date), + origin: date, }, }) } }) - // all.push({ - // type: column, - // items: news, - // }) - // }) - // console.log(all) - return { - name: `联合早报`, - type, - updateTime: Date.now(), - // items: all[0].items, - items: news, - } -} + return news.sort((m, n) => n.extra!.origin > m.extra!.origin ? 1 : -1) + .slice(0, 20) + .map(item => ({ + ...item, + extra: { + date: tranformToUTC(item.extra!.origin), + }, + })) +}) diff --git a/server/utils/index.ts b/server/utils/index.ts index 5ff5138..5bbcda3 100644 --- a/server/utils/index.ts +++ b/server/utils/index.ts @@ -1 +1,26 @@ -import type { SourceInfo } from "@shared/types" +import type { NewsItem, SourceInfo } from "@shared/types" + +export function defineSource(source: () => Promise): () => Promise { + return async () => ({ + updatedTime: Date.now(), + items: await source(), + }) +} + +export function defineRSSSource(url: string): () => Promise { + return async () => { + const source = await rss2json(url) + if (!source?.items.length) throw new Error("Cannot fetch data") + return { + updatedTime: source.updatedTime ?? Date.now(), + items: source.items.slice(0, 20).map(item => ({ + title: item.title, + url: item.link, + id: item.link, + extra: { + date: item.created, + }, + })), + } + } +} diff --git a/server/utils/rss2json.ts b/server/utils/rss2json.ts index d598345..bdb254f 100644 --- a/server/utils/rss2json.ts +++ b/server/utils/rss2json.ts @@ -1,8 +1,9 @@ +import type { RSSInfo } from "@shared/types" import { XMLParser } from "fast-xml-parser" import { $fetch } from "ofetch" -export async function rss2json(url: string) { - if (!/^https?:\/\/[^\s$.?#].\S*/i.test(url)) return null +export async function rss2json(url: string): Promise { + if (!/^https?:\/\/[^\s$.?#].\S*/i.test(url)) return const data = await $fetch(url) @@ -23,6 +24,7 @@ export async function rss2json(url: string) { link: channel.link && channel.link.href ? channel.link.href : channel.link, image: channel.image ? channel.image.url : channel["itunes:image"] ? channel["itunes:image"].href : "", category: channel.category || [], + updatedTime: channel.lastBuildDate ?? channel.updated, items: [], } @@ -39,8 +41,7 @@ export async function rss2json(url: string) { description: val.summary && val.summary.$text ? val.summary.$text : val.description, link: val.link && val.link.href ? val.link.href : val.link, author: val.author && val.author.name ? val.author.name : val["dc:creator"], - published: val.created ? Date.parse(val.created) : val.pubDate ? Date.parse(val.pubDate) : Date.now(), - created: val.updated ? Date.parse(val.updated) : val.pubDate ? Date.parse(val.pubDate) : val.created ? Date.parse(val.created) : Date.now(), + created: val.updated ?? val.pubDate ?? val.created, category: val.category || [], content: val.content && val.content.$text ? val.content.$text : val["content:encoded"], enclosures: val.enclosure ? (Array.isArray(val.enclosure) ? val.enclosure : [val.enclosure]) : [], diff --git a/shared/consts.ts b/shared/consts.ts index 0bc3b07..88b3ac5 100644 --- a/shared/consts.ts +++ b/shared/consts.ts @@ -1 +1,5 @@ export const TTL = 15 * 60 * 1000 +/** + * 默认刷新间隔,否则复用缓存 + */ +export const Interval = 30 * 60 * 1000 diff --git a/shared/data.ts b/shared/data.ts index 9ad769c..cb239c1 100644 --- a/shared/data.ts +++ b/shared/data.ts @@ -5,10 +5,19 @@ export const sectionIds = ["focus", "social", "china", "world", "digital"] as co export const sources = { "36kr": { name: "36氪", + type: "人气榜", + interval: 10, + home: "https://36kr.com", + }, + "36kr-quick": { + name: "36氪", + type: "快讯", + interval: 3, home: "https://36kr.com", }, "douyin": { name: "抖音", + interval: 1, home: "https://www.douyin.com", }, "hupu": { @@ -17,18 +26,24 @@ export const sources = { }, "zhihu": { name: "知乎", + interval: 10, home: "https://www.zhihu.com", }, "weibo": { name: "微博", + type: "实时热搜", + interval: 1, home: "https://weibo.com", }, "tieba": { name: "百度贴吧", + interval: 2, home: "https://tieba.baidu.com", }, "zaobao": { name: "联合早报", + type: "实时新闻", + interval: 10, home: "https://www.zaobao.com", }, "thepaper": { @@ -37,6 +52,7 @@ export const sources = { }, "toutiao": { name: "今日头条", + interval: 2, home: "https://www.toutiao.com", }, "cankaoxiaoxi": { @@ -49,6 +65,15 @@ export const sources = { }, } as const satisfies Record @@ -63,7 +88,7 @@ export const metadata: Metadata = { }, china: { name: "国内", - sourceList: ["peopledaily", "36kr", "toutiao"], + sourceList: ["peopledaily", "36kr", "toutiao", "36kr-quick"], }, world: { name: "国外", diff --git a/shared/types.ts b/shared/types.ts index c037f7e..cafb985 100644 --- a/shared/types.ts +++ b/shared/types.ts @@ -19,9 +19,7 @@ export interface NewsItem { // 路由数据 export interface SourceInfo { - name: string - type: string - updateTime: number | string + updatedTime: number | string items: NewsItem[] } @@ -33,13 +31,19 @@ export type OResponse = { message?: string } -export interface RSS2JSON { - id?: string +export interface RSSInfo { title: string description: string link: string - published: number - created: number + image: string + updatedTime: string + items: RSSItem[] +} +export interface RSSItem { + title: string + description: string + link: string + created?: string } export interface CacheInfo { diff --git a/src/components/section/Card.tsx b/src/components/section/Card.tsx index 5bf6e52..c8a548c 100644 --- a/src/components/section/Card.tsx +++ b/src/components/section/Card.tsx @@ -1,4 +1,4 @@ -import type { OResponse, SourceID, SourceInfo } from "@shared/types" +import type { NewsItem, SourceID, SourceInfo } from "@shared/types" import { OverlayScrollbarsComponent } from "overlayscrollbars-react" import type { UseQueryResult } from "@tanstack/react-query" import { useQuery } from "@tanstack/react-query" @@ -6,7 +6,7 @@ import { relativeTime } from "@shared/utils" import clsx from "clsx" import { useInView } from "react-intersection-observer" import { useAtom } from "jotai" -import { forwardRef, useCallback, useEffect, useImperativeHandle, useRef } from "react" +import { forwardRef, useCallback, useImperativeHandle, useRef } from "react" import { sources } from "@shared/data" import type { SyntheticListenerMap } from "@dnd-kit/core/dist/hooks/utilities" import { focusSourcesAtom, refetchSourceAtom } from "~/atoms" @@ -45,7 +45,7 @@ export const CardWrapper = forwardRef(({ id, isDragg
- {id} e.currentTarget.hidden = true} /> + {id} e.currentTarget.hidden = true} /> {sources[id].name}
- + {/* @ts-expect-error -_- */} + {sources[id]?.type}
{subTitle} -} - function UpdateTime({ query }: Query) { - const updateTime = query.data?.updateTime + const updateTime = query.data?.updatedTime if (updateTime) return {`${relativeTime(updateTime)}更新`} if (query.isError) return 获取失败 return @@ -157,6 +153,24 @@ function Num({ num }: { num: number }) { ) } +function ExtraInfo({ item }: { item: NewsItem }) { + if (item?.extra?.date) { + return ( + + {relativeTime(item.extra.date)} + + ) + } + + if (item?.extra?.icon) { + return ( + + + + ) + } +} + function NewsList({ query }: Query) { const items = query.data?.items if (items?.length) { @@ -165,15 +179,11 @@ function NewsList({ query }: Query) { {items.slice(0, 20).map((item, i) => ( ))} diff --git a/src/routes/__root.tsx b/src/routes/__root.tsx index 399ff59..8a9bfd2 100644 --- a/src/routes/__root.tsx +++ b/src/routes/__root.tsx @@ -28,8 +28,12 @@ export function RootComponent() { >
- - + { import.meta.env.DEV && ( + <> + + + + )} ) }