feat: add some sources

package.json

@@ -58,6 +58,7 @@
     "react-dom": "^18.3.1",
     "react-use": "^17.5.1",
     "sonner": "^1.5.0",
+    "uncrypto": "^0.1.3",
     "zod": "^3.23.8"
   },
   "devDependencies": {

pnpm-lock.yaml

@@ -113,6 +113,9 @@ importers:
       sonner:
         specifier: ^1.5.0
         version: 1.5.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
+      uncrypto:
+        specifier: ^0.1.3
+        version: 0.1.3
       zod:
         specifier: ^3.23.8
         version: 3.23.8

server/api/s/[id].ts

@@ -20,7 +20,7 @@ export default defineEventHandler(async (event): Promise<SourceResponse> => {
 
     const cacheTable = useCache()
     const now = Date.now()
-    if (cacheTable) {
+    if (process.env.NODE_ENV === "production" && cacheTable) {
       if (process.env.INIT_TABLE !== "false") await cacheTable.init()
       const cache = await cacheTable.get(id)
       if (cache) {

server/sources/36kr.ts

@@ -2,7 +2,8 @@ import type { NewsItem } from "@shared/types"
 import { load } from "cheerio"
 
 const quick = defineSource(async () => {
-  const url = "https://www.36kr.com/newsflashes"
+  const baseURL = "https://www.36kr.com"
+  const url = `${baseURL}/newsflashes`
   const response = await $fetch(url) as any
   const $ = load(response)
   const news: NewsItem[] = []
@@ -15,7 +16,7 @@ const quick = defineSource(async () => {
     const relativeDate = $el.find(".time")
     if (url && title && relativeDate) {
       news.push({
-        url: `https://www.36kr.com${url}`,
+        url: `${baseURL}${url}`,
         title,
         id: url,
         extra: {

server/sources/cankaoxiaoxi.ts

@@ -12,7 +12,6 @@ interface Res {
 
 export default defineSource(async () => {
   const res = await Promise.all(["zhongguo", "guandian", "gj"].map(k => $fetch(`https://china.cankaoxiaoxi.com/json/channel/${k}/list.json`) as Promise<Res>))
-  if (!res?.[0]?.list?.length) throw new Error("Cannot fetch data")
   return res.map(k => k.list).flat().map(k => ({
     id: k.data.id,
     title: k.data.title,
@@ -20,5 +19,5 @@ export default defineSource(async () => {
       date: tranformToUTC(k.data.publishTime),
     },
     url: k.data.url,
-  })).sort((m, n) => m.extra.date < n.extra.date ? 1 : -1).slice(0, 30)
+  })).sort((m, n) => m.extra.date < n.extra.date ? 1 : -1)
 })

server/sources/cls/index.ts

@@ -0,0 +1,68 @@
+import { getSearchParams } from "./utils"
+
+interface Item {
+  id: number
+  title?: string
+  brief: string
+  shareurl: string
+  // need *1000
+  ctime: number
+  // 1
+  is_ad: number
+}
+interface TelegraphRes {
+  data: {
+    roll_data: Item[]
+  }
+}
+
+interface Depthes {
+  data: {
+    top_article: Item[]
+    depth_list: Item[]
+  }
+}
+
+const depth = defineSource(async () => {
+  const apiUrl = `https://www.cls.cn/v3/depth/home/assembled/1000`
+  const res: Depthes = await $fetch(apiUrl, {
+    query: await getSearchParams(),
+  })
+  return res.data.depth_list.sort((m, n) => n.ctime - m.ctime).map((k) => {
+    return {
+      id: k.id,
+      title: k.title || k.brief,
+      mobileUrl: k.shareurl,
+      extra: {
+        date: k.ctime * 1000,
+      },
+      url: `https://www.cls.cn/detail/${k.id}`,
+    }
+  })
+})
+
+// hot 失效
+
+const telegraph = defineSource(async () => {
+  const apiUrl = `https://www.cls.cn/nodeapi/updateTelegraphList`
+  const res: TelegraphRes = await $fetch(apiUrl, {
+    query: await getSearchParams({ }),
+  })
+  return res.data.roll_data.filter(k => !k.is_ad).map((k) => {
+    return {
+      id: k.id,
+      title: k.title || k.brief,
+      mobileUrl: k.shareurl,
+      extra: {
+        date: k.ctime * 1000,
+      },
+      url: `https://www.cls.cn/detail/${k.id}`,
+    }
+  })
+})
+
+export default defineSource({
+  "cls": telegraph,
+  "cls-telegraph": telegraph,
+  "cls-depth": depth,
+})

server/sources/cls/utils.ts

@@ -0,0 +1,13 @@
+// https://github.com/DIYgod/RSSHub/blob/master/lib/routes/cls/utils.ts
+const params = {
+  appName: "CailianpressWeb",
+  os: "web",
+  sv: "7.7.5",
+}
+
+export async function getSearchParams(moreParams?: any) {
+  const searchParams = new URLSearchParams({ ...params, ...moreParams })
+  searchParams.sort()
+  searchParams.append("sign", await md5(await myCrypto(searchParams.toString(), "SHA-1")))
+  return searchParams
+}

server/sources/coolapk/index.ts

@@ -23,9 +23,9 @@ interface Res {
 export default defineSource(async () => {
   const url = "https://api.coolapk.com/v6/page/dataList?url=%2Ffeed%2FstatList%3FcacheExpires%3D300%26statType%3Dday%26sortField%3Ddetailnum%26title%3D%E4%BB%8A%E6%97%A5%E7%83%AD%E9%97%A8&title=%E4%BB%8A%E6%97%A5%E7%83%AD%E9%97%A8&subTitle=&page=1"
   const r: Res = await $fetch(url, {
-    headers: genHeaders(),
+    headers: await genHeaders(),
   })
-  if (!r.data || r.data.length === 0) throw new Error("Failed to fetch")
+  if (!r.data.length) throw new Error("Failed to fetch")
   return r.data.filter(k => k.id).map(i => ({
     id: i.id,
     title: i.editor_title || load(i.message).text().split("\n")[0],

server/sources/coolapk/utils.ts

@@ -1,6 +1,5 @@
 // https://github.com/DIYgod/RSSHub/blob/master/lib/routes/coolapk/utils.ts
 import { Buffer } from "node:buffer"
-import md5 from "md5"
 
 function getRandomDEVICE_ID() {
   const r = [10, 6, 6, 6, 14]
@@ -8,22 +7,22 @@ function getRandomDEVICE_ID() {
   return id.join("-")
 }
 
-function get_app_token() {
+async function get_app_token() {
   const DEVICE_ID = getRandomDEVICE_ID()
   const now = Math.round(Date.now() / 1000)
   const hex_now = `0x${now.toString(16)}`
-  const md5_now = md5(now.toString())
+  const md5_now = await md5(now.toString())
   const s = `token://com.coolapk.market/c67ef5943784d09750dcfbb31020f0ab?${md5_now}$${DEVICE_ID}&com.coolapk.market`
-  const md5_s = md5(Buffer.from(s).toString("base64"))
+  const md5_s = await md5(Buffer.from(s).toString("base64"))
   const token = md5_s + DEVICE_ID + hex_now
   return token
 }
 
-export function genHeaders() {
+export async function genHeaders() {
   return {
     "X-Requested-With": "XMLHttpRequest",
     "X-App-Id": "com.coolapk.market",
-    "X-App-Token": get_app_token(),
+    "X-App-Token": await get_app_token(),
     "X-Sdk-Int": "29",
     "X-Sdk-Locale": "zh-CN",
     "X-App-Version": "11.0",

server/sources/douyin.ts

@@ -1,6 +1,6 @@
 interface Res {
-  data?: {
-    word_list?: {
+  data: {
+    word_list: {
       sentence_id: string
       word: string
       event_time: string
@@ -33,16 +33,11 @@ export default defineSource(async () => {
       Cookie: `passport_csrf_token=${cookie}`,
     },
   })
-  if (!res?.data?.word_list || res.data.word_list.length === 0) throw new Error("Cannot fetch data")
   return res.data.word_list
-    .slice(0, 30)
     .map((k) => {
       return {
         id: k.sentence_id,
         title: k.word,
-        extra: {
-          info: k.hot_value,
-        },
         url: `https://www.douyin.com/hot/${k.sentence_id}`,
       }
     })

server/sources/index.ts

@@ -1,4 +1,4 @@
-import type { SourceID } from "@shared/types"
+import type { DisabledSourceID, SourceID } from "@shared/types"
 import weibo from "./weibo"
 import zaobao from "./zaobao"
 import v2ex from "./v2ex"
@@ -6,11 +6,12 @@ import ithome from "./ithome"
 import zhihu from "./zhihu"
 import cankaoxiaoxi from "./cankaoxiaoxi"
 import coolapk from "./coolapk"
-import sputniknewscn from "./sputniknewscn"
 import kr36 from "./36kr"
 import wallstreetcn from "./wallstreetcn"
 import douyin from "./douyin"
 import toutiao from "./toutiao"
+import cls from "./cls"
+import sputniknewscn from "./sputniknewscn"
 import type { SourceGetter } from "#/types"
 
 export const sourcesGetters = {
@@ -22,8 +23,9 @@ export const sourcesGetters = {
   coolapk,
   cankaoxiaoxi,
   sputniknewscn,
-  wallstreetcn,
+  ...wallstreetcn,
   douyin,
+  ...cls,
   toutiao,
   ...kr36,
-} as Record<SourceID, SourceGetter>
+} as Record<SourceID, SourceGetter> & Partial<Record<DisabledSourceID, SourceGetter>>

server/sources/ithome.ts

@@ -28,5 +28,4 @@ export default defineSource(async () => {
     }
   })
   return news.sort((m, n) => n.extra!.date > m.extra!.date ? 1 : -1)
-    .slice(0, 30)
 })

server/sources/sputniknewscn.ts

@@ -1,9 +1,8 @@
 import * as cheerio from "cheerio"
 import type { NewsItem } from "@shared/types"
-import { $fetch } from "ofetch"
 
 export default defineSource(async () => {
-  const response = await $fetch("https://sputniknews.cn/services/widget/lenta/")
+  const response: any = await $fetch("https://sputniknews.cn/services/widget/lenta/")
   const $ = cheerio.load(response)
   const $items = $(".lenta__item")
   const news: NewsItem[] = []
@@ -24,5 +23,5 @@ export default defineSource(async () => {
       })
     }
   })
-  return news.slice(0, 30)
+  return news
 })

server/sources/toutiao.ts

@@ -1,5 +1,5 @@
 interface Res {
-  data?: {
+  data: {
     ClusterIdStr: string
     Title: string
     HotValue: string
@@ -12,16 +12,11 @@ interface Res {
 export default defineSource(async () => {
   const url = "https://www.toutiao.com/hot-event/hot-board/?origin=toutiao_pc"
   const res: Res = await $fetch(url)
-  if (!res.data || res.data.length === 0) throw new Error("Cannot fetch data")
   return res.data
-    .slice(0, 30)
     .map((k) => {
       return {
         id: k.ClusterIdStr,
         title: k.Title,
-        extra: {
-          info: k.HotValue,
-        },
         url: `https://www.toutiao.com/trending/${k.ClusterIdStr}/`,
       }
     })

server/sources/v2ex.ts

@@ -18,7 +18,6 @@ interface Res {
 
 const share = defineSource(async () => {
   const res = await Promise.all(["create", "ideas", "programmer", "share"].map(k => $fetch(`https://www.v2ex.com/feed/${k}.json`) as Promise< Res>))
-  if (!res?.[0]?.items?.length) throw new Error("Cannot fetch data")
   return res.map(k => k.items).flat().map(k => ({
     id: k.id,
     title: k.title,

server/sources/wallstreetcn.ts

@@ -1,33 +1,87 @@
-interface Res {
+interface Item {
+  uri: string
+  id: number
+  title?: string
+  // ad
+  resource_type?: string
+  content_text: string
+  content_short: string
+  display_time: number
+  type?: string
+}
+interface LiveRes {
+  data: {
+    items: Item[]
+  }
+}
+
+interface NewsRes {
   data: {
     items: {
-      uri: string
-      id: number
-      title?: string
-      content_text: string
-      display_time: number
+      resource: Item
     }[]
   }
 }
 
-// https://github.com/DIYgod/RSSHub/blob/master/lib/routes/wallstreetcn/live.ts
-export default defineSource(async () => {
-  const category = "global"
-  const apiRootUrl = "https://api-one.wallstcn.com"
-  const apiUrl = `${apiRootUrl}/apiv1/content/lives?channel=${category}-channel&limit=30`
+interface HotRes {
+  data: {
+    day_items: Item[]
+  }
+}
 
-  const res: Res = await $fetch(apiUrl)
-  if (!res?.data?.items || res.data.items.length === 0) throw new Error("Cannot fetch data")
+// https://github.com/DIYgod/RSSHub/blob/master/lib/routes/wallstreetcn/live.ts
+const live = defineSource(async () => {
+  const apiUrl = `https://api-one.wallstcn.com/apiv1/content/lives?channel=global-channel&limit=30`
+
+  const res: LiveRes = await $fetch(apiUrl)
   return res.data.items
-    .slice(0, 30)
     .map((k) => {
       return {
         id: k.id,
         title: k.title || k.content_text,
         extra: {
-          date: new Date(k.display_time * 1000).getTime(),
+          date: k.display_time * 1000,
         },
         url: k.uri,
       }
     })
 })
+
+const news = defineSource(async () => {
+  const apiUrl = `https://api-one.wallstcn.com/apiv1/content/information-flow?channel=global-channel&accept=article&limit=30`
+
+  const res: NewsRes = await $fetch(apiUrl)
+  return res.data.items
+    .filter(k => k.resource.resource_type !== "ad" && k.resource.type !== "live")
+    .map(({ resource: h }) => {
+      return {
+        id: h.id,
+        title: h.title || h.content_short,
+        extra: {
+          date: h.display_time * 1000,
+        },
+        url: h.uri,
+      }
+    })
+})
+
+const hot = defineSource(async () => {
+  const apiUrl = `https://api-one.wallstcn.com/apiv1/content/articles/hot?period=all`
+
+  const res: HotRes = await $fetch(apiUrl)
+  return res.data.day_items
+    .map((h) => {
+      return {
+        id: h.id,
+        title: h.title!,
+        url: h.uri,
+      }
+    })
+})
+
+export default defineSource({
+  "wallstreetcn": live,
+  "wallstreetcn-quick": live,
+  "wallstreetcn-news": news,
+  "wallstreetcn-hot": hot,
+})

server/sources/weibo.ts

@@ -28,10 +28,8 @@ interface Res {
 export default defineSource(async () => {
   const url = "https://weibo.com/ajax/side/hotSearch"
   const res: Res = await $fetch(url)
-  if (!res.ok || res.data.realtime.length === 0) throw new Error("Cannot fetch data")
   return res.data.realtime
     .filter(k => !k.is_ad)
-    .slice(0, 30)
     .map((k) => {
       const keyword = k.word_scheme ? k.word_scheme : `#${k.word}#`
       return {

server/sources/zaobao.ts

@@ -31,5 +31,4 @@ export default defineSource(async () => {
     }
   })
   return news.sort((m, n) => n.extra!.date > m.extra!.date ? 1 : -1)
-    .slice(0, 30)
 })

server/sources/zhihu.ts

@@ -22,7 +22,6 @@ interface Res {
 export default defineSource(async () => {
   const url = "https://www.zhihu.com/api/v3/feed/topstory/hot-lists/total?limit=20&desktop=true"
   const res: Res = await $fetch(url)
-  if (!res.data || res.data.length === 0) throw new Error("Cannot fetch data")
   return res.data
     .slice(0, 30)
     .map((k) => {

server/utils/crypto.ts

@@ -0,0 +1,24 @@
+import _md5 from "md5"
+import { subtle as _ } from "uncrypto"
+
+type T = typeof crypto.subtle
+const subtle: T = _
+
+export async function md5(s: string) {
+  try {
+    // https://developers.cloudflare.com/workers/runtime-apis/web-crypto/
+    // cloudflare worker support md5
+    return await myCrypto(s, "MD5")
+  } catch {
+    return _md5(s)
+  }
+}
+
+type Algorithm = "MD5" | "SHA-1" | "SHA-256" | "SHA-384" | "SHA-512"
+export async function myCrypto(s: string, algorithm: Algorithm) {
+  const sUint8 = new TextEncoder().encode(s)
+  const hashBuffer = await subtle.digest(algorithm, sUint8)
+  const hashArray = Array.from(new Uint8Array(hashBuffer))
+  const hashHex = hashArray.map(b => b.toString(16).padStart(2, "0")).join("")
+  return hashHex
+}

server/utils/source.ts

@@ -1,8 +1,8 @@
-import type { SourceID } from "@shared/types"
+import type { AllSourceID, SourceID } from "@shared/types"
 import defu from "defu"
 import type { FallbackResponse, RSSHubOption, RSSHubInfo as RSSHubResponse, SourceGetter, SourceOption } from "#/types"
 
-type X = SourceGetter | Partial<Record<SourceID, SourceGetter>>
+type X = SourceGetter | Partial<Record<AllSourceID, SourceGetter>>
 export function defineSource<T extends X>(source: T): T {
   return source
 }

shared/metadata.ts

@@ -19,7 +19,7 @@ const originMetadata: Metadata = {
   },
   finance: {
     name: "财经",
-    sources: ["wallstreetcn", "36kr-quick"],
+    sources: ["cls-telegraph", "cls-depth", "wallstreetcn", "wallstreetcn-hot", "wallstreetcn-news"],
   },
   focus: {
     name: "关注",

shared/sources.ts

@@ -52,16 +52,30 @@ export const originSources = {
   },
   "wallstreetcn": {
     name: "华尔街见闻",
-    interval: Time.Fast,
-    type: "realtime",
     color: "blue",
     home: "https://wallstreetcn.com/",
-    title: "快讯",
+    sub: {
+      quick: {
+        type: "realtime",
+        interval: Time.Fast,
+        title: "实时快讯",
+      },
+      news: {
+        title: "最新资讯",
+        interval: Time.Common,
+      },
+      hot: {
+        title: "最热文章",
+        type: "hottest",
+        interval: Time.Common,
+      },
+    },
   },
   "36kr": {
     name: "36氪",
     type: "realtime",
     color: "blue",
+    disable: true,
     home: "https://36kr.com",
     sub: {
       quick: {
@@ -115,6 +129,22 @@ export const originSources = {
     interval: Time.Common,
     home: "https://china.cankaoxiaoxi.com",
   },
+  "cls": {
+    name: "财联社",
+    color: "red",
+    home: "https://www.cls.cn",
+    sub: {
+      telegraph: {
+        title: "电报",
+        interval: Time.Fast,
+        type: "realtime",
+      },
+      depth: {
+        title: "深度头条",
+        interval: Time.Common,
+      },
+    },
+  },
 } as const satisfies Record<string, OriginSource>
 
 export const sources = genSources()

shared/types.ts

@@ -15,6 +15,15 @@ export type SourceID = {
     }[keyof SubSource] | Key : Key;
 }[MainSourceID]
 
+export type AllSourceID = {
+  [Key in MainSourceID]: ConstSources[Key] extends { sub?: infer SubSource } ? keyof {
+    // @ts-expect-error >_<
+    [SubKey in keyof SubSource as `${Key}-${SubKey}`]: never
+  } | Key : Key
+}[MainSourceID]
+
+export type DisabledSourceID = Exclude<SourceID, MainSourceID>
+
 export type ColumnID = (typeof columnIds)[number]
 export type Metadata = Record<ColumnID, Column>
 

vite.config.ts

@@ -74,6 +74,7 @@ export default defineConfig({
       experimental: {
         database: true,
       },
+      sourceMap: false,
       database: {
         default: {
           connector: isCF ? "cloudflare-d1" : "sqlite",