From 1eaf523b289c62ea3d677d325f975f625690bc16 Mon Sep 17 00:00:00 2001 From: longsizhuo Date: Wed, 15 Apr 2026 17:27:01 +0000 Subject: [PATCH 01/19] =?UTF-8?q?fix(api/docs/history):=20=E5=85=BC?= =?UTF-8?q?=E5=AE=B9=20fumadocs=20=E7=9A=84=E7=9B=B8=E5=AF=B9=E8=B7=AF?= =?UTF-8?q?=E5=BE=84=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fumadocs 的 page.file.path 返回相对 app/docs/ 的路径（如 ai/xxx/index.mdx） 而不是仓库根路径，导致前端调 /api/docs/history 时报 400。 在 normalizeDocsPath 里补上 app/docs/ 前缀兼容。 --- app/api/docs/history/route.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/api/docs/history/route.ts b/app/api/docs/history/route.ts index 08ad6e30..714799a4 100644 --- a/app/api/docs/history/route.ts +++ b/app/api/docs/history/route.ts @@ -47,6 +47,11 @@ function normalizeDocsPath(raw: string): string | null { if (normalized.startsWith("docs/")) { normalized = `app/${normalized}`; } + // fumadocs 的 page.file.path 返回"相对 app/docs/"路径（如 ai/xxx/index.mdx） + // 而不是仓库根。这里补上前缀，和 page.tsx 传参保持兼容。 + if (!normalized.startsWith("app/")) { + normalized = `app/docs/${normalized}`; + } // 必须落在 app/docs/ 下才放行 if (!normalized.startsWith("app/docs/")) { return null; From 001fd1651b24e7a3e5683c923c73f3d4b32e1d46 Mon Sep 17 00:00:00 2001 From: longsizhuo Date: Wed, 15 Apr 2026 17:27:20 +0000 Subject: [PATCH 02/19] =?UTF-8?q?style(ui):=20=E5=AF=B9=E9=BD=90=20editori?= =?UTF-8?q?al=20=E8=AE=BE=E8=AE=A1=E7=B3=BB=E7=BB=9F=EF=BC=8C=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=206=20=E5=A4=84=E8=A7=86=E8=A7=89=E4=B8=80=E8=87=B4?= =?UTF-8?q?=E6=80=A7=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - DocShareButton: 去掉 rounded-md / h-11，改为 font-mono uppercase + border + 翻转色 hover - EditOnGithub: 同步改为相同风格，图标 h-8→h-4 - DocHistoryPanel: 骨架屏和头像去掉 rounded-full，保持方角报纸风 - ContributorRow: Dialog 硬编码 #111111/#F9F9F7 换成 CSS 变量；POWER LEVEL 标签移动端隐藏 - HotDocsTab: 加 sessionStorage 5 分钟缓存，减少重复打后端 - SettingsForm: AI 提供商从原生 select 改为 segment button group；中文 label 去掉 uppercase tracking-widest --- app/components/DocHistoryPanel.tsx | 10 +-- app/components/DocShareButton.tsx | 2 +- app/components/EditOnGithub.tsx | 4 +- app/components/rank/ContributorRow.tsx | 24 +++---- app/components/rank/HotDocsTab.tsx | 34 ++++++++- app/settings/SettingsForm.tsx | 95 +++++++++++++++++--------- 6 files changed, 115 insertions(+), 54 deletions(-) diff --git a/app/components/DocHistoryPanel.tsx b/app/components/DocHistoryPanel.tsx index b1a3298a..577c33c0 100644 --- a/app/components/DocHistoryPanel.tsx +++ b/app/components/DocHistoryPanel.tsx @@ -50,10 +50,10 @@ function relativeTime(dateStr: string): string { function SkeletonRow() { return (
-
+
-
-
+
+
); @@ -129,7 +129,7 @@ export function DocHistoryPanel({ path }: DocHistoryPanelProps) { href={item.htmlUrl} target="_blank" rel="noopener noreferrer" - className="flex items-start gap-3 py-2.5 group hover:bg-neutral-50 dark:hover:bg-neutral-900 rounded transition-colors px-1 -mx-1" + className="flex items-start gap-3 py-2.5 group hover:bg-neutral-50 dark:hover:bg-neutral-900 transition-colors px-1 -mx-1" > {/* 头像 */} {item.authorLogin} diff --git a/app/components/DocShareButton.tsx b/app/components/DocShareButton.tsx index f71fdeaa..9d0b5659 100644 --- a/app/components/DocShareButton.tsx +++ b/app/components/DocShareButton.tsx @@ -38,7 +38,7 @@ export function DocShareButton() {
{/* 提交按钮 */} From 967b07b41dedc5741c789b0ba0326068a2c961f3 Mon Sep 17 00:00:00 2001 From: longsizhuo Date: Wed, 15 Apr 2026 17:27:41 +0000 Subject: [PATCH 03/19] =?UTF-8?q?fix(settings):=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E4=B8=BB=E9=A2=98=E5=88=87=E6=8D=A2=E5=8F=8C=20Provider=20?= =?UTF-8?q?=E5=86=B2=E7=AA=81=20+=20=E5=8F=8C=E5=90=91=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 问题： - fumadocs 的 RootProvider 内置 next-themes，与自己的 ThemeProvider 同时往 写 light/dark，导致进入 /settings 时从黑闪白 - Header 右上角 ThemeToggle 切换主题后，/settings 表单选中态不更新 - Settings 保存时的语言设置写不到文档页能读到的位置 修复： - RootProvider 加 theme={{ enabled: false }}，禁用 fumadocs 内置主题 - SettingsForm 用 useTheme() 读当前主题作为初始值 - 监听 currentTheme 变化，同步到表单选中态 - 切主题按钮立即 setTheme（所见即所得，不用等保存） - handleSave 保存成功后把 language 写 cookie，供 /docs Server Component 读 --- app/layout.tsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/layout.tsx b/app/layout.tsx index c28ec0f2..c98c631a 100644 --- a/app/layout.tsx +++ b/app/layout.tsx @@ -206,6 +206,9 @@ export default async function RootLayout({ 写 light/dark 导致闪烁和状态不同步 + theme={{ enabled: false }} search={{ SearchDialog: CustomSearchDialog, // 使用静态索引，兼容 next export 与本地开发 From cf6997d6839bfe75d1ca74063770a989ab77dd99 Mon Sep 17 00:00:00 2001 From: longsizhuo Date: Wed, 15 Apr 2026 17:28:21 +0000 Subject: [PATCH 04/19] =?UTF-8?q?fix(nav):=20=E9=9D=9E=E9=A6=96=E9=A1=B5?= =?UTF-8?q?=E7=82=B9=E5=87=BB=20Header=20=E4=B9=9F=E8=83=BD=E5=9B=9E?= =?UTF-8?q?=E4=B8=BB=E9=A1=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - BrandMark 外层包 ，点 logo 回首页 - Header 三个锚点从 #features 改为 /#features 等绝对路径， 从 /rank 或任意子页点导航能跳回主页对应区块 方案 F（最小修复），比引入 Header 全站 layout 或新组件成本更低。 --- app/components/BrandMark.tsx | 5 +++-- app/components/Header.tsx | 15 ++++++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/app/components/BrandMark.tsx b/app/components/BrandMark.tsx index efbfa1f2..b86ac1cc 100644 --- a/app/components/BrandMark.tsx +++ b/app/components/BrandMark.tsx @@ -12,6 +12,7 @@ * @param {boolean} priority - 是否优先加载 */ import Image from "next/image"; +import Link from "next/link"; import { cn } from "@/lib/utils"; export const BRAND_NAME = "Involution Hell"; @@ -38,7 +39,7 @@ export function BrandMark({ const width = Math.round(imageSize * BRAND_LOGO_ASPECT_RATIO); return ( -
+
{BRAND_NAME} -
+ ); } diff --git a/app/components/Header.tsx b/app/components/Header.tsx index 2a75f041..a3184096 100644 --- a/app/components/Header.tsx +++ b/app/components/Header.tsx @@ -31,7 +31,16 @@ export function Header() {
-
- {(() => { - const rawData = leaderboardData as { - id: string; - name: string; - points: number; - avatarUrl: string; - }[]; - const filteredData = rawData.filter( - (user) => !MAINTAINERS.includes(user.name), - ); - const top3 = filteredData.slice(0, 3); - const maxPoints = top3.length > 0 ? top3[0].points : 100; +
+
+ {(() => { + const rawData = leaderboardData as { + id: string; + name: string; + points: number; + avatarUrl: string; + }[]; + const filteredData = rawData.filter( + (user) => !MAINTAINERS.includes(user.name), + ); + const top3 = filteredData.slice(0, 3); + const maxPoints = top3.length > 0 ? top3[0].points : 100; - return top3.map((user, idx) => ( -
-
- #{idx + 1} -
-
- {user.name} -
-
- {user.name} -
-
- {user.points.toLocaleString()} PTS + return top3.map((user, idx) => ( +
+
+ #{idx + 1} +
+
+ {user.name} +
+
+ {user.name} +
+
+ {user.points.toLocaleString()} PTS +
+
- - {/* Visual bar chart representing points using motion */} - -
- )); - })()} + )); + })()} +
+
+ +
diff --git a/app/components/HotDocsPreview.tsx b/app/components/HotDocsPreview.tsx new file mode 100644 index 00000000..37260593 --- /dev/null +++ b/app/components/HotDocsPreview.tsx @@ -0,0 +1,82 @@ +import Link from "next/link"; + +interface TopDocDto { + path: string; + title: string; + views: number; +} + +async function fetchTopDocs(): Promise { + const backendUrl = process.env.BACKEND_URL ?? "http://localhost:8081"; + try { + const res = await fetch( + `${backendUrl}/analytics/top-docs?window=7d&limit=5`, + { next: { revalidate: 300 } }, + ); + if (!res.ok) return []; + const json = await res.json(); + // 后端用 ApiResponse> 包裹，data 字段存实际数据 + return json.data ?? json; + } catch { + return []; + } +} + +export async function HotDocsPreview() { + const docs = await fetchTopDocs(); + + return ( +
+
+
+
+ Hot This Week +
+
+ 本周最热 +
+
+ + MORE + + → + + +
+ + {docs.length === 0 ? ( +

暂无数据

+ ) : ( +
    + {docs.map((doc, idx) => ( +
  1. + + {String(idx + 1).padStart(2, "0")} + +
    + + {doc.title} + +
    + {doc.views.toLocaleString()} views +
    +
    +
    2. + ))} +
+ )} +
+ ); +} From 93c1648c84eee75f8110079ef7936bccd79da070 Mon Sep 17 00:00:00 2001 From: longsizhuo Date: Wed, 15 Apr 2026 17:29:17 +0000 Subject: [PATCH 07/19] =?UTF-8?q?feat(scripts):=20contributors=20=E8=84=9A?= =?UTF-8?q?=E6=9C=AC=E8=B7=B3=E8=BF=87=E7=BF=BB=E8=AF=91=E7=89=88=E6=96=87?= =?UTF-8?q?=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 翻译版文件 frontmatter 有 translatedFrom 字段，这类文件由 AI 产出、不应污染 contributor 统计。 在 parseDocFrontmatter 新增 isTranslation 字段，主循环里 检测到翻译版则打印跳过日志并 continue，不拉 commit 历史、 不写入 doc_contributors 表。 影响：generate-leaderboard.mjs 无需改动（它从 DB 聚合， 源头已过滤），DB schema 不变。 --- scripts/backfill-contributors.mjs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/backfill-contributors.mjs b/scripts/backfill-contributors.mjs index 272b70e5..6444b989 100644 --- a/scripts/backfill-contributors.mjs +++ b/scripts/backfill-contributors.mjs @@ -165,15 +165,19 @@ async function listDocFiles() { .sort((a, b) => a.relative.localeCompare(b.relative)); } -// 解析 frontmatter，取 docId / title +// 解析 frontmatter，取 docId / title / isTranslation function parseDocFrontmatter(content) { const parsed = matter(content); const data = parsed.data || {}; const docId = typeof data.docId === "string" ? data.docId.trim() : ""; const title = typeof data.title === "string" ? data.title.trim() : ""; + // 有 translatedFrom 字段即为翻译版，不计入贡献者统计 + const isTranslation = + typeof data.translatedFrom === "string" && data.translatedFrom.length > 0; return { docId: docId || null, title: title || null, + isTranslation, frontmatter: data, }; } @@ -485,6 +489,11 @@ async function main() { log(` ⚠️ 跳过 ${repoRelative}：缺少 docId`); continue; } + // 翻译版（frontmatter 有 translatedFrom）不统计贡献者 + if (meta.isTranslation) { + log(` ⏭ 跳过翻译版：${repoRelative}`); + continue; + } const set = currentDocIdPaths.get(meta.docId) ?? new Set(); set.add(repoRelative); currentDocIdPaths.set(meta.docId, set); From 32031b75e1b5bc735ee8c12dc09099bbdaabedd1 Mon Sep 17 00:00:00 2001 From: longsizhuo Date: Wed, 15 Apr 2026 17:29:37 +0000 Subject: [PATCH 08/19] =?UTF-8?q?feat(docs):=20i18n=20MVP=20=E2=80=94=20?= =?UTF-8?q?=E7=BF=BB=E8=AF=91=205=20=E7=AF=87=E4=BB=A3=E8=A1=A8=E6=80=A7?= =?UTF-8?q?=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 覆盖不同场景的 MVP 样本，用于验证翻译 agent 的能力边界： - ai/reinforcement-learning-overview (zh→en) — AI 术语密集 - ai/compute-platforms-handbook (zh→en) — 代码块保留 - computer-science/01-singly-linked-list (en→zh) — 长文 + ASCII 图 - jobs/bq (zh→en) — 文化语境 / STAR / BQ 行话 - CommunityShare/Geek/cloudflare-r2-sharex (zh→en) — 表格 + 中文 URL 编码 每份翻译版： - 继承原文 docId（共享同一文档的版本） - frontmatter 有 translatedFrom/translatedAt 标记，contributors 脚本会跳过 - 代码块 / math / URL / MDX 属性名原样保留 - 仅翻译 title、description、正文文本与 MDX 文本内容 MVP 5 篇总 token ~29.4k，平均 5.9k/篇。 --- ...dflare-r2-sharex-free-image-hosting.en.mdx | 153 +++++ .../compute-platforms-handbook.en.mdx | 219 +++++++ .../reinforcement-learning-overview.en.mdx | 154 +++++ .../linked-list/01-singly-linked-list.zh.mdx | 570 ++++++++++++++++++ app/docs/jobs/interview-prep/bq.en.md | 70 +++ 5 files changed, 1166 insertions(+) create mode 100644 app/docs/CommunityShare/Geek/cloudflare-r2-sharex-free-image-hosting.en.mdx create mode 100644 app/docs/ai/compute-platforms/compute-platforms-handbook.en.mdx create mode 100644 app/docs/ai/reinforcement-learning/reinforcement-learning-overview.en.mdx create mode 100644 app/docs/computer-science/data-structures/linked-list/01-singly-linked-list.zh.mdx create mode 100644 app/docs/jobs/interview-prep/bq.en.md diff --git a/app/docs/CommunityShare/Geek/cloudflare-r2-sharex-free-image-hosting.en.mdx b/app/docs/CommunityShare/Geek/cloudflare-r2-sharex-free-image-hosting.en.mdx new file mode 100644 index 00000000..0bf00661 --- /dev/null +++ b/app/docs/CommunityShare/Geek/cloudflare-r2-sharex-free-image-hosting.en.mdx @@ -0,0 +1,153 @@ +--- +title: Building a Personal/Team "Permanent" Image Host with Cloudflare R2 + ShareX +description: "" +date: "2025-09-27" +tags: + - tag-one +docId: gj4bn01un0s0841berfvwrn5 +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T00:00:00Z +translatorAgent: claude-opus-4-6 +--- + +# Building a Personal/Team "Permanent" Image Host with Cloudflare R2 + ShareX + +This guide walks you from zero to a working image host — high-performance, reliable, near-zero cost, and fully under your control — built on Cloudflare R2's free tier and ShareX's workflow automation. + +**End result**: press a hotkey to take a screenshot, have the image automatically uploaded to your own storage (or upload existing local images manually), and get a Markdown-formatted link copied straight to your clipboard for seamless writing. + +--- + +## Contents + +1. [Part 1: Configure Cloudflare R2 (cloud storage)](#part-1-configure-cloudflare-r2-cloud-storage) +2. [Part 2: Configure ShareX (desktop client)](#part-2-configure-sharex-desktop-client) +3. [Part 3: Optimize the ShareX workflow](#part-3-optimize-the-sharex-workflow) +4. [F.A.Q. and troubleshooting](#faq-and-troubleshooting) + +--- + +## Part 1: Configure Cloudflare R2 (cloud storage) + +First, enable R2 storage in Cloudflare. +![](https://pub-85d4dcece16844bf8290aa4b33608ccd.r2.dev/ShareX/2025/09/%E5%9B%BE%E7%89%87_20250927143514.png) + +### 1.1 Create an R2 bucket + +A bucket is the container that holds all your images. + +1. Sign in to the Cloudflare dashboard and open **R2** from the left-hand menu. +2. Click **Create bucket**. +3. **Bucket name**: enter a globally unique bucket name (for example, `your-org-images-2025`). +4. **Location**: leave the default **Automatic**. +5. Click **Create bucket**. + +![](https://pub-85d4dcece16844bf8290aa4b33608ccd.r2.dev/ShareX/2025/09/xrRAVMB2Sz.png) + +### 1.2 Enable public access on the bucket + +To let uploaded images be served externally, turn on public access. + +1. Open the bucket you just created and click the **Settings** tab at the top. + ![1r0AU3aWkD.png](https://pub-85d4dcece16844bf8290aa4b33608ccd.r2.dev/ShareX/2025/09/1r0AU3aWkD.png) +2. In the **Public Development URL** section below, click **Enable** on the right. +3. Type the confirmation text. + ![fgc4amk7S7.png](https://pub-85d4dcece16844bf8290aa4b33608ccd.r2.dev/ShareX/2025/09/fgc4amk7S7.png) +4. **Note down** the `https://pub-....r2.dev` URL shown here — this is your public-access domain. + ![RmQwxSxpLi.png](https://pub-85d4dcece16844bf8290aa4b33608ccd.r2.dev/ShareX/2025/09/RmQwxSxpLi.png) + +### 1.3 Create an API token for uploads + +The API token is the "key" that lets ShareX upload files to R2. + +1. Return to the R2 overview page and click **Manage API Tokens** in the top-right corner. + ![CTzhiiSl04.png](https://pub-85d4dcece16844bf8290aa4b33608ccd.r2.dev/ShareX/2025/09/CTzhiiSl04.png) +1. Click **Create Account API token**. + ![FBEzXXohz7.png](https://pub-85d4dcece16844bf8290aa4b33608ccd.r2.dev/ShareX/2025/09/FBEzXXohz7.png) +1. **Permissions**: **you must select `Object Read & Write`**. This is the critical step — read-only permissions will cause uploads to fail. + ![xB4pYQOeEI.png](https://pub-85d4dcece16844bf8290aa4b33608ccd.r2.dev/ShareX/2025/09/xB4pYQOeEI.png) +1. Click **Create API token**. +1. **⚠️ Copy and save immediately!** The page displays your `Access Key ID` and `Secret Access Key`. **These two secrets are shown only once** — copy them immediately and paste them somewhere safe. The `Default endpoints` link at the bottom also needs to be saved. + ![kg9E8tEozI.png](https://pub-85d4dcece16844bf8290aa4b33608ccd.r2.dev/ShareX/2025/09/kg9E8tEozI.png) + +--- + +## Part 2: Configure ShareX (desktop client) + +### 2.1 Download and install ShareX + +Grab the latest version from the official site: [https://getsharex.com/](https://getsharex.com/) + +### 2.2 Configure the S3 upload destination + +This is the heart of the setup. + +1. Open ShareX and, from the main window, click `Destination settings...`. +2. In the popup, select `Amazon S3` on the left and fill in your configuration on the right. +3. Fill in your R2 details exactly as shown in the table below: + +| ShareX field | What to enter | Notes | +| --------------------- | --------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | +| **Access key ID** | Paste the `Access Key ID` you saved | | +| **Secret access key** | Paste the `Secret Access Key` you saved | | +| **Region** | Leave blank | | +| **Endpoints** | Leave blank | +| **Endpoint** | `https://.r2.cloudflarestorage.com` | If you saved this earlier, paste it as-is. Make sure there's no trailing `/`. | +| **Bucket name** | Your R2 bucket name (for example, `your-org-images-2025`) | | +| **Upload path** | `img/%y/%mo/%d/` | Organizes images as `img/year/month/day/`, which helps with management. ShareX uses `%` for variables. (Customize as you like.) | +| **Use custom domain** | **Check this box** | | +| (Custom domain field) | `https://` | **Note**: only paste the Public Development URL you saved earlier — don't append `$key$`. Modern ShareX handles that automatically. | + +#### 2.2.1 Critical Advanced settings + +At the bottom of the S3 configuration window, find the **Advanced** section and apply these settings: + +- `Set public-read ACL on file`: **must be unchecked**. R2 doesn't support this; leaving it checked will produce `403 Forbidden` errors. +- `Use path style request`: **must be checked**. R2 requires this request URL style. + +### 2.3 Set S3 as the default image uploader + +1. Return to the ShareX main window. +2. Click `Destinations` -> `Image uploader` -> `File uploader` -> and pick `Amazon S3`. + ![Code_nqStY1UhqR.png](https://pub-85d4dcece16844bf8290aa4b33608ccd.r2.dev/ShareX/2025/09/Code_nqStY1UhqR.png) + +--- + +## Part 3: Optimize the ShareX workflow + +### 3.1 Auto-copy a Markdown link + +1. In the ShareX main window, click `Task settings...`. +2. In the popup, select **`Advanced`** from the lower part of the left sidebar. + ![ShareX_ZWFVlZZu0W.png](https://pub-85d4dcece16844bf8290aa4b33608ccd.r2.dev/ShareX/2025/09/ShareX_ZWFVlZZu0W.png) +3. Click `ClipboardContentFormat` under `After upload`. +4. Replace the contents with the Markdown format `![$filename]($result)`. + +### 3.2 Enable auto-copy to clipboard + +1. From the main window, under `After upload tasks`, tick `Copy URL to clipboard` on the right. + ![ShareX_zf6qftjnu6.png](https://pub-85d4dcece16844bf8290aa4b33608ccd.r2.dev/ShareX/2025/09/ShareX_zf6qftjnu6.png) + +### 3.3 Change the hotkey + +You can change the hotkey under `Hotkey settings` on the main window, giving you a one-shot flow: screenshot → upload → Markdown on your clipboard. + +--- + +## F.A.Q. and Troubleshooting + +**Q1: I get a `(403) Forbidden` error when uploading — what now?** +**A1:** This is the most common issue. Check the following two S3 Advanced settings: + +1. Make sure **`Set public-read ACL on file`** is **unchecked**. +2. Make sure **`Use path style request`** is **checked**. +3. If it still fails, regenerate an **API token with `Object Read & Write` permissions** and update it in ShareX. + +**Q2: I don't want to upload screenshots — I want to upload existing local images.** +**A2:** +**Use the Upload tab on the left**: pick the type of content you want to upload from inside that tab. +![ShareX_34TAgHco1T.png](https://pub-85d4dcece16844bf8290aa4b33608ccd.r2.dev/ShareX/2025/09/ShareX_34TAgHco1T.png) + +**Q3: Why can't I use PicGo?** +**A3:** In my testing, PicGo's S3 plugin has compatibility issues with Cloudflare R2 — filenames and paths aren't handled correctly. ShareX's S3 implementation is more standard and currently the more reliable choice. diff --git a/app/docs/ai/compute-platforms/compute-platforms-handbook.en.mdx b/app/docs/ai/compute-platforms/compute-platforms-handbook.en.mdx new file mode 100644 index 00000000..281d5777 --- /dev/null +++ b/app/docs/ai/compute-platforms/compute-platforms-handbook.en.mdx @@ -0,0 +1,219 @@ +--- +title: Compute Platforms +description: A guide to AI development on cloud compute platforms such as AutoDL and InternStudio +date: "2025-01-27" +tags: + - compute-platforms + - autodl + - internstudio + - gpu-cloud + - development-environment +docId: d73h3kyjnzytk1y2nizulyr6 +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T00:00:00Z +translatorAgent: claude-opus-4-6 +--- + +Training and inference on AI models requires substantial compute. This section walks through the mainstream compute platforms and cloud services to help developers pick the right resources. + +## AutoDL + +### Overview + +- **Website**: [https://www.autodl.com/home](https://www.autodl.com/home) +- **Positioning**: A dedicated GPU cloud service +- **Strengths**: Affordable pricing and a simple workflow — well suited for individual developers and small teams + +### Documentation + +- **Full docs**: [AutoDL official docs](https://www.autodl.com/docs/) +- **Coverage**: + - Instance creation and management + - Environment setup + - Data upload and download + - Billing and cost + +### Step-by-Step: Connecting PyCharm Professional to AutoDL + +**Configuration steps**: + +1. **Create an AutoDL instance**: Pick a suitable GPU configuration +2. **Collect connection info**: Note the IP address, port, and username +3. **Configure PyCharm**: Set up the remote interpreter +4. **File sync**: Configure automatic upload and download +5. **Debug and run**: Remote debugging and code execution + +**Network configuration**: + +- SSH connection settings +- Port forwarding +- File transfer tuning +- Ensuring a stable connection + +**Development workflow**: + +- Write code locally +- Sync code to the remote machine +- Schedule GPU resources +- Download result files + +## InternStudio + +### Platform Introduction + +- **Website**: [https://studio.intern-ai.org.cn/](https://studio.intern-ai.org.cn/user/account) +- **Highlights**: A free compute platform provided by the Shanghai AI Laboratory +- **Use cases**: Learning, research, and small-scale project development + +### Connection and Usage + +**SSH connection setup**: + +- [SSH connection and port forwarding tutorial](https://juejin.cn/post/7446234916049829939) +- Supports remote development workflows +- Provides a JupyterLab interface + +**Camp 4 training resources**: + +- **GitHub**: [Tutorial (Camp 4)](https://github.com/InternLM/Tutorial/tree/camp4) +- **Linux basics**: [InternStudio basic commands](https://aicarrier.feishu.cn/wiki/XZChwwDsciyFyHk5mGTc1EKinkc) + +### Open-Source Community Project Applications + +**Compute grants**: + +- 🔥 [Intern LLM open-source community project application](https://openxlab.org.cn/apps) 🔥 +- Available for open-source projects and academic research +- Offers long-term, stable compute support + +## Platform Comparison + +### When to Choose AutoDL + +**Strengths**: + +- Hourly billing keeps costs predictable +- Rich set of preinstalled environments +- Good Chinese-language support +- Stable network connectivity + +**Ideal users**: + +- Individual developers +- Beginners and students +- Short-term project needs +- Budget-constrained teams + +### When to Choose InternStudio + +**Strengths**: + +- Free usage quota +- Academic-friendly +- Integrated with the InternLM ecosystem +- Rich educational resources + +**Ideal users**: + +- Students and researchers +- InternLM model users +- Teaching and training +- Open-source project development + +### Other Cloud Options + +#### International Platforms + +- **Google Colab**: Free GPU — good for learning and lightweight work +- **AWS EC2**: Enterprise-grade service with broad features but higher cost +- **Microsoft Azure**: Integrates well with the Windows ecosystem +- **Lambda Labs**: Specialized GPU cloud provider + +#### China-Based Platforms + +- **Alibaba Cloud**: Enterprise-grade with a mature ecosystem +- **Tencent Cloud**: Tuned for gaming and social workloads +- **Baidu Cloud**: AI platform built around the PaddlePaddle ecosystem +- **Huawei Cloud**: Support for Ascend AI processors + +## Tips and Best Practices + +### Cost Optimization + +1. **On-demand usage**: Shut down idle instances promptly +2. **Preinstalled images**: Pick an image that matches your stack +3. **Data management**: Plan storage usage up front +4. **Alerts**: Set budget and resource-usage alerts + +### Developer Productivity + +1. **Environment management**: Use Docker or conda +2. **Code sync**: Set up Git or a file-sync tool +3. **Debugging**: Master remote debugging workflows +4. **Resource monitoring**: Watch GPU and memory usage in real time + +### Data Security + +1. **Regular backups**: Back up critical data across multiple locations +2. **Version control**: Manage code with Git +3. **Access control**: Use strong SSH keys +4. **Compliance**: Follow relevant data-handling regulations + +## Environment Setup Guide + +### Deep Learning Environment + +**Core components**: + +- CUDA/cuDNN +- Python 3.8+ +- PyTorch/TensorFlow +- Jupyter Notebook + +**Common libraries**: + +```bash +# PyTorch ecosystem +pip install torch torchvision transformers datasets + +# Scientific computing +pip install numpy pandas matplotlib seaborn + +# Machine learning +pip install scikit-learn xgboost lightgbm + +# Deep learning utilities +pip install wandb tensorboard +``` + +### Development Tool Setup + +- **IDEs**: PyCharm Professional, VS Code +- **Debugging**: pdb, ipdb +- **Profiling**: nvidia-smi, htop +- **Version control**: Git, DVC + +## Troubleshooting + +### Common Issues + +1. **Connection timeouts**: Check network and firewall settings +2. **GPU unavailable**: Verify CUDA installation and driver version +3. **Out of memory**: Reduce batch size or shrink model parameters +4. **Out of disk space**: Clean up temporary files and logs + +### Performance Tuning + +1. **GPU utilization**: Monitor and optimize GPU usage +2. **I/O optimization**: Speed up data loading and preprocessing +3. **Memory management**: Tune caching and batch size appropriately +4. **Parallelism**: Leverage multi-GPU and distributed training + +## Learning Suggestions + +1. **Know one platform deeply**: Develop expertise on at least one major platform +2. **Stay cost-aware**: Learn to plan and control compute spend +3. **Environment management**: Master configuration and dependency management +4. **Monitor and tune**: Track resource usage and optimize performance +5. **Security practices**: Take data security and access control seriously diff --git a/app/docs/ai/reinforcement-learning/reinforcement-learning-overview.en.mdx b/app/docs/ai/reinforcement-learning/reinforcement-learning-overview.en.mdx new file mode 100644 index 00000000..a188f695 --- /dev/null +++ b/app/docs/ai/reinforcement-learning/reinforcement-learning-overview.en.mdx @@ -0,0 +1,154 @@ +--- +title: Reinforcement Learning +description: Fundamentals of reinforcement learning, Chain-of-Thought (CoT), GRPO, and their applications in large language models +date: "2025-01-27" +tags: + - reinforcement-learning + - rlhf + - cot + - grpo + - ppo +docId: s4fuhmdf6hj49jx1l7k87d4p +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T00:00:00Z +translatorAgent: claude-opus-4-6 +--- + +Reinforcement learning plays a vital role in the era of large language models, particularly in reasoning and alignment. From RLHF to Chain-of-Thought reasoning, RL provides key technical foundations for enhancing LLM capabilities. + +## Recommended Learning Resources + +### Prof. Shiyu Zhao's RL Course (Westlake University) + +**Highlights**: The mathematical foundations of reinforcement learning, from scratch to deep understanding + +**Resources**: + +- **Book & Slides**: [GitHub repository](https://github.com/MathFoundationRL/Book-Mathmatical-Foundation-of-Reinforcement-Learning) +- **Video lectures**: [Full course on Bilibili](https://www.bilibili.com/video/BV1sd4y167NS/) +- **Why this course**: Rigorous math derivations with a solid theoretical foundation + +### RethinkFun RL Series + +**Recommended creator**: [@RethinkFun](https://space.bilibili.com/18235884/) + +**Core videos**: + +- [RL fundamentals explained](https://www.bilibili.com/video/BV1rooaYVEk8/) — plain-language explanations with illustrated principles and formula derivations +- [PPO and GRPO algorithm walkthroughs](https://www.bilibili.com/video/BV15cZYYvEhz/) — illustrated algorithm internals + +### Beginner Tutorials + +- **Minimal Introduction to Reinforcement Learning** — an intuitive walkthrough of MDP, DP/MC/TD, Q-learning, policy gradients, and PPO + +## RL Papers and Projects (Pending Review) + +> 📊 **Paper shortlist**: Click to view the full list of RL papers pending review +> +> Status categories: Not Started, Evaluating, Completed, Not Recommended, Not Open-Sourced + +## GRPO Reproduction References + +### TRL Framework + +- **Project**: [TRL (Transformer Reinforcement Learning)](https://github.com/huggingface/trl) +- **Highlights**: HuggingFace's official RL framework +- **Supported algorithms**: GRPO, PPO, DPO, and more + +## Chain-of-Thought (CoT) + +### Core Concept + +Chain-of-Thought reasoning is a key technique for exposing an LLM's reasoning process, improving both interpretability and reasoning capability. + +### Notable Papers and Projects + +#### CoT-Valve: Length-Compressible Chain-of-Thought Tuning + +- **Highlights**: A technique for tuning Chain-of-Thought reasoning with compressible length +- **Source**: [HuggingFace Daily Papers](https://huggingface.co/papers/date/2025-07-25) + +#### MCoT (Multi-Chain-of-Thought) + +- **Project**: [Awesome-MCoT](https://github.com/yaotingwangofficial/Awesome-MCoT) +- **Highlights**: Multi-chain reasoning that improves performance on complex reasoning tasks + +#### Latent CoT + +- **Project**: [Awesome-Latent-CoT](https://github.com/awesome-latent-cot) +- **Core idea**: Move reasoning from linguistic symbols into the latent space to capture richer and more complex thought processes + +### Multimodal CoT + +Chain-of-Thought reasoning that combines visual and textual information — showing strong capability on multimodal tasks. + +### Survey on Latent-Space Reasoning + +- **Key survey**: [HIT's first survey on latent-space reasoning](https://zhuanlan.zhihu.com/p/1930639357087298531) +- **Core argument**: Reshapes the boundaries of LLM reasoning by exploring reasoning mechanisms in the latent space + +## DeepSeek-R1 Deep Dive + +DeepSeek-R1, as a model with standout reasoning capabilities, has technical details worth in-depth study: + +- Reasoning mechanism design +- Training strategy analysis +- Performance evaluation methodology + +## Suggested Learning Path + +### Foundations + +1. **Math prerequisites**: probability, dynamic programming, optimization theory +2. **Core concepts**: MDP, value functions, policies, returns +3. **Classical algorithms**: Q-learning, policy gradients, Actor-Critic + +### Intermediate + +1. **Modern algorithms**: PPO, TRPO, SAC, TD3 +2. **LLM applications**: RLHF, Constitutional AI +3. **Chain-of-Thought techniques**: CoT, MCoT, Latent CoT + +### Practice + +1. **Frameworks**: OpenAI Gym, Stable Baselines3, TRL +2. **Hands-on projects**: game AI, dialogue system optimization +3. **Paper reproduction**: reproducing and improving upon key algorithms + +## Application Areas + +### LLM Alignment + +- **RLHF**: Learning from human feedback to improve output quality +- **Constitutional AI**: Principle-based approach to AI alignment +- **DPO**: Direct Preference Optimization — a simplified alternative to RLHF + +### Reasoning Capability + +- **Chain-of-Thought reasoning**: Improves performance on complex reasoning tasks +- **Tool use**: Training models to invoke external tools +- **Code generation**: Improves programming capability + +### Multi-Agent Systems + +- **Cooperative learning**: Multiple agents solve problems together +- **Competitive learning**: Individual capabilities improve through competition +- **Social learning**: Agents learn from the behavior of other agents + +## Frontier Trends + +1. **Offline RL**: Learning policies from static datasets +2. **Meta-learning**: Algorithms that adapt quickly to new tasks +3. **Safe RL**: Ensuring the safety of both the learning process and the learned policy +4. **Explainable RL**: Improving the interpretability of decision-making + +## Additional Notes + +- Chain-of-Thought / Multi-step CoT (MCoT) / Latent CoT + +- GRPO learning resources: + - Bilibili playlist: https://space.bilibili.com/18235884/search?keyword=GRPO + - PPO/GRPO algorithm explainer: https://www.bilibili.com/video/BV15cZYYvEhz/ + - Paper and resource collection: https://github.com/yaotingwangofficial/Awesome-MCoT +- RL math foundations textbook: GitHub https://github.com/MathFoundationRL/Book-Mathmatical-Foundation-of-Reinforcement-Learning diff --git a/app/docs/computer-science/data-structures/linked-list/01-singly-linked-list.zh.mdx b/app/docs/computer-science/data-structures/linked-list/01-singly-linked-list.zh.mdx new file mode 100644 index 00000000..b50e08b7 --- /dev/null +++ b/app/docs/computer-science/data-structures/linked-list/01-singly-linked-list.zh.mdx @@ -0,0 +1,570 @@ +--- +title: 单链表 +description: "单链表的实现、操作与应用" +date: "2024-01-07" +tags: + - singly-linked-list + - pointer-operations + - basic-data-structure +docId: gkjk6stzpb44n9lv8u2ij7xx +lang: zh +translatedFrom: en +translatedAt: 2026-04-15T00:00:00Z +translatorAgent: claude-opus-4-6 +--- + +# 单链表 + +单链表是链表最基本的形式，每个节点包含数据和指向下一个节点的指针。它支持动态内存分配，并能高效地完成插入和删除操作。 + +## 节点结构 + +```javascript +class ListNode { + constructor(data) { + this.data = data; // 数据域 + this.next = null; // 指针域，指向下一个节点 + } +} +``` + +## 完整实现 + +```javascript +class SinglyLinkedList { + constructor() { + this.head = null; // 头指针 + this.size = 0; // 链表长度 + } + + // 头部插入节点 + prepend(data) { + const newNode = new ListNode(data); + newNode.next = this.head; + this.head = newNode; + this.size++; + } + + // 尾部插入节点 + append(data) { + const newNode = new ListNode(data); + + // 若链表为空，新节点成为头节点 + if (!this.head) { + this.head = newNode; + this.size++; + return; + } + + // 找到最后一个节点 + let current = this.head; + while (current.next) { + current = current.next; + } + + // 链接新节点 + current.next = newNode; + this.size++; + } + + // 在指定位置插入节点 + insert(index, data) { + if (index < 0 || index > this.size) { + throw new Error("Index out of bounds"); + } + + // 头部插入 + if (index === 0) { + this.prepend(data); + return; + } + + const newNode = new ListNode(data); + let current = this.head; + + // 找到插入位置的前一个节点 + for (let i = 0; i < index - 1; i++) { + current = current.next; + } + + // 插入新节点 + newNode.next = current.next; + current.next = newNode; + this.size++; + } + + // 删除头节点 + removeFirst() { + if (!this.head) { + return null; + } + + const removedData = this.head.data; + this.head = this.head.next; + this.size--; + return removedData; + } + + // 删除尾节点 + removeLast() { + if (!this.head) { + return null; + } + + // 只有一个节点 + if (!this.head.next) { + const removedData = this.head.data; + this.head = null; + this.size--; + return removedData; + } + + // 找到倒数第二个节点 + let current = this.head; + while (current.next.next) { + current = current.next; + } + + const removedData = current.next.data; + current.next = null; + this.size--; + return removedData; + } + + // 删除指定位置的节点 + remove(index) { + if (index < 0 || index >= this.size) { + throw new Error("Index out of bounds"); + } + + // 删除头节点 + if (index === 0) { + return this.removeFirst(); + } + + let current = this.head; + + // 找到被删节点的前一个节点 + for (let i = 0; i < index - 1; i++) { + current = current.next; + } + + const removedData = current.next.data; + current.next = current.next.next; + this.size--; + return removedData; + } + + // 删除指定值的第一个节点 + removeByValue(data) { + if (!this.head) { + return false; + } + + // 如果头节点就是要删除的节点 + if (this.head.data === data) { + this.head = this.head.next; + this.size--; + return true; + } + + let current = this.head; + while (current.next && current.next.data !== data) { + current = current.next; + } + + // 找到了要删除的节点 + if (current.next) { + current.next = current.next.next; + this.size--; + return true; + } + + return false; // 未找到 + } + + // 查找元素 + indexOf(data) { + let current = this.head; + let index = 0; + + while (current) { + if (current.data === data) { + return index; + } + current = current.next; + index++; + } + + return -1; // 未找到 + } + + // 获取指定位置的元素 + get(index) { + if (index < 0 || index >= this.size) { + throw new Error("Index out of bounds"); + } + + let current = this.head; + for (let i = 0; i < index; i++) { + current = current.next; + } + + return current.data; + } + + // 检查链表是否包含指定元素 + contains(data) { + return this.indexOf(data) !== -1; + } + + // 获取链表长度 + length() { + return this.size; + } + + // 检查链表是否为空 + isEmpty() { + return this.size === 0; + } + + // 清空链表 + clear() { + this.head = null; + this.size = 0; + } + + // 转换为数组 + toArray() { + const result = []; + let current = this.head; + + while (current) { + result.push(current.data); + current = current.next; + } + + return result; + } + + // 遍历链表 + forEach(callback) { + let current = this.head; + let index = 0; + + while (current) { + callback(current.data, index); + current = current.next; + index++; + } + } + + // 反转链表 + reverse() { + let prev = null; + let current = this.head; + let next = null; + + while (current) { + next = current.next; // 保存下一个节点 + current.next = prev; // 反转指针 + prev = current; // 移动 prev + current = next; // 移动 current + } + + this.head = prev; + } + + // 打印链表 + toString() { + if (!this.head) { + return "Empty list"; + } + + const elements = []; + let current = this.head; + + while (current) { + elements.push(current.data); + current = current.next; + } + + return elements.join(" -> "); + } +} +``` + +## 使用示例 + +```javascript +// 创建链表 +const list = new SinglyLinkedList(); + +// 添加元素 +list.append(1); +list.append(2); +list.append(3); +list.prepend(0); +console.log(list.toString()); // "0 -> 1 -> 2 -> 3" + +// 插入元素 +list.insert(2, 1.5); +console.log(list.toString()); // "0 -> 1 -> 1.5 -> 2 -> 3" + +// 查找元素 +console.log(list.indexOf(2)); // 3 +console.log(list.get(1)); // 1 +console.log(list.contains(3)); // true + +// 删除元素 +list.remove(0); // 删除索引 0 的元素 +list.removeByValue(1.5); // 删除值为 1.5 的元素 +console.log(list.toString()); // "1 -> 2 -> 3" + +// 反转链表 +list.reverse(); +console.log(list.toString()); // "3 -> 2 -> 1" + +// 遍历链表 +list.forEach((data, index) => { + console.log(`Index ${index}: ${data}`); +}); +``` + +## 关键算法细节 + +### 1. 插入操作中的指针变化 + +``` +插入前：A -> B -> C +在 A 与 B 之间插入 X： + +步骤 1：newNode.next = A.next + A -> B -> C + X ----↗ + +步骤 2：A.next = newNode + A -> X -> B -> C +``` + +### 2. 删除操作中的指针变化 + +``` +删除前：A -> B -> C -> D +删除 B： + +步骤 1：找到 B 的前驱节点 A +步骤 2：A.next = B.next + A -----> C -> D + （B 被跳过，等待垃圾回收） +``` + +### 3. 反转算法细节 + +```javascript +// 反转过程示意 +// 初始：1 -> 2 -> 3 -> null +// 目标：null <- 1 <- 2 <- 3 + +function reverse(head) { + let prev = null; + let current = head; + + while (current !== null) { + let next = current.next; // 保存下一个节点 + current.next = prev; // 反转当前节点的指针 + prev = current; // prev 前移 + current = next; // current 前移 + } + + return prev; // 此时 prev 指向新的头节点 +} +``` + +## 常见面试题 + +### 1. 检测链表中的环 + +```javascript +function hasCycle(head) { + if (!head || !head.next) return false; + + let slow = head; + let fast = head; + + while (fast && fast.next) { + slow = slow.next; + fast = fast.next.next; + + if (slow === fast) { + return true; // 发现环 + } + } + + return false; +} +``` + +### 2. 找到链表的中点 + +```javascript +function findMiddle(head) { + if (!head) return null; + + let slow = head; + let fast = head; + + while (fast.next && fast.next.next) { + slow = slow.next; + fast = fast.next.next; + } + + return slow; +} +``` + +### 3. 合并两个有序链表 + +```javascript +function mergeTwoLists(l1, l2) { + const dummy = new ListNode(0); + let current = dummy; + + while (l1 && l2) { + if (l1.data <= l2.data) { + current.next = l1; + l1 = l1.next; + } else { + current.next = l2; + l2 = l2.next; + } + current = current.next; + } + + // 连接剩余的节点 + current.next = l1 || l2; + + return dummy.next; +} +``` + +## 性能优化技巧 + +### 1. 尾指针优化 + +```javascript +class OptimizedSinglyLinkedList { + constructor() { + this.head = null; + this.tail = null; // 维护尾指针 + this.size = 0; + } + + append(data) { + const newNode = new ListNode(data); + + if (!this.head) { + this.head = this.tail = newNode; + } else { + this.tail.next = newNode; + this.tail = newNode; + } + + this.size++; + } + + // 现在 append 操作是 O(1) 而不是 O(n) +} +``` + +### 2. 哨兵节点优化 + +```javascript +class SentinelLinkedList { + constructor() { + this.sentinel = new ListNode(null); // 哨兵节点 + this.sentinel.next = null; + this.size = 0; + } + + // 有了哨兵节点后，许多操作的边界处理会更简单 + prepend(data) { + const newNode = new ListNode(data); + newNode.next = this.sentinel.next; + this.sentinel.next = newNode; + this.size++; + } +} +``` + +## 实际应用场景 + +### 1. 实现栈 + +```javascript +class Stack { + constructor() { + this.list = new SinglyLinkedList(); + } + + push(item) { + this.list.prepend(item); + } + + pop() { + return this.list.removeFirst(); + } + + peek() { + return this.list.isEmpty() ? null : this.list.get(0); + } + + isEmpty() { + return this.list.isEmpty(); + } +} +``` + +### 2. 实现队列 + +```javascript +class Queue { + constructor() { + this.head = null; + this.tail = null; + } + + enqueue(item) { + const newNode = new ListNode(item); + if (!this.tail) { + this.head = this.tail = newNode; + } else { + this.tail.next = newNode; + this.tail = newNode; + } + } + + dequeue() { + if (!this.head) return null; + + const item = this.head.data; + this.head = this.head.next; + if (!this.head) this.tail = null; + + return item; + } +} +``` + +## 注意事项 + +1. **空指针检查**：总是先判断节点是否为空 +2. **边界情况**：特别留意空链表和单节点的情况 +3. **内存管理**：在需要手动管理内存的语言中，记得释放已删除的节点 +4. **指针操作顺序**：插入和删除时注意指针修改的先后顺序 + +## 小结 + +单链表是理解链表这一数据结构的基础。虽然它在随机访问上不如数组高效，但在动态的插入与删除操作中表现出色。掌握单链表的实现与操作，是学习更复杂数据结构的必经之路。 + +下一节我们会学习双向链表，看看它如何通过增加反向指针带来更大的灵活性。 diff --git a/app/docs/jobs/interview-prep/bq.en.md b/app/docs/jobs/interview-prep/bq.en.md new file mode 100644 index 00000000..625ab3d1 --- /dev/null +++ b/app/docs/jobs/interview-prep/bq.en.md @@ -0,0 +1,70 @@ +--- +title: Behavioral Interview +description: First page +date: "2025-09-11" +tags: + - intro +docId: u68pjetu592c9zvs3f5xa82j +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T00:00:00Z +translatorAgent: claude-opus-4-6 +--- + +# Ten Universal Angles for Behavioral Interviews + +After sitting through a stretch of interviews, I realized "behavioral" questions — the BQ ones — may look endlessly varied, but they really boil down to ten directions. Prepare one solid story for each and you'll be able to handle almost any variant. + +## 1. Collaboration & Communication + +This category shows up constantly. Common questions: + +**Teamwork** +Don't just say "I'm a team player." Show how you work with colleagues and managers from different backgrounds to hit a shared goal. + +**Conflict** +Conflict questions are unavoidable. Don't shy away from describing a disagreement — the point is how you kept calm, communicated, and found a way forward. + +**Client service** +Often overlooked. Plenty of JDs are really probing how you understand client needs, solve their problems, and sometimes deliver above expectations. + +## 2. Leadership & Organization + +Not just for management roles. Common questions: + +**Leadership** +The focus isn't "how I directed people." It's how you took ownership and drove momentum inside a team or project. + +**Planning** +The interviewer wants to hear how you break work down, stage it, and handle surprises along the way. + +**Multitasking** +How do you juggle parallel work streams? Be clear about how you prioritize and still deliver on time. + +## 3. Learning & Growth + +Roles evolve fast, so learning agility is always on the rubric. Common questions: + +**Learn new skills** +Use a specific example to show how you picked up a new skill or approach on your own and applied it at work right away. + +**Motivation** +The classic "three whys": why the industry, why the company, why the role. Answer by tying the company's positioning to your personal drivers — that's what lands. + +## 4. Challenges & Response + +Shows up in almost every interview. Common questions: + +**Challenge** +A major challenge you faced at work — don't drift into life anecdotes. Focus on how you decomposed the problem and pushed through. + +**Failure** +A failure story is a plus, not a minus. Be honest about what went wrong, and — more importantly — how you adjusted and grew afterward. + +## Summary + +Teamwork, conflict, client; leadership, planning, multitasking; learning, motivation; challenge, failure. +Prep these ten and any reworded question becomes just another variation. + +Remember: use the **STAR framework** (Situation, Task, Action, Result) +to package your answer as a short story you can pull out on demand. From db1a25cedf82c242bcf0463020c4641ab4e34070 Mon Sep 17 00:00:00 2001 From: longsizhuo Date: Wed, 15 Apr 2026 17:43:53 +0000 Subject: [PATCH 09/19] =?UTF-8?q?feat(docs):=20i18n=20CommunityShare=20?= =?UTF-8?q?=E5=85=A8=E9=83=A8=E7=BF=BB=E8=AF=91=E5=AE=8C=E6=88=90=20(13=20?= =?UTF-8?q?=E7=AF=87)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit translator-community 产出。双向翻译： - Geek: git101 / picturecdn / swanlab / raspberry-guide / CommonUsedMarkdown / Katex×2 (全部 zh→en)，leworldmodel (en→zh) - RAG: context_engineering_intro / embedding / rag (全部 zh→en) - Amazing-AI-Tools: perplexity-comet (zh→en)， prompt-repetition-improves-non-reasoning-llms (en→zh) 跳过：3 个 索引页、1 个 MVP 已翻译文件。 所有翻译版 frontmatter 继承原文 docId，带 translatedFrom 标记 供 contributors 脚本跳过统计。 --- .../Amazing-AI-Tools/perplexity-comet.en.md | 104 ++++++ ...petition-improves-non-reasoning-llms.zh.md | 20 ++ .../Geek/CommonUsedMarkdown.en.md | 81 +++++ .../CommunityShare/Geek/Katex/Seb1.en.mdx | 84 +++++ .../CommunityShare/Geek/Katex/Seb2.en.mdx | 113 +++++++ app/docs/CommunityShare/Geek/git101.en.mdx | 49 +++ .../CommunityShare/Geek/leworldmodel.zh.md | 18 + .../CommunityShare/Geek/picturecdn.en.mdx | 164 +++++++++ .../CommunityShare/Geek/raspberry-guide.en.md | 313 ++++++++++++++++++ app/docs/CommunityShare/Geek/swanlab.en.mdx | 99 ++++++ .../RAG/context_engineering_intro.en.md | 72 ++++ app/docs/CommunityShare/RAG/embedding.en.mdx | 67 ++++ app/docs/CommunityShare/RAG/rag.en.mdx | 107 ++++++ 13 files changed, 1291 insertions(+) create mode 100644 app/docs/CommunityShare/Amazing-AI-Tools/perplexity-comet.en.md create mode 100644 app/docs/CommunityShare/Amazing-AI-Tools/prompt-repetition-improves-non-reasoning-llms.zh.md create mode 100644 app/docs/CommunityShare/Geek/CommonUsedMarkdown.en.md create mode 100644 app/docs/CommunityShare/Geek/Katex/Seb1.en.mdx create mode 100644 app/docs/CommunityShare/Geek/Katex/Seb2.en.mdx create mode 100644 app/docs/CommunityShare/Geek/git101.en.mdx create mode 100644 app/docs/CommunityShare/Geek/leworldmodel.zh.md create mode 100644 app/docs/CommunityShare/Geek/picturecdn.en.mdx create mode 100644 app/docs/CommunityShare/Geek/raspberry-guide.en.md create mode 100644 app/docs/CommunityShare/Geek/swanlab.en.mdx create mode 100644 app/docs/CommunityShare/RAG/context_engineering_intro.en.md create mode 100644 app/docs/CommunityShare/RAG/embedding.en.mdx create mode 100644 app/docs/CommunityShare/RAG/rag.en.mdx diff --git a/app/docs/CommunityShare/Amazing-AI-Tools/perplexity-comet.en.md b/app/docs/CommunityShare/Amazing-AI-Tools/perplexity-comet.en.md new file mode 100644 index 00000000..4f104631 --- /dev/null +++ b/app/docs/CommunityShare/Amazing-AI-Tools/perplexity-comet.en.md @@ -0,0 +1,104 @@ +--- +title: "Perplexity Comet: The AI Browser That Acts Like a Personal Assistant" +description: "" +date: "2025-10-03" +tags: + - perlexity + - comet + - productivity +docId: eej2awin6irhbdgcy8vvs3xb +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +## Key Info + +**Free for students — claim here: [Get Comet with free Perplexity Pro. The AI browser built for students.](https://www.perplexity.ai/students)** +Follow the link to get a free year of Perplexity Pro membership. +Then you can download the Comet browser here: +[Comet Browser: a Personal AI Assistant](https://www.perplexity.ai/comet/) + +## What Is Comet? + +This is the first post in the series, and I'm starting with Perplexity Comet — an AI browser. +I picked it first because it's the tool I use most, even more than Cursor. It's a browser, after all — something you have open every single day. + +There are quite a few AI browsers on the market right now: Dia, Microsoft's Edge has leaned into AI, and Chrome is ramping up too. +But from what I've heard, Comet pulls ahead in a few specific ways — probably thanks to Perplexity's early head start. + +![ab3becf9-fe7f-40f3-92f3-8e861e68f1fc.png](https://img.coly.cc/obs-img/2025/10/1df9eb3648e7893e32c0139de7ad5a6d.png) + +What makes it stand out comes down to two things: + +1. It can operate the browser on my behalf — handling tasks I don't want to do myself. The speed and accuracy aren't always perfect, but the point is you can hand it something and go do something else. +2. It's replaced my to-do app. It acts like a personal scheduler: consolidating my emails, surfacing upcoming tasks, and sending me reminders. + +### Browser Automation + +Comet's browser automation works in two modes: + +1. Background mode — visits pages and executes tasks without you watching +2. Foreground mode — automates actions directly on your current page + +#### Background Mode: Use Cases + +Checking your email, browsing Moodle to look up grades, and similar tasks. + +If you're already logged into Moodle, you can type a request directly into Comet's input box: +![image.png](https://img.coly.cc/obs-img/2025/10/2979472a93976749b4f45b5960906c57.png) +![image.png](https://img.coly.cc/obs-img/2025/10/bff8830b849d3d64ad5ee40f4eb589d8.png) +You'll then see it navigating pages and executing steps in the background: +![image.png](https://img.coly.cc/obs-img/2025/10/5e31b513d633a1383ef637293e320c3e.png) +Until it comes back with a result. + +Similarly, you can ask it to check your inbox and delete spam. I won't go through every example — once you get the idea, you'll know what kinds of tasks this background AI can handle. + +One thing to note: Comet defaults to using Perplexity's own model, which is pretty mediocre. Remember to manually switch it to whichever model is currently the best: +![image.png](https://img.coly.cc/obs-img/2025/10/aeba741491cad07e3c105db3f49cdf1a.png) + +#### Foreground Mode: Use Cases + +Automating Coles — adding every item on your grocery list to the cart. +![8c574a67ac53577520eb8758b7e892a5.png](https://img.coly.cc/obs-img/2025/10/0759eb80fe077ad310c0f235bf8843c3.png) +The way to use it: open the Coles page while logged in, click the Assistant button in the top right, and give it an instruction like "add [item] to my cart." +You'll see a blue overlay appear on the page as it gets to work. +Even in foreground mode, you can switch to another tab and carry on — it keeps going in the background. +I'm honestly not entirely sure what the difference is between the two modes, but it seems like some pages are harder to operate in background mode and more reliable in foreground mode. + +### Your Personal Planner: Email + Calendar Integration + +The first thing I do every morning is type my custom command into the input box: +![image.png](https://img.coly.cc/obs-img/2025/10/98605f02a120be2cb67a090c85a40b2b.png) + +Remind me when to bring laundry in (Sydney's rainy season has soaked me enough times), +tell me when the exchange rate is favorable, +surface anything coming up that I need to deal with, +and summarize my emails. + +First, go to Settings > Connectors and add your accounts. I mainly use Google services — if you use others, those can be connected too. +![image.png](https://img.coly.cc/obs-img/2025/10/ed069ee778622782832ed06b947c24d6.png) + +Then add a shortcut in settings so you don't have to type a long prompt every day — just a short command and it runs: +![image.png](https://img.coly.cc/obs-img/2025/10/feb661e4aa4c71b5cf2c68fd5284f294.png) + +Here's what it looks like in action: +![image.png](https://img.coly.cc/obs-img/2025/10/26eb7f9cb633787058b0efe5ecfa86e6.png) +![image.png](https://img.coly.cc/obs-img/2025/10/751ab79cce1abfd9c194d09666cb6061.png) +Pretty handy. + +My favorite feature is the calendar integration. Since I started using it, I haven't opened TickTick or Google Calendar once. Course schedules get added to calendar events automatically, and Comet reminds me. If I have something I want to remember, I just tell it — it logs the event in my calendar, and when my daily command runs, it surfaces any upcoming events and sends reminders via browser notification and email. Hard to forget things. + +## About Perplexity + +Beyond Comet, Perplexity is primarily an AI search engine company. Their core product is decent, but doesn't stand out dramatically compared to GPT or similar tools. What it does well is let you use models from multiple providers freely — so it's a solid AI chat option overall. +![image.png](https://img.coly.cc/obs-img/2025/10/544381e5bde62b83b952ce3ad96b3820.png) + +## Coming Up Next + +That's Perplexity Comet covered — the tool I use most. Next up is Cursor. The topic will probably be what a lot of people actually want to know: how a complete beginner can use it to ship projects quickly and build up their portfolio. + +Cursor is genuinely impressive — over the past year I've used it to knock out a bunch of solid projects in a fraction of the time it would've taken otherwise. +The downside is that heavy reliance on it weakens your raw coding skills over time. +So I'm actually on the fence about whether to write that post. diff --git a/app/docs/CommunityShare/Amazing-AI-Tools/prompt-repetition-improves-non-reasoning-llms.zh.md b/app/docs/CommunityShare/Amazing-AI-Tools/prompt-repetition-improves-non-reasoning-llms.zh.md new file mode 100644 index 00000000..9030b5c3 --- /dev/null +++ b/app/docs/CommunityShare/Amazing-AI-Tools/prompt-repetition-improves-non-reasoning-llms.zh.md @@ -0,0 +1,20 @@ +--- +title: Prompt Repetition Improves Non-Reasoning LLMs +description: 复读机或可提高大模型能力 +date: "2026-03-05" +tags: + - AI + - LLMs + - arXiv +docId: l6eepr5ctjgrhdgupy3twr1t +lang: zh +translatedFrom: en +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +<https://arxiv.org/pdf/2512.14982> + +在不使用推理模式的情况下，重复输入提示词能够提升主流模型（Gemini、GPT、Claude、DeepSeek）的表现，且不会增加生成 token 数量或推理延迟。 + +1. 提示词重复：LLM 通常以因果语言模型的方式训练，即过去的 token 无法关注到未来的 token。因此，用户查询中 token 的排列顺序会影响预测性能。例如，"选项在前、问题在后"的查询形式与"问题在前、选项在后"的形式往往表现不同（见图 1）。我们提出重复提示词的方法：将输入从单次提示词转换为重复两次的形式。这使得每个提示词 token 都能关注到其他所有提示词 token，从而解决上述问题。在不使用推理模式时，提示词重复能够提升 LLM 的性能（图 1），且不会增加生成输出的长度或推理延迟。 diff --git a/app/docs/CommunityShare/Geek/CommonUsedMarkdown.en.md b/app/docs/CommunityShare/Geek/CommonUsedMarkdown.en.md new file mode 100644 index 00000000..8f9f10cf --- /dev/null +++ b/app/docs/CommunityShare/Geek/CommonUsedMarkdown.en.md @@ -0,0 +1,81 @@ +--- +title: Common Markdown Syntax +date: 2025-09-20T14:25:39.000Z +docId: xqz5iiv3p52h6d9g3c0w2baf +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +## Headings & Font Styles + +`# Heading 1` + +`## Heading 2` + +And so on. + +**Bold** `**Bold**` + +_Italic_ `*Italic*` + +**_Bold Italic_** `***Bold Italic***` + +~~Strikethrough~~ `~~Strikethrough~~` + +Highlight `Highlight` + +Half width space + +`Half width space` + +Full width space + +`Full width space` + +## Unordered & Ordered Lists + +- Unordered item 1 `* Unordered item 1` +- Unordered item 2 `* Unordered item 2` +- Unordered item 3 `* Unordered item 3` +- Unordered item 4 `* Unordered item 4` + +1. Ordered item 1 `1. Ordered item 1` +2. Ordered item 2 `2. Ordered item 2` +3. Ordered item 3 `3. Ordered item 3` +4. Ordered item 4 `4. Ordered item 4` + +## Tables + +| Header | Left-aligned | Centered | Right-aligned | +| ------ | :----------- | :------: | ------------: | +| Cell | Cell | Cell | Cell | +| Cell | Cell | Cell | Cell | + +``` +| Header | Left-aligned | Centered | Right-aligned | +| - | :- | :-: | -: | +| Cell | Cell | Cell | Cell | +| Cell | Cell | Cell | Cell | +``` + +## Links + +`[Link text](target URL)` + +**Example** + +[Download Anaconda here](https://www.anaconda.com/download) + +`[Download Anaconda here](https://www.anaconda.com/download)` + +## Images + +`![Alt text](./your-image.jpg "Custom hover title")` + +**Example** + +![Test image](./CommonUsedMarkdown.assets/testpic1.jpg "autumn") + +`![Test image](./CommonUsedMarkdown.assets/testpic1.jpg "autumn")` diff --git a/app/docs/CommunityShare/Geek/Katex/Seb1.en.mdx b/app/docs/CommunityShare/Geek/Katex/Seb1.en.mdx new file mode 100644 index 00000000..70d4352e --- /dev/null +++ b/app/docs/CommunityShare/Geek/Katex/Seb1.en.mdx @@ -0,0 +1,84 @@ +--- +title: Commonly Used Symbols +date: 2025-09-20T14:25:39.000Z +docId: r0inttjcby48tly602p410vo +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +## Greek Letters + +| Result | Command | Result | Command | +| :--------: | :--------: | :-------: | :-------: | +| $\alpha$ | `\alpha` | $\sigma$ | `\sigma` | +| $\beta$ | `\beta` | $\tau$ | `\tau` | +| $\gamma$ | `\gamma` | $\Gamma$ | `\Gamma` | +| $\delta$ | `\delta` | $\Delta$ | `\Delta` | +| $\epsilon$ | `\epsilon` | $\psi$ | `\psi` | +| $\zeta$ | `\zeta` | $\omega$ | `\omega` | +| $\eta$ | `\eta` | $\Omega$ | `\Omega` | +| $\theta$ | `\theta` | $\Theta$ | `\Theta` | +| $\lambda$ | `\lambda` | $\phi$ | `\phi` | +| $\mu$ | `\mu` | $\varphi$ | `\varphi` | +| $\pi$ | `\pi` | $\xi$ | `\xi` | +| $\rho$ | `\rho` | | | + +## Arrows + +| Result | Command | Result | Command | +| :---------------: | :---------------: | :------------------: | :------------------: | +| $\leftarrow$ | `\leftarrow` | $\Leftarrow$ | `\Leftarrow` | +| $\rightarrow$ | `\rightarrow` | $\Rightarrow$ | `\Rightarrow` | +| $\leftrightarrow$ | `\leftrightarrow` | $\Leftrightarrow$ | `\Leftrightarrow` | +| $\uparrow$ | `\uparrow` | $\mapsto$ | `\mapsto` | +| $\downarrow$ | `\downarrow` | $\rightleftharpoons$ | `\rightleftharpoons` | + +## Operators + +| Result | Command | Result | Command | +| :------: | :------: | :-------: | :-------: | +| $\times$ | `\times` | $\wedge$ | `\wedge` | +| $\div$ | `\div` | $\vee$ | `\vee` | +| $\cap$ | `\cap` | $\oplus$ | `\oplus` | +| $\cup$ | `\cup` | $\otimes$ | `\otimes` | +| $\cdot$ | `\cdot` | $\cdots$ | `\cdots` | + +## Relational Symbols + +| Result | Command | Result | Command | +| :-----: | :-----: | :---------: | :---------: | +| $\neq$ | `\neq` | $\subset$ | `\subset` | +| $\leq$ | `\leq` | $\subseteq$ | `\subseteq` | +| $\geq$ | `\geq` | $\approx$ | `\approx` | +| $\ll$ | `\ll` | $\equiv$ | `\equiv` | +| $\gg$ | `\gg` | $\cong$ | `\cong` | +| $\prec$ | `\prec` | $\parallel$ | `\parallel` | +| $\succ$ | `\succ` | $\perp$ | `\perp` | +| $\in$ | `\in` | $\notin$ | `\notin` | + +## Infinity, Partial Derivatives & Proof Symbols + +| Result | Command | Result | Command | +| :-----------: | :-----------: | :----------: | :----------: | +| $\infty$ | `\infty` | $\exists$ | `\exists` | +| $\partial$ | `\partial` | $\neg$ | `\neg` | +| $\varnothing$ | `\varnothing` | $\because$ | `\because` | +| | | $\therefore$ | `\therefore` | + +## Superscripts & Decorators + +| Result | Command | Result | Command | +| :-----------------------: | :-----------------------: | :------------------------: | :------------------------: | +| $\bar{a}$ | `\bar{a}` | | | +| $\hat{a}$ | `\hat{a}` | | | +| $\widehat{abc}$ | `\widehat{abc}` | | | +| $\tilde{a}$ | `\tilde{a}` | $\utilde{a}$ | `\utilde{a}` | +| $\widetilde{abc}$ | `\widetilde{abc}` | | | +| $\vec{A}$ | `\vec{A}` | | | +| $\overline{AB}$ | `\overline{AB}` | $\underline{AB}$ | `\underline{AB}` | +| $\overleftarrow{AB}$ | `\overleftarrow{AB}` | $\underleftarrow{AB}$ | `\underleftarrow{AB}` | +| $\overleftrightarrow{AB}$ | `\overleftrightarrow{AB}` | $\underleftrightarrow{AB}$ | `\underleftrightarrow{AB}` | +| $\overleftharpoon{AB}$ | `\overleftharpoon{AB}` | | | +| $\overgroup{AB}$ | `\overgroup{AB}` | $\undergroup{AB}$ | `\undergroup{AB}` | diff --git a/app/docs/CommunityShare/Geek/Katex/Seb2.en.mdx b/app/docs/CommunityShare/Geek/Katex/Seb2.en.mdx new file mode 100644 index 00000000..20518a41 --- /dev/null +++ b/app/docs/CommunityShare/Geek/Katex/Seb2.en.mdx @@ -0,0 +1,113 @@ +--- +title: Math Formula Syntax +date: 2025-09-20T14:25:39.000Z +docId: khcrztruqdku9fntd3dwzvwe +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +## Summation, Product & Binomial + +| Example | Command | +| :---------------: | :-------------: | +| $$\sum_{i=1}^n$$ | `\sum_{i=1}^n` | +| $$\prod_{i=1}^n$$ | `\prod_{i=1}^n` | +| $$\tbinom{a}{b}$$ | `\tbinom{a}{b}` | + +## Matrices & Determinants + +$$ +\left(\begin{matrix} +a & b \\ +c & d +\end{matrix}\right) +$$ + +```latex +$$ +\left(\begin{matrix} +a & b \\ +c & d +\end{matrix}\right) +$$ +``` + +$$ +\left|\begin{matrix} +a & b \\ +c & d +\end{matrix}\right| +$$ + +```latex +$$ +\left|\begin{matrix} +a & b \\ +c & d +\end{matrix}\right| +$$ +``` + +$$ +\left(\begin{matrix} +1 & 2 & \cdots & n \\ +2 & 3 & \cdots & n+1 \\ +\vdots & \vdots & \ddots & n^2 - 1\\ +n & n+1 & \cdots & n^2 +\end{matrix}\right) +$$ + +```latex +$$ +\left(\begin{matrix} +1 & 2 & \cdots & n \\ +2 & 3 & \cdots & n+1 \\ +\vdots & \vdots & \ddots & n^2 - 1\\ +n & n+1 & \cdots & n^2 +\end{matrix}\right) +$$ +``` + +## Aligned Equations + +$$ +\begin{aligned} + f(X|\theta) &= f(x_1,x_2,\cdots,x_n|\theta)\\ + &= f(x_1|\theta) \cdot f(x_2|\theta) \cdots f(x_n|\theta)\\ + &= \prod_{i=1}^n \frac{1}{x_i !} e^{-\theta}\theta^{x_i}\\ + &= \left(\prod_{i=1}^n \frac{1}{x_i !}\right) e^{-n \theta}\theta^{\sum_{i=1}^n x_i} +\end{aligned} +$$ + +```latex +$$ +\begin{aligned} + f(X|\theta) &= f(x_1,x_2,\cdots,x_n|\theta)\\ + &= f(x_1|\theta) \cdot f(x_2|\theta) \cdots f(x_n|\theta)\\ + &= \prod_{i=1}^n \frac{1}{x_i !} e^{-\theta}\theta^{x_i}\\ + &= \left(\prod_{i=1}^n \frac{1}{x_i !}\right) e^{-n \theta}\theta^{\sum_{i=1}^n x_i} +\end{aligned} +$$ +``` + +## Piecewise Functions (differs from standard LaTeX) + +Since `\equation` and the standard `\cases` syntax are not supported here, a workaround is needed: + +$$ +f(x,\theta) = \left\{ \begin{array}{ll} +\dfrac{1}{x\sqrt{2\pi \theta}} e^{-\frac{1}{2\theta} [\log(x)]^2} &,\text{ $x>0$} \\ +0 &,\text{ otherwise} +\end{array} \right. +$$ + +```latex +$$ +f(x,\theta) = \left\{ \begin{array}{ll} +\dfrac{1}{x\sqrt{2\pi \theta}} e^{-\frac{1}{2\theta} [\log(x)]^2} &,\text{ $x>0$} \\ +0 &,\text{ otherwise} +\end{array} \right. +$$ +``` diff --git a/app/docs/CommunityShare/Geek/git101.en.mdx b/app/docs/CommunityShare/Geek/git101.en.mdx new file mode 100644 index 00000000..1fd63e67 --- /dev/null +++ b/app/docs/CommunityShare/Geek/git101.en.mdx @@ -0,0 +1,49 @@ +--- +title: Git Getting Started Guide — Git Tips Every Developer Should Know +description: "" +date: "2025-09-19" +tags: + - tag-one +docId: tksz80mfqqyzwzzer5p3uxtg +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +## Most Common Git Commands + +| Basic | Branch & Remote | +| ------------------------------------------ | --------------------------------------------------- | +| `git init ` → Initialize a repo | `git branch` → List / create branches | +| `git clone ` → Clone a remote repo | `git checkout -b ` → New branch + switch | +| `git add ` → Stage a file | `git merge ` → Merge a branch | +| `git commit -m "msg"` → Commit changes | `git pull ` → Fetch + merge remote | +| `git push ` → Push branch | `git remote add ` → Add remote | +| `git status` → Check file status | `git fetch ` → Fetch remote branch | +| `git log --oneline` → One-line history | `git pull --rebase ` → Fetch + rebase | +| `git diff` → View unstaged changes | `git push --tags` → Push all tags | + +## Other Git Commands + +| Undo / Reset / Log / Diff | Config / Advanced Push / Rebase | +| ----------------------------------------------------------------------- | ----------------------------------------------------------- | +| `git commit --amend` → Modify the last commit | `git config user.name ` → Set repo author name | +| `git revert ` → Revert a commit | `git config --global user.name ` → Global author name | +| `git reset ` → Unstage a file | `git config --global user.email ` → Global email | +| `git reset --soft ` → Roll back commit, keep staged & working | `git config --global alias. ` → Create alias | +| `git reset --mixed ` → Roll back commit, keep working (default) | `git config --system core.editor ` → Set editor | +| `git reset --hard ` → Roll back commit, discard changes | `git config --global --edit` → Edit global config | +| `git clean -n` → Preview deletion of untracked files | `git push --force` → Force push (dangerous) | +| `git reflog` → View HEAD history | `git push --all` → Push all branches | +| `git log -` → Limit number of commits | `git rebase ` → Rebase a branch | +| `git log --stat` → File change stats | `git rebase -i ` → Interactive rebase | +| `git log -p` → Detailed commit diff | `git rebase --continue` → Continue rebase after conflict | +| `git log ..` → History in a range | `git rebase --abort` → Abort rebase | +| `git log -- ` → History for a specific file | `git log --author=""` → Search by author | +| `git diff HEAD` → Working tree vs. latest commit | `git log --grep=""` → Search by message | +| `git diff --cached` → Staged vs. latest commit | `git log --graph --decorate` → Graphical history | + +## Practice Exercises + +Here's a recommended interactive sandbox for learning Git — great for beginners to get hands-on quickly: https://learngitbranching.js.org/ diff --git a/app/docs/CommunityShare/Geek/leworldmodel.zh.md b/app/docs/CommunityShare/Geek/leworldmodel.zh.md new file mode 100644 index 00000000..eb05a472 --- /dev/null +++ b/app/docs/CommunityShare/Geek/leworldmodel.zh.md @@ -0,0 +1,18 @@ +--- +title: LeWorldModel +description: 从像素端到端稳定训练的联合嵌入预测架构 +date: "2026-04-08" +tags: + - 世界模型 +docId: boo70qqm8nos8b0q9h7zjrki +lang: zh +translatedFrom: en +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +联合嵌入预测架构（JEPA）为在紧凑隐空间中学习世界模型提供了一个颇具吸引力的框架，但现有方法仍然较为脆弱，往往依赖复杂的多项损失、指数移动平均、预训练编码器或辅助监督来避免表征坍塌。在本研究中，我们提出 LeWorldModel（LeWM）——首个仅用两项损失函数（下一嵌入预测损失 + 强制隐嵌入服从高斯分布的正则项）就能从原始像素端到端稳定训练的 JEPA。与当前唯一存在的端到端替代方案相比，可调损失超参数从六个降至一个。LeWM 拥有 1500 万参数，可在单张 GPU 上数小时内完成训练，规划速度最高比基于基础模型的世界模型快 48 倍，同时在多种 2D 和 3D 控制任务上保持竞争力。除控制任务外，我们还通过对物理量的探针分析表明，LeWM 的隐空间编码了有意义的物理结构。惊讶度评估进一步确认，该模型能可靠地检测出物理上不合理的事件。 + +
+ +<https://arxiv.org/pdf/2603.19312> diff --git a/app/docs/CommunityShare/Geek/picturecdn.en.mdx b/app/docs/CommunityShare/Geek/picturecdn.en.mdx new file mode 100644 index 00000000..dee8ac23 --- /dev/null +++ b/app/docs/CommunityShare/Geek/picturecdn.en.mdx @@ -0,0 +1,164 @@ +--- +title: How to Deploy Your Own GitHub Image Hosting with PictureCDN +description: "" +date: "2025-09-27" +tags: + - tag-one +docId: e6udpzrorhvgeeda6xpy1e0s +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +# 1. What Is an Image Host? + +- An image host is essentially a "bed" for storing images — an online service that lets you upload, store, and share images. Once you upload an image, you get a link you can share anywhere without worrying about storage capacity or bandwidth. + +## What Problems Does an Image Host Solve? + +- 1. Individual users and small websites often lack the server space and bandwidth to store and serve large numbers of images. An image host provides cheap or even free storage so you can keep as many images as you need without worry. +- 2. Image hosts usually include management features, making it easy to organize, search, and reuse your uploaded images. +- 3. Performance: Image hosts typically run on powerful servers with CDN (Content Delivery Network) infrastructure, enabling fast image delivery to users worldwide. This is critical for a good user experience. + +# 2. Creating an Image Hosting Repository on GitHub + +## 2.1 Create a Repository + +![Create repository](https://ravencaffeine.github.io/PictureCDN/images/20250927160645295.png) + +![](https://ravencaffeine.github.io/PictureCDN/images/20250927160645296.png) + +## 2.2 Generate a Personal Access Token + +- Click your profile picture or avatar in the top-right corner. +- Select "Settings" from the dropdown menu. + ![](https://ravencaffeine.github.io/PictureCDN/images/20250927160645297.png) + +### 2.2.1 In "Developer settings", click "Personal access tokens". + +- In the left sidebar, click "Developer settings". + +![](https://ravencaffeine.github.io/PictureCDN/images/20250927160645298.png) + +![](https://ravencaffeine.github.io/PictureCDN/images/20250927160645299.png) + +### 2.2.2 Generate new token (classic) + +![](https://ravencaffeine.github.io/PictureCDN/images/20250927160645300.png) + +- In the "Note" field, enter a descriptive name so you remember what this token is for. +- Choose an "Expiration" date. You can set it to never expire or pick a specific date. +- Select the appropriate "Scopes" (permissions). If you only need repository access, choosing "repo" is sufficient. +- Scroll down and click "Generate token". + +![](https://ravencaffeine.github.io/PictureCDN/images/20250927160645301.png) + +- Once generated, you'll see the token in plain text. **Copy and save it immediately** — this is the only time you'll see it. +- After leaving the page, you can only see whether the token exists, not its value. + +![](https://ravencaffeine.github.io/PictureCDN/images/20250927160645302.png) + +Note: + +- Personal access tokens are highly sensitive — treat them like passwords. Never share them or hard-code them in your source code. +- If you suspect a token has been compromised, revoke it immediately in GitHub settings. +- Tokens can perform a wide range of actions on your GitHub account, so choose permissions carefully. + +# 3. PicGo + +PicGo is an open-source image upload tool that supports multiple image hosting services, including Qiniu Cloud, Upyun, SM.MS, and GitHub. It helps you upload local images to your chosen host and generates shareable links automatically. + +## 3.1 What Problems Does PicGo Solve? + +1. **Image upload**: PicGo uploads local images to your image host quickly, saving manual upload time. +2. **Image management**: PicGo supports batch uploads and makes it easy to find and reuse previously uploaded images. +3. **Link generation**: After uploading, PicGo automatically generates a shareable link you can use anywhere. +4. **Multi-host support**: PicGo works with multiple image hosting services and lets you switch between them at any time. + +## 3.2 Download + +- Shandong University mirror: +- https://mirrors.sdu.edu.cn/github-release/Molunerfinn_PicGo/v2.3.1/ +- Original GitHub repository (releases tab): +- https://github.com/Molunerfinn/PicGo +- After installing, open it from the system tray icon in the bottom-left corner. + ![](https://ravencaffeine.github.io/PictureCDN/images/20250927160645303.png) + +## 3.3 Configuring the GitHub Image Host + +- Open Image Host Settings +- Select GitHub + ![](https://ravencaffeine.github.io/PictureCDN/images/20250927160645304.png) + +- **Repository name**: Enter `/` — the repo you created in step 2.1. +- **Branch**: The branch where your images will be stored. Usually `main` or `master`. +- **Token**: Paste the token you copied in step 2.2. +- **Storage path**: A folder within your repo. Setting it to `img/` is a common convention — it will be created automatically. +- **Custom domain**: Set this if you want to use GitHub Pages (see section 4). + +![](https://ravencaffeine.github.io/PictureCDN/images/20250927160645305.png) + +![](https://ravencaffeine.github.io/PictureCDN/images/20250927160645306.png) + +# 4. Setting Up a Custom Domain with GitHub Pages + +## 4.1 GitHub Pages Basics + +GitHub Pages is a static website hosting service. There are three types: + +#### 1. User / Organization Site: + +- Each GitHub account can have exactly one user or organization site. +- The repository must be named `.github.io` (or `.github.io` for organizations). +- The URL will be `https://.github.io` or `https://.github.io`. +- Limit: one user/org site per account. + +#### 2. Project Site: + +- A project site is tied to a specific repository, with URL `https://.github.io/`. +- Any repository can have its own project site. +- Limit: no explicit cap — one project site per repository. + +#### 3. General GitHub Pages Limits: + +- Recommended source repository size: under 1 GB. +- Published site size: no more than 1 GB. +- Monthly bandwidth: 100 GB (soft limit). +- Hourly build limit: 10 builds (soft limit, applies to GitHub Actions builds). + +## 4.2 Enabling GitHub Pages + +To get a cleaner image URL, enable GitHub Pages for your repository: + +- Go to your repository on GitHub and click "Settings". +- Scroll down to the "Pages" section. +- Under "Source", select your `main` branch (or `master`, depending on your default) and click "Save". +- After a moment, GitHub will generate a Pages URL (e.g., `https://your-username.github.io/image-hosting`). +- Back in PicGo's GitHub image host settings, set "Custom Domain" to this Pages URL. + +![](https://ravencaffeine.github.io/PictureCDN/images/20250927160645308.png) + +![](https://ravencaffeine.github.io/PictureCDN/images/20250927160645309.png) + +# 5. Uploading Images + +- **Drag and drop**: Open PicGo and drag an image directly into the window — it uploads automatically. +- **Manual file selection**: Click the "Upload" button in PicGo and select a local image file. + +![](https://ravencaffeine.github.io/PictureCDN/images/20250927160645310.png) + +![](https://ravencaffeine.github.io/PictureCDN/images/20250927160645311.png) + +- The image will be automatically committed to your repository. + ![](https://ravencaffeine.github.io/PictureCDN/images/20250927160645312.png) + +- You can now grab the image link. + ![](https://ravencaffeine.github.io/PictureCDN/images/20250927160645313.png) + +# References + +- 1. Complete tutorial: "Set Up a Personal Image Host on GitHub in 2025 — Step-by-Step with Screenshots" + https://www.cnblogs.com/ljbguanli/p/18928090 +- 2. CSDN post by "三金C_C": PicGo Configuration — Free Image Hosting +- Original link: https://blog.csdn.net/QAZJOU/article/details/146449613 diff --git a/app/docs/CommunityShare/Geek/raspberry-guide.en.md b/app/docs/CommunityShare/Geek/raspberry-guide.en.md new file mode 100644 index 00000000..d8227b09 --- /dev/null +++ b/app/docs/CommunityShare/Geek/raspberry-guide.en.md @@ -0,0 +1,313 @@ +--- +title: Building a Minecraft Server on an Idle Raspberry Pi +date: 2025-08-05T18:53:40.000Z +tags: null +docId: i0xmpskau105p83vq35wnxls +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +A walkthrough of setting up a Raspberry Pi Minecraft server from scratch and exposing it to the internet using FRP port forwarding. + +# Hardware Prerequisites + +One idle Raspberry Pi, and one VPS server with a public IP address. + +# Setting Up the Raspberry Pi + +Flash the Raspberry Pi OS image, set up your username and password, then power it on. + +## Assign a Static IP to the Raspberry Pi + +Open your router's settings panel and find the DHCP static IP allocation section. Assign a fixed IP to your Raspberry Pi. If you've forgotten the current IP, you can check it with: + +```shell +$ hostname -I # Example output: 192.168.2.102 +``` + +## Enable VNC Remote Desktop + +Log in via SSH and open the configuration panel: + +```shell +$ sudo raspi-config +``` + +1. Select **Interface Options** +2. Select **VNC** +3. When asked **Would you like the VNC Server to be enabled?**, select **YES** + +Open a VNC client on your computer and log in with your username and password to access the virtual desktop. + +## Install Java + +```sh +$ sudo -i # Temporarily gain admin privileges +$ cd /usr/local +``` + +Open the Raspberry Pi browser and download the [JDK](https://www.oracle.com/java/technologies/downloads/#java21). The file will be in `/home//Downloads/`. + +```sh +$ mkdir java +$ mv /home//Downloads/* /usr/local/java/ +$ cd java +$ tar -zxvf jdk-21_linux-aarch64_bin.tar.gz +# Some log output will appear +``` + +Configure environment variables: + +```sh +$ nano /etc/profile +``` + +Add the following at the end of the file: + +```sh +# Adjust the JDK version number as needed +export JAVA_HOME=/usr/local/java/jdk-21.0.8 +export CLASSPATH=.:$JAVA_HOME/lib/ +export PATH=.:$JAVA_HOME/bin:$PATH +# To exit: Ctrl+O, Enter, Ctrl+X +``` + +Reload the file: + +```sh +$ source /etc/profile +``` + +Verify the installation: + +```sh +$ java -version + +# Success looks like this: +java version "21.0.8" 2025-07-15 LTS +Java(TM) SE Runtime Environment (build 21.0.8+12-LTS-250) +Java HotSpot(TM) 64-Bit Server VM (build 21.0.8+12-LTS-250, mixed mode, sharing) +``` + +## Download the Minecraft Server Jar + +Open the browser and download the server jar. I'm using the [Fabric](https://fabricmc.net/use/server/) server loader. + +```sh +$ cd .. # Should return to /usr/local/ +$ mkdir minecraft +$ mv /home//Downloads/* /usr/local/minecraft +``` + +The first run will fail because you haven't agreed to the EULA yet: + +```sh +# My Pi has 8 GB RAM, so I'm allocating 4 GB here +# Note: the jar name will differ depending on the version you downloaded +$ java -Xmx4G -jar fabric-server-mc.1.21.1-loader.0.17.0-launcher.1.1.0.jar +``` + +Accept the EULA: + +```sh +$ nano eula.txt # Change eula=false to eula=true +$ # To exit: Ctrl+O, Enter, Ctrl+X +``` + +## Configure FRP + +The Raspberry Pi uses the ARM architecture. Download [frp](https://github.com/fatedier/frp/releases) — the latest version at the time of writing is 0.63.0. + +```sh +$ cd .. # Should be in /usr/local/java +$ wget https://github.com/fatedier/frp/releases/download/v0.63.0/frp_0.63.0_linux_arm64.tar.gz +# Some log output will appear +$ tar -zxvf frp_0.63.0_linux_arm64.tar.gz +# Some log output will appear +$ mv frp_0.63.0_linux_arm64 frp # Rename for convenience +``` + +Edit the configuration: + +```sh +$ cd frp +$ nano frpc.toml # 'c' in frpc stands for client +``` + +Edit `frpc.toml`: + +```toml +serverAddr = "Your VPS public IP" +serverPort = 7000 # Default value +auth.method = "token" +auth.token = "set a password here" + +[[proxies]] +name = "choose a service name" +type = "tcp" +localIP = "192.168.2.102" # Raspberry Pi IP +localPort = 25565 # MC server default port +remotePort = 25565 # MC server default port + +## To exit: Ctrl+O, Enter, Ctrl+X +``` + +## Install tmux + +This guide uses tmux for session management. You could also use `screen`, but I prefer tmux. + +```shell +$ apt install tmux +``` + +# Setting Up the VPS + +Log in via SSH using the root credentials from your provider's welcome email. + +## Configure FRP + +Install [frp](https://github.com/fatedier/frp/releases) on the VPS: + +```shell +$ cd /usr/local +$ wget https://github.com/fatedier/frp/releases/download/v0.63.0/frp_0.63.0_linux_amd64.tar.gz +$ # Some log output will appear +$ tar -zxvf frp_0.63.0_linux_amd64.tar.gz +$ # Some log output will appear +$ mv frp_0.63.0_linux_amd64 frp # Rename for convenience +``` + +Edit the configuration: + +```sh +$ cd frp +$ nano frps.toml # 's' in frps stands for server +``` + +Edit `frps.toml`: + +```toml +bindPort = 7000 +auth.method = "token" +auth.token = "same password as on the Raspberry Pi" + +[webServer] # Optional dashboard — remove if not needed +addr = "127.0.0.1" # Not exposed publicly; access via SSH tunnel. Use "0.0.0.0" to expose publicly +port = 7500 +user = "?" +password = "********" +``` + +## Run FRP in the Background + +Same process as on the Raspberry Pi: + +```sh +$ apt install tmux +``` + +Open the required ports: + +```shell +$ ufw allow 7000/tcp +$ ufw allow 25565/tcp +``` + +Start frp in a tmux session: + +```sh +$ tmux new -s +$ cd /usr/local/frp +$ ./frps -c frps.toml +# Press Ctrl+B then D to detach +``` + +Re-attach to the session later: + +```shell +$ tmux attach -t +``` + +Verify everything is running: + +```sh +$ tmux ls +# frp: 1 windows (created ) +$ ss -tlnp | grep 7000 +# Output here means frp is working +``` + +# Starting the Server on the Raspberry Pi + +## Write a Start Script + +```sh +$ cd /usr/local/minecraft +$ nano start.sh +``` + +Set the contents of `start.sh` to: + +```sh +#!/bin/bash +java -Xmx4G -jar fabric-server-mc.1.21.1-loader.0.17.0-launcher.1.1.0.jar nogui +``` + +Make it executable: + +```shell +$ chmod +x start.sh +``` + +## Run FRP and the Server + +### Run FRP + +```sh +$ tmux new -s frp +$ cd /usr/local/frp +$ ./frpc -c frpc.toml +# Press Ctrl+B then D to detach +``` + +### Run the Server + +```sh +$ tmux new -s mcserver +$ cd /usr/local/minecraft +$ ./start.sh +# Press Ctrl+B then D to detach +``` + +Verify both are running: + +```sh +$ tmux ls +# frp: 1 windows (created ) +# mcserver: 1 windows (created ) +``` + +# (Optional) Configure a Domain Name + +At this point you can already connect to the game using your VPS's public IP: + +``` +?.?.?.?:25565 # VPS public IP +``` + +If you have your own domain, log in to the [Cloudflare dashboard](https://dash.cloudflare.com/) and add a DNS record: + +| | | +| :---------------------- | :-------------------- | +| Type | A | +| Name (required) | rasp | +| IPv4 address (required) | <VPS public IP> | +| Proxy status | DNS only | + +Then you can connect using your domain: + +``` +rasp.:25565 +``` diff --git a/app/docs/CommunityShare/Geek/swanlab.en.mdx b/app/docs/CommunityShare/Geek/swanlab.en.mdx new file mode 100644 index 00000000..00764a2d --- /dev/null +++ b/app/docs/CommunityShare/Geek/swanlab.en.mdx @@ -0,0 +1,99 @@ +--- +title: SwanLab Quick Start Guide +description: "" +date: "2025-09-23" +tags: + - tag-one +docId: mhyoknm6vj8jmp186oli5f5c +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +I've been running a lot of experiments lately and things were getting messy fast — results scattered everywhere, and I was seriously considering just tracking everything in Excel. + +Then I remembered a teammate from group 9444 mentioning SwanLab for experiment tracking and visualization. So I put together this quick start guide — it'll be part of my own experiment pipeline going forward. + +If you have a better system for managing research experiments, feel free to share it. + +Anyway, let's get into SwanLab. Here's the quick start workflow for experiment tracking with SwanLab. + +## 1. Create an Account & Get Your API Key + +First, open the SwanLab website: [https://swanlab.cn/](https://swanlab.cn/) +If you don't have an account yet, register on the site first. + +Once you're logged in, create a project. You'll see a Quick Start guide — just follow it step by step and your API key will be right there. I've copied the key steps here for convenience. + +## 2. Install the SwanLab Library + +In any environment with Python 3, install the SwanLab client library via pip: + +```bash +pip install swanlab +``` + +## 3. Log In to SwanLab + +Run the following command in your terminal: + +```bash +swanlab login +``` + +You'll see a prompt like this: + +``` +swanlab: Logging into swanlab cloud. +swanlab: You can find your API key at: https://swanlab.cn/settings +swanlab: Paste an API key from your profile and hit enter, or press 'CTRL-C' to quit: +``` + +Paste the API key you copied from your profile settings page and you're in. ([docs.swanlab.cn](https://docs.swanlab.cn/en/guide_cloud/general/quick-start.html)) + +## 4. Submit an Experiment + +```python +import swanlab +import random + +# Initialize a new SwanLab run to track this script +swanlab.init( + # Set the project this run belongs to + project="mteb-ailastatue", + workspace="mira", + # Track hyperparameters and run metadata + config={ + "learning_rate": 0.02, + "architecture": "CNN", + "dataset": "CIFAR-100", + "epochs": 10 + } +) + +# Simulate training +epochs = 10 +offset = random.random() / 5 +for epoch in range(2, epochs): + acc = 1 - 2 ** -epoch - random.random() / epoch - offset + loss = 2 ** -epoch + random.random() / epoch + offset + + # Log training metrics to SwanLab + swanlab.log({"acc": acc, "loss": loss}) + +# [Optional] Mark the run as finished — required in notebook environments +swanlab.finish() + + +``` + +--- + +## 5. View Your Results! + +After running the code, navigate to your newly created project to compare different experiments and their metrics. + +## References + +Official docs: [Quick Start Guide](https://docs.swanlab.cn/en/guide_cloud/general/quick-start.html) diff --git a/app/docs/CommunityShare/RAG/context_engineering_intro.en.md b/app/docs/CommunityShare/RAG/context_engineering_intro.en.md new file mode 100644 index 00000000..2291bcfb --- /dev/null +++ b/app/docs/CommunityShare/RAG/context_engineering_intro.en.md @@ -0,0 +1,72 @@ +--- +title: A Quick Introduction to Context Engineering +description: "" +date: "2025-10-03" +tags: + - tag-one +docId: wdqqrepoy43jiieyyjmaekk1 +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +# A Quick Introduction to Context Engineering + +## Key Concepts + +Problems it addresses: + +- Most models have a very limited context window +- Noisy, poorly organized inputs hurt model understanding +- More input = higher cost (tokens are expensive) + +**Context**: Everything fed to the model as input — the user's question, background information, reference material, available tools, tool execution results, conversation history, and so on. The model generates answers based on all of this. + +**Context window**: The maximum amount of input a model can process. Measured in tokens — for example, Gemini 2.5 Pro has a 1 million token context window, meaning it can handle up to 1 million tokens of input at once. + +**Context engineering**: The deliberate design of what goes into the model's input. The goal is to help the model understand more accurately, respond better, and spend fewer tokens — all within a limited context window. + +The common experience of LLMs "forgetting" earlier parts of a conversation happens precisely because of context window size limits. + +Context engineering is especially important when building agents. + +## Approaches + +### Saving Context + +A classic example is ChatGPT's long-term memory feature. + +Store the context you want the model to remember in a database or on disk, and retrieve it when needed. + +### Selecting Context + +Choose the most relevant information from a large pool of data to include in the model's input. + +**Static selection**: Content that always goes into the context — for example, a system prompt that guides the model's behavior and ensures safe, reliable outputs. + +**Dynamic selection**: Content chosen based on what's most relevant to the current query — for example, pulling entries from a long-term memory store in ChatGPT, or an agent selecting which tools to include based on the current task. + +RAG is one implementation of dynamic selection. + +### Compressing Context + +The two biggest space consumers in a context are model-generated text and tool execution results. + +One practice from Claude Code: when the context reaches a certain size, it runs auto-compact — discarding raw content and keeping only a summary of what was there. + +### Isolating Context + +This typically comes up in multi-agent scenarios. + +Anthropic's approach: + +![Anthropic's approach](https://img.coly.cc/obs-img/2025/10/7110909d5366ba7747f037ae9300f7bc.png) + +Different agents each have their own dedicated tools, independent execution histories, and separate memory systems. Their contexts remain isolated from one another. + +### Further Reading + +LangChain — Context Engineering: http://blog.langchain.com/context-engineering-for-agents/ + +Cognition: https://cognition.ai/blog/dont-build-multi-agents diff --git a/app/docs/CommunityShare/RAG/embedding.en.mdx b/app/docs/CommunityShare/RAG/embedding.en.mdx new file mode 100644 index 00000000..5107780f --- /dev/null +++ b/app/docs/CommunityShare/RAG/embedding.en.mdx @@ -0,0 +1,67 @@ +--- +title: A Beginner's Guide to Fine-Tuning Embedding Models +description: "" +date: "2025-09-22" +tags: + - RAG +docId: eyd32o3ebd5q69hfbb2enxqi +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +Embedding models are something many people haven't heard of, and a fair question to ask is: modern LLMs are already so powerful — do we really need a dedicated embedding model on top of that? + +Here's the thing. LLMs are impressive, but they're also large. Deploying them requires serious hardware, and when the barrier to deployment is high, your application is constrained. And if you're calling them via API, the ongoing cost adds up fast. Beyond that, a general-purpose LLM with strong generalization might actually underperform a smaller, task-specific fine-tuned embedding model in specialized domains. If you're applying a RAG system to financial compliance, an internal knowledge base, or technical documentation Q&A, you'll notice that even the best general-purpose embedding models often struggle with domain-specific terminology and context. + +That's why more and more people are turning to fine-tuning — to help embedding models better understand their own data. And if you can master a domain-adaptive fine-tuning approach for embedding models, you'll have something applicable across virtually every industry. Every company could fine-tune an embedding model for their own domain. The potential is real. + +## Why Fine-Tune? + +The value of fine-tuning shows up first in task-specific performance. In finance or healthcare, for example, general-purpose embeddings often fail to capture domain-specific terminology or context precisely. A fine-tuned model can significantly improve retrieval quality, and that improvement flows directly into the final Q&A output. + +That said, fine-tuning isn't a cure-all. Many performance issues come from elsewhere: sometimes the query fundamentally needs keyword matching, sometimes the chunking strategy is poor, sometimes the model's embedding dimension is simply too small. Fine-tuning is only worth the investment once you've ruled out these other issues and confirmed that the bottleneck is actually in semantic understanding. + +What's more interesting is that fine-tuning doesn't always mean "bigger and stronger." In practice, a small model with task-specific fine-tuning can often match or beat large commercial models, while delivering lower latency and more predictable costs. And don't overlook a fundamental truth: data is the real ceiling on embedding quality. Without data, or with low-quality data, even the best fine-tuning process won't produce a miracle. + +## Recent Innovations + +A few developments in recent blog posts and papers are worth paying attention to. The dual-LLM synthesis-and-evaluation chain is a particularly elegant approach: use one LLM to generate diverse queries from documents, then use a second LLM as a "judge" to filter out low-quality samples. This gets you high-quality training pairs with almost no manual labeling. + +Then there's the fine-tuning vs. re-ranking tradeoff. Fine-tuning improves relevance without adding latency, but it requires re-embedding your entire corpus. Re-ranking avoids that re-embedding step but adds API calls and latency. The best practice isn't to pick one — it's to combine them based on your situation: use fine-tuning for a stable core corpus, and rely on re-ranking for frequently updated portions. + +There's also an often-overlooked point about small models. With the right fine-tuning, a small model can compete directly with large ones. High-quality synthetic data plays a big role here — techniques like paraphrasing, diversity control, and hard negative mining can build training sets that are both larger in scale and broader in coverage, significantly boosting fine-tuning effectiveness. Researchers have noted that the performance ceiling for embedding models is typically constrained by the coverage and quality of training data, not model parameter count. This further underscores why high-quality synthetic datasets matter so much. Looking ahead, multi-task and instruction-based training are becoming the norm — integrating signals across different domains into a single small model to give it stronger cross-domain generalization. + +On the data processing side, it's worth keeping an eye on [OpenDCAI/DataFlow](https://github.com/OpenDCAI/DataFlow). DataFlow is a data-centric AI system built specifically for fine-tuning and RAG workflows. It automates parsing, generation, cleaning, and quality assessment from noisy raw data sources (PDFs, web pages, low-quality QA pairs) through modular pipelines, making it much easier to build high-quality training sets. It has been validated in healthcare, finance, and legal domains. + +## Practical Takeaways + +If you're actually planning to fine-tune in production, you need a clear decision path. First, diagnose — confirm the bottleneck really is in semantic understanding. Only if the answer is yes should you proceed to fine-tuning. Second, construct your data. The standard approach: generate diverse queries from domain documents, filter with an LLM quality check, and build positive-negative pairs — making sure to include hard negatives and deduplicating to avoid data leakage. + +During training, contrastive learning is the standard choice: multiple negatives, triplet loss, or cosine embedding loss. Pair that with small-scale hyperparameter search to get learning rate, batch size, epoch count, and pooling strategy into a reasonable range. During evaluation, don't just look at Recall@k and MRR — also check end-to-end Q&A accuracy and grounding hit rate, and factor latency and cost into your decisions. When you deploy, start by applying fine-tuning to your stable core corpus to gain consistent high relevance, then use re-ranking or hybrid retrieval for high-churn data — that combination tends to be robust in practice. + +## Summary + +The value of fine-tuning an embedding model isn't about chasing a universal "optimal model." It's about making the model genuinely understand your data and your task. It's a systems engineering effort: diagnose first, confirm the bottleneck; then fine-tune with high-quality data and systematic experimentation; finally, close the loop with re-ranking for a stable, production-ready system. + +For most organizations, the most practical approach is: fine-tune on stable core data for low-latency, high-relevance retrieval; use re-ranking on high-churn data for flexibility; and expand the model's boundaries incrementally through synthetic data and multi-task training. The end goal isn't point-optimal performance — it's achieving a dynamic balance across performance, latency, and cost for the entire RAG system. + +## References + +ACM Digital Library. Exploring Parameter-Efficient Fine-Tuning Techniques for Code Models. +https://dl.acm.org/doi/10.1145/3714461 + +Databricks. Improving Retrieval and RAG with Embedding Model Finetuning. +https://www.databricks.com/blog/improving-retrieval-and-rag-embedding-model-finetuning + +NAACL 2025. Little Giants: Synthesizing High-Quality Embedding Data at Scale. +https://aclanthology.org/2025.naacl-long.64/ + +Q. Zhou et al. Embedding Technical Report. + +arXiv. Multi-task Retriever Fine-tuning for Domain-specific and General-purpose Tasks. +https://arxiv.org/abs/2501.04652 + +Weaviate. Why, When and How to Fine-Tune a Custom Embedding Model. +https://weaviate.io/blog/fine-tune-embedding-model diff --git a/app/docs/CommunityShare/RAG/rag.en.mdx b/app/docs/CommunityShare/RAG/rag.en.mdx new file mode 100644 index 00000000..9ca4be76 --- /dev/null +++ b/app/docs/CommunityShare/RAG/rag.en.mdx @@ -0,0 +1,107 @@ +--- +title: RAG +description: "" +date: "2025-09-19" +tags: + - tag-one +docId: zywri1bs64awfi9utfjy14ll +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +# RAG + +This RAG toy demo is designed to help people unfamiliar with RAG understand the full pipeline in under 10 minutes. + +You can also pair it with "Mark's Tech Notes" for a deeper read. + +## RAG — Retrieval-Augmented Generation + +**Problem it solves:** Intelligent Q&A, building external knowledge bases, and specializing LLMs for specific domains. + +**Tools you need:** An LLM, an external knowledge base, and domain-specific data. + +**The core idea of RAG** is: + +1. Search the domain-specific files for relevant information +2. Use the retrieved information as supplementary context +3. Bundle it together with the user's query and send everything to the LLM to get an answer + +## RAG Pipeline + +Chunking → Embedding → Store in vector DB → User query → Retrieval → Re-ranking → Pack processed data + query → Feed to LLM → Get answer + +## Part 1: Building Your Own Minimal RAG + +### 1. Chunking + +Split your document into specific segments. + +**Example:** + +- Our document: "Today the weather is great" +- If we split by character: the entire document becomes "T", "o", "d", "a", "y", "t", "h", "e", "w", "e", "a", "t", "h", "e", "r", "i", "s", "g", "r", "e", "a", "t" — 22 separate pieces + +**Optimization direction:** In real-world engineering, chunking strategy is a key optimization lever for any RAG system. Individual characters carry very little semantic information in isolation. You could instead chunk by word, sentence, paragraph, page, or chapter — the right strategy can dramatically improve RAG performance. + +Chunks produced by this step are referred to as "slices" in the rest of this guide. + +### 2. Embedding the Slices + +Embedding converts the text in each slice into a vector representation that computers can process. + +There are many approaches. You'll need an embedding model — typically you can use a pre-trained model from Hugging Face via the `sentence-transformers` library. For very small datasets, one-hot encoding works too. Beginners don't need to understand the internals — just know that tools exist to convert text slices into vectors. This falls under NLP (Natural Language Processing). + +- For example, "Today" might become `[0.23, 0.34, 0.54, 0.23, 0.76]`. + +Because embedding can be computationally expensive and the underlying documents don't change, we don't need to re-embed every time. We embed everything once and store the results in a database. + +**Optimization direction:** Different embedding models produce different results. You can use open-source pre-trained models, or if you have high-quality domain data, train your own embedding model. + +### 3. Storing in a Vector Database + +A vector database is built for storing and querying vectors. + +To avoid re-embedding documents every time, we store both the vector and its corresponding slice text together in a vector database. + +Vector databases come in different forms: lightweight local embeddings (stored on your machine) and cloud-hosted deployments (queried over the network). For lightweight projects, local libraries like `faiss` or `MongoDB` work great. + +**Optimization direction:** The choice of vector database can also affect overall RAG system performance. + +### 4. The Base LLM + +The LLM can be deployed locally or accessed via API. + +For example, you could use Gemini, which has a free daily token quota. Search for how to get a Gemini API key. + +**Optimization direction:** The base LLM has a direct impact on RAG quality. Besides general-purpose models like GPT and Gemini, some practitioners fine-tune a general LLM or train a specialized model for specific domains — for example, a model focused on medical diagnosis or legal consultation. These specialized models may lag behind in general capability but excel within their target domain. + +## Part 2: Using Your RAG System + +Assuming you've already built a RAG system and have a ChatGPT-style interface to interact with, here's how a query flows through the system. + +### 5. User Query & Retrieval + +Say the user asks: "What's the weather like today?" + +We embed this query using the same embedding model used during setup, then compare the resulting vector against all vectors in our database. The matching process retrieves the slices most similar to the query — this is called retrieval. + +Retrieval prioritizes recall over precision: we'd rather include everything relevant and filter later than miss something important. + +### 6. Re-ranking + +Re-ranking is a further filtering step on the retrieved results. For example, if we retrieved 20 slices, we select the top 5 most relevant to the user's query to reduce noise. + +**Optimization direction:** Different indexing strategies and re-ranking algorithms can yield very different results. + +### 7. Calling the LLM and Getting an Answer + +Finally, we bundle the user query and the top 5 slices together, apply prompt engineering, and send everything to the LLM. The LLM uses both the query and the relevant context slices to generate an answer. + +**Optimization direction:** Investing in prompt engineering pays off here. + +**For example, the basic logic of CoT (Chain of Thought):** + +- "A is True then B is true" and "A is True" = "B is True" From 7a15e9bd5c79a4dbdc6b847ce97004ca9645b380 Mon Sep 17 00:00:00 2001 From: longsizhuo Date: Wed, 15 Apr 2026 17:44:52 +0000 Subject: [PATCH 10/19] =?UTF-8?q?feat(docs):=20i18n=20ai/=20=E5=89=8D?= =?UTF-8?q?=E5=8D=8A=E9=83=A8=E5=88=86=E5=85=A8=E9=83=A8=E7=BF=BB=E8=AF=91?= =?UTF-8?q?=E5=AE=8C=E6=88=90=20(24=20=E7=AF=87)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit translator-ai-1 产出。全部 zh→en。覆盖： - Introduction-of-Multi-agents-system (1) - MoE (2) - Multi-agents-system-on-Code-Translation (1) - agents-todo (2) - ai-math-basics (9) — 含 calculus / linear-algebra / probability 等子目录 - compute-platforms (1，handbook 已在 MVP 提交) - foundation-models (7) — lifecycle / datasets / training / finetune 等 - generative-todo (1) 所有翻译版 frontmatter 继承原文 docId，带 translatedFrom: zh 标记供 contributors 脚本跳过统计。 --- .../introduction_of_multi-agents_system.en.md | 159 +++++++ app/docs/ai/MoE/MOE-intro.en.md | 147 ++++++ app/docs/ai/MoE/moe-update.en.md | 231 +++++++++ .../code-translation-intro.en.mdx | 24 + .../ai/agents-todo/agent-ecosystem.en.mdx | 63 +++ .../ai/agents-todo/cs294-194-196/index.en.mdx | 15 + .../calculus-optimization/index.en.mdx | 49 ++ .../information-theory/index.en.mdx | 47 ++ .../linear-algebra/index.en.mdx | 54 +++ .../linear-algebra/resources/index.en.mdx | 20 + .../ai/ai-math-basics/math-foundations.en.mdx | 89 ++++ app/docs/ai/ai-math-basics/math_books.en.md | 123 +++++ .../numerical-analysis/index.en.mdx | 40 ++ .../probability-statistics/index.en.mdx | 58 +++ .../resources/index.en.mdx | 16 + .../model-compuational-resource-demand.en.md | 198 ++++++++ .../foundation-models/datasets/index.en.mdx | 327 +++++++++++++ .../deploy-infer/index.en.mdx | 443 ++++++++++++++++++ .../foundation-models/evaluation/index.en.mdx | 287 ++++++++++++ .../foundation-models/finetune/index.en.mdx | 348 ++++++++++++++ .../foundation-models-lifecycle.en.mdx | 119 +++++ .../qkv-interview/index.en.mdx | 110 +++++ .../foundation-models/training/index.en.mdx | 293 ++++++++++++ .../generative-models-plan.en.mdx | 12 + 24 files changed, 3272 insertions(+) create mode 100644 app/docs/ai/Introduction-of-Multi-agents-system/introduction_of_multi-agents_system.en.md create mode 100644 app/docs/ai/MoE/MOE-intro.en.md create mode 100644 app/docs/ai/MoE/moe-update.en.md create mode 100644 app/docs/ai/Multi-agents-system-on-Code-Translation/code-translation-intro.en.mdx create mode 100644 app/docs/ai/agents-todo/agent-ecosystem.en.mdx create mode 100644 app/docs/ai/agents-todo/cs294-194-196/index.en.mdx create mode 100644 app/docs/ai/ai-math-basics/calculus-optimization/index.en.mdx create mode 100644 app/docs/ai/ai-math-basics/information-theory/index.en.mdx create mode 100644 app/docs/ai/ai-math-basics/linear-algebra/index.en.mdx create mode 100644 app/docs/ai/ai-math-basics/linear-algebra/resources/index.en.mdx create mode 100644 app/docs/ai/ai-math-basics/math-foundations.en.mdx create mode 100644 app/docs/ai/ai-math-basics/math_books.en.md create mode 100644 app/docs/ai/ai-math-basics/numerical-analysis/index.en.mdx create mode 100644 app/docs/ai/ai-math-basics/probability-statistics/index.en.mdx create mode 100644 app/docs/ai/ai-math-basics/probability-statistics/resources/index.en.mdx create mode 100644 app/docs/ai/compute-platforms/model-compuational-resource-demand.en.md create mode 100644 app/docs/ai/foundation-models/datasets/index.en.mdx create mode 100644 app/docs/ai/foundation-models/deploy-infer/index.en.mdx create mode 100644 app/docs/ai/foundation-models/evaluation/index.en.mdx create mode 100644 app/docs/ai/foundation-models/finetune/index.en.mdx create mode 100644 app/docs/ai/foundation-models/foundation-models-lifecycle.en.mdx create mode 100644 app/docs/ai/foundation-models/qkv-interview/index.en.mdx create mode 100644 app/docs/ai/foundation-models/training/index.en.mdx create mode 100644 app/docs/ai/generative-todo/generative-models-plan.en.mdx diff --git a/app/docs/ai/Introduction-of-Multi-agents-system/introduction_of_multi-agents_system.en.md b/app/docs/ai/Introduction-of-Multi-agents-system/introduction_of_multi-agents_system.en.md new file mode 100644 index 00000000..0c14307c --- /dev/null +++ b/app/docs/ai/Introduction-of-Multi-agents-system/introduction_of_multi-agents_system.en.md @@ -0,0 +1,159 @@ +--- +title: Introduction of Multi-Agent Systems (For Any Task You Want) +description: "" +date: "2025-09-29" +tags: + - tag-one +docId: h53uwefhlykt9ietsx9x0vtn +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +# Introduction of Multi-Agent Systems (For Any Task You Want) + +Overview of Multi-Agent Systems + +## 1. What Is a Multi-Agent System (MAS)? + +A Multi-Agent System (MAS) is a computational system composed of multiple relatively autonomous agents that interact, cooperate, or compete within a shared environment to achieve individual or collective goals. +It focuses not on the optimal behavior of a single agent, but on the organization, coordination, and emergent behavior at the group level. +Note: Emergent behavior refers to behaviors that arise from the interaction and collaboration of multiple agents that no single agent could accomplish alone. For example, flocks of birds follow simple rules to produce elegant formations that resist air currents — formations that were never explicitly designed. +Intuitive understanding: Think of LLMs as multiple "roles" that simulate team/department collaboration to complete tasks together. + +## 2. Typical Applications and Problem Types + +Real-world distributed problems: power grid scheduling, intelligent transportation, supply chains, disaster response — all naturally exhibit distributed, dynamic, and uncertain characteristics that monolithic systems struggle to handle with global optimality and robustness. + +Example research directions: generation, translation, repair, judge, etc. + +## 3. Core Concepts in Multi-Agent Systems + +### 3.1 Agent + +A computational entity that operates in an environment through a Perception — Deliberation/Policy — Action cycle. + +Typical properties: autonomy, reactivity, proactiveness (initiative), sociability (ability to interact). + +### 3.2 Environment + +The object that agents perceive and act upon; can be fully/partially observable, deterministic/stochastic, static/dynamic, discrete/continuous. + +| Dimension | Definition | Characteristics / Key Points | Typical Examples | Impact on Agent Design | +| ---------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Fully Observable vs Partially Observable | Whether the agent can perceive the full state of the environment at every moment | If fully observable, the agent can make decisions based on the current state directly; if partially observable, hidden information exists and the agent may need internal memory and uncertainty reasoning | Chess is fully observable; poker (where opponents' hands are hidden) is partially observable | In partially observable environments, agents typically need to maintain a **belief state** (probability distribution over true states) or an internal state model, making policies more complex | +| Deterministic vs Stochastic / Nondeterministic | Whether a given state + action leads to a unique next state/outcome, or multiple possible outcomes/probability distributions | Deterministic: action + current state uniquely determine the next state; stochastic/nondeterministic: multiple possible transitions with probability distributions | Board games (e.g., chess) are approximately deterministic; real-world robot manipulation and traffic systems often involve stochasticity | In stochastic environments, agent policies must account for expectations/distributions/risk, e.g., probabilistic policies, reinforcement learning, robust design | +| Static vs Dynamic | Whether the environment can change while the agent is deliberating/acting | Static: environment remains unchanged during the agent's decision-making; dynamic: the environment may evolve while the agent thinks/acts | In a turn-based board game, the environment is static during the current agent's turn; traffic systems are dynamic as other vehicles/pedestrians continuously change | In dynamic environments, agents need fast response, real-time planning, and future prediction capabilities — they cannot afford costly computation delays | +| Discrete vs Continuous | Whether the state, action, and time of the environment form a discrete/enumerable set or a continuous/real-valued domain | Discrete: states/actions/time are enumerable or discrete; continuous: these quantities vary over real-valued domains | Board games, grid worlds, turn-based games are discrete; robot positions/velocities/accelerations, drone control are continuous | In continuous environments, agents typically use function approximation (neural networks, control models), continuous policies, differential equations, or continuous action optimization; in discrete environments, enumeration, search, and discrete RL apply | + +### 3.3 Interaction + +Forms include communication, negotiation, competition, cooperation, game theory, etc. + +### 3.4 Organization + +The totality of roles, hierarchies, norms, protocols, and team structures. + +| Component | Meaning / Function | Common Design Approaches / Examples | Considerations / Trade-offs | +| :------------------------------------: | :---------------------------------------------------------------------------------------------------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Roles | Each agent's functional position and behavioral responsibilities in the organization. Roles abstract behavioral interfaces and capability constraints | "Planner" role handles task decomposition; "Executor" handles execution; "Critic" handles evaluation; "Communicator" handles information relay | Responsibilities must be clear and not overly overlapping; avoid strong role coupling; capabilities and resource allocation must match | +| Hierarchy | Superior-subordinate relationships among roles/agents, directing control, supervision, and command flow | Manager/Worker architecture: high-level agents make strategic decisions, low-level agents execute; multi-level nesting (macro → meso → micro) | Hierarchy helps manage complexity and maintain clear command flow; but too many levels can cause communication bottlenecks, delays, and single points of failure | +| Norms / Normative Rules | Conventions or hard rules that constrain agent behavior, coordinate conflicts, and ensure safety | E.g., "cannot access the same resource simultaneously," "respond to urgent tasks first," "cannot act beyond assigned roles" | Too loose leads to chaos; too strict reduces flexibility; penalty mechanisms / compliance checks must be designed | +| Protocols / Interaction Protocols | Mechanisms and conventions for how agents communicate, negotiate, trade, synchronize, and deliberate | Auction, Contract Net, Negotiation Protocol, Consensus | Must consider performance (communication cost, latency), robustness (error handling, failure recovery), expressiveness (whether semantic interaction is sufficient) | +| Team Structure / Coalitions / Grouping | How agents are organized into sub-teams or collaborative groups, and how these groups cooperate | Static teams (fixed groupings), dynamic teams (task-triggered groupings), cross-team coalitions | Must adapt to task requirements and capability distribution; dynamic structures increase flexibility but incur reorganization costs and coordination overhead | + +### 3.5 Goals/Utility + +Individual goals and global social welfare may be aligned or conflicting, involving mechanism design. The ultimate aim should be toward task completion and utility maximization. + +## 4. System Composition and Typical Architectures + +### 4.1 Agent Internal Architecture + +**Reactive/Behavior-based**: e.g., subsumption architecture (layered behaviors) — fast response but weak planning. + +**BDI (Belief–Desire–Intention)**: models rational decision-making through beliefs/desires/intentions, suitable for interpretable planning scenarios. + +**Learning-based**: based on RL/supervised/self-supervised learning; in MARL, policies can be shared or trained independently. + +**LLM-Agent**: uses a large language model as the core, combined with tool calling, memory, retrieval, reflection, and actuators; excels at complex reasoning and open-environment tasks. + +### 4.2 Multi-Agent Architectures + +**Centralized Orchestration (Orchestrator)**: central scheduling (e.g., Planner/Router) assigns tasks; provides a strong global view but has a single point of failure. + +**Distributed Cooperation (Peer-to-Peer)**: agents interact as equals; high elasticity but complex protocols. + +**Hierarchical/Hybrid**: upper-level planning, lower-level execution; balances global and local efficiency. + +**Blackboard / Shared Memory**: agents exchange hypotheses and partial solutions through a shared workspace. + +### 4.3 Communication and Coordination Mechanisms + +Communication languages/protocols: early examples include KQML, FIPA-ACL; in engineering practice, MQ/HTTP/gRPC and structured messages (JSON/Proto) are commonly used. + +### 4.4 Coordination Methods + +**Contract Net and Auction/Bidding**: suitable for task assignment and resource competition. + +**Negotiation/Voting/Consensus**: e.g., Paxos/Raft or multi-party voting strategies. + +**Formation/Grouping and Role Switching**: formation control, dynamic role assignment. + +**Mechanism Design**: uses incentive-compatible rules to guide individual rational behavior toward desired collective outcomes. + +**Organizational Structures**: Hierarchy, Holarchy, Team/Coalition, and Roles & Norms-based social organization. + +### 4.5 Key Points in Multi-Agent Reinforcement Learning (MARL) + +**Non-stationarity**: changes in others' policies cause the environment to appear non-static to any individual agent, making training harder. + +**Training-execution paradigm**: Centralized Training, Decentralized Execution (CTDE) is the dominant approach. + +### 4.6 Method Families (Examples) + +**Value decomposition**: VDN, QMIX decompose global value into individual values. + +**Actor-Critic**: e.g., MADDPG (centralized Critic, decentralized Actor). + +**Opponent modeling / Game learning**: Nash equilibrium, transferable policies, meta-learning. + +Key challenges: credit assignment, scalability, partial observability, exploration-exploitation balance, communication bandwidth and latency. + +## 5. LLM-Driven Multi-Agent Paradigm (Main Focus) + +### 5.1 Role Division + +- Planner +- Researcher (retrieval/analysis) +- Coder/Executor (tool execution) +- Critic/Verifier (review and validation) +- Refiner (repair) + +### 5.2 Collaboration Patterns + +**Debate/Deliberation**: mutual evaluation to improve reasoning robustness. + +**Reflection/Memory**: experience summarization, long-term memory stores, external knowledge retrieval. + +**Graph-of-Agents**: explicitly represents task workflow as a DAG/state machine. + +### 5.3 Engineering Considerations + +- Prompt templating +- Tool/database/code executor integration +- Message routing and caching +- Cost and latency control +- Security (privilege escalation / data leakage / injection) + +## 6. Recommended Classic Papers / Works + +- AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation +- CAMEL: Communicative Agents for "Mind" Exploration of LLM Society +- Improving Factuality and Reasoning in Language Models through Multi-Agent Debate +- Should We Be Going MAD? A Look at Multi-Agent Debate +- Reflexion: Language Agents with Verbal Reinforcement Learning +- Self-Refine: Iterative Refinement with Self-Feedback +- Language Agents as Optimizable Graphs (GPTSwarm) +- Graph of Thoughts: Solving Elaborate Problems with LLMs diff --git a/app/docs/ai/MoE/MOE-intro.en.md b/app/docs/ai/MoE/MOE-intro.en.md new file mode 100644 index 00000000..3493165e --- /dev/null +++ b/app/docs/ai/MoE/MOE-intro.en.md @@ -0,0 +1,147 @@ +--- +title: A Brief Introduction to MoE +description: "A brief introduction to the Mixture of Experts (MoE) architecture" +tags: + - MoE + - AI +docId: qftv72k0kzwiz8ddksbcl2aw +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +# Mixture of Experts (MoE) Architecture + +To effectively scale model parameters without significantly increasing computational demands, the MoE architecture has emerged as a viable solution. MoE leverages a set of specialized sub-models and a gating mechanism to dynamically select the appropriate "expert network" for a given input. This enables the model to allocate computational resources on demand — a concept known as **conditional computation**. + +MoE has been widely adopted in large language models (LLMs), enabling these models to achieve corresponding capability improvements as their parameter counts scale significantly. +For example, **Mixtral-8x7B** proposed by Mixtral AI activates only 13 billion parameters yet **outperforms or matches Llama-2-70B and GPT-3.5 on multiple benchmarks**. + +--- + +## Traditional MoE Architecture + +Since MoE was first introduced into the Transformer architecture, it has primarily served as a replacement module for the **Feed-Forward Network (FFN)**. Typically, each expert in a MoE layer directly replicates the FFN structure it replaces, with a Router trained to decide which expert should handle a given input. + +![](./MOE-intro.assets/img-20250920112106486.png) + +MoE is applied mainly to the FFN layer rather than the self-attention layer, for the following reasons: + +- **Attention layers**: lower sparsity, better suited for global interactions. +- **FFN layers**: higher sparsity, more domain-specific. + DS-MoE found that when using Wikitext as the task, only **20%** of FFN experts were activated, + while the attention layer activation rate was as high as **80%**. This high utilization rate indicates that the core communication mechanism of attention layers is incompatible with expert specialization. Conversely, FFN layers — with their sparse characteristics — have the full potential for multi-expert specialization. + +![](./MOE-intro.assets/img-20250920112106518.png) + +--- + +## Routing Mechanisms: Dense MoE vs Sparse MoE + +![](./MOE-intro.assets/img-20250920112106554.png) + +- **Dense MoE** + - The gate uses a **softmax** routing mechanism for input tokens, passing a certain weight to each expert. + - Advantage: training is stable. + - Disadvantage: all experts must be computed every time, leading to high computational cost. + +- **Sparse MoE** + - Uses a **Top-K** routing mechanism, activating only the K experts with the highest weights. + - Advantage: drastically reduces computation — this is the strategy used by current mainstream models (e.g., GShard, Switch Transformer, Mixtral, DeepSeek-MoE). + - Disadvantage: Router training becomes more complex, prone to the issue where "popular experts are used too often while underutilized experts learn nothing" — known as **routing collapse**. + - Solution: additional **load balancing loss** must be introduced during training. + +--- + +## Choosing the Number of Experts + +**GLaM (Google, 2021)** explored combinations of different expert counts and gating strategies: +It found that **64 experts (per layer) + Top-2 gating** achieves the best balance between performance and computational efficiency. +Top-2 gating significantly improves results and is more stable than a single expert. The 64-expert configuration also performs well in **zero-shot, one-shot, and few-shot** settings. Consequently, many subsequent MoE works (e.g., Mixtral, DBRX, DeepSeekMoE) have adopted a scale of ≤64 experts, making this design a practical reference. + +--- + +## MoE and PEFT + +There remains substantial interest in PEFT (Parameter-Efficient Fine-Tuning). +The paper [_Pushing Mixture of Experts to the Limit: Extremely Parameter Efficient MoE for Instruction Tuning_](https://arxiv.org/abs/2309.05444) was the first to propose **combining LoRA-type PEFT methods with the MoE framework**. +The core idea is to apply LoRA not to the entire large model, but specifically within the MoE expert modules. Since each MoE expert is an FFN (MLP) — the key location for knowledge storage — only a small set of LoRA experts are updated at a time, greatly enhancing scalability. + +![](./MOE-intro.assets/img-20250920112106588.png) + +The core idea of this method is to use **low-rank approximate updates** to avoid high-cost fine-tuning. + +1. **Input (Input → Embedding)** + - Input tokens (characters or subwords) first pass through an Embedding layer. + - This part is the same as in a standard Transformer. + +2. **Multi-Head Attention** + - Input embeddings enter the multi-head attention module. + - Q, K, V remain fully intact and are not modified by LoRAMoE. + - The output goes through **Add & Norm**, and the result is passed to the FFN. + +3. **FFN → MoE (Expert Routing)** + - The standard Transformer FFN is replaced by a **LoRA + MoE expert network**. + - The Router selects a subset of experts based on the input; each expert is a **LoRA-adapted (low-rank) module**, not a fully trainable FFN. + - Frozen parts (❄️) are the pre-trained backbone of the large model. + - Fire (🔥) represents the LoRA Adapter (trainable parameters, low-rank matrices). + - The weighted combination output by the Router: + + $$ + y = \sum_i \alpha_i \cdot Expert_i(x) + $$ + + where $\alpha_i$ is the weight computed by the Router for the given input. + +4. **Output (Add & Norm → Residual)** + - The expert-mixed output from the Router, combined with the residual connection, enters Add & Norm and continues to the next layer. + +--- + +#### LoRA Breakdown + +The core idea of LoRA (Low-Rank Adaptation): + +For a large linear layer weight $W \in \mathbb{R}^{d_{out} \times d_{in}}$, instead of training the entire matrix, a low-rank approximate update is added: + +$$ +W' = W + \Delta W, \quad \Delta W = BA +$$ + +- $A \in \mathbb{R}^{r \times d_{in}}, B \in \mathbb{R}^{d_{out} \times r}$ +- Rank $r \ll d_{in}, d_{out}$, typically a single digit to a few tens +- $W$: frozen (❄️, pre-trained parameters) +- $A, B$: trainable (🔥, significantly fewer parameters) + +Thus, when an input vector $x$ passes through the LoRA linear layer: + +$$ +Wx + BAx +$$ + +equals **the original backbone output + a small low-rank correction**. + +Returning to the diagram, each expert $Expert_i$ is not a brand new large FFN, but rather **a combination of LoRA adapters over an FFN**: + +$$ +Expert_i(x) = B_i A_i x +$$ + +- The Router computes a distribution $\alpha$ over the input hidden state, then applies a weighted combination: + +$$ +y = \sum_i \alpha_i \cdot Expert_i(x) +$$ + +- The final result is added to the backbone (frozen FFN weight output): + +$$ +y_{final} = W_{FFN}x + \sum_i \alpha_i \cdot B_i A_i x +$$ + +--- + +Author: **Yang Lewis** +Non-commercial reproduction must credit the source. +For commercial use, contact the author: **840691168ly@gmail.com** diff --git a/app/docs/ai/MoE/moe-update.en.md b/app/docs/ai/MoE/moe-update.en.md new file mode 100644 index 00000000..5f927c4c --- /dev/null +++ b/app/docs/ai/MoE/moe-update.en.md @@ -0,0 +1,231 @@ +--- +title: "Theory of MoE" +description: "" +date: "2025-10-05" +tags: + - tag-one +docId: db3qwg25h6l0bh8f2sdabdqc +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +# Theory of MoE + +## Basic Formula Definitions + +For a vector $w$, let $\|w\|_2$ and $\|w\|_\infty$ denote its $\ell_2$ norm and $\ell_\infty$ norm, respectively. + +Given positive constants $c_1, c_2$, we define: + +- $x = \Omega(y)$, if $x > c_2 |y|$; +- $x = \Theta(y)$, if $c_1 |y| \lt x \lt c_2 |y|$; +- $x = O(y)$, if $x \lt c_1 |y|$; +- $x = o(y)$, if $\dfrac{x}{y} \to 0$. + +Where: + +- $O(y)$: upper bound, meaning "grows no faster than $y$". +- $\Omega(y)$: lower bound, meaning "grows at least as fast as $y$". +- $\Theta(y)$: both upper and lower bounds are on the order of $y$, meaning "same order as $y$". +- $o(y)$: strictly much smaller than $y$, ultimately approaching $0$. + +## **Key Assumptions**: + +1. This paper aims only to derive closed-form forgetting formulas, so it simplifies directly to a linear model: $f(X)=X^{\top}w,\; w\in \mathbb{R}^d$ + +2. This paper only discusses task-wise routing methods. During data generation, each sample contains only one signal, with all other entries as Gaussian noise. This is again for model simplification. In practical engineering, tokens are implicitly routed to various experts rather than using manually specified routing. + +> ### Dataset Generation Rules +> +> At each training round $t \in [T]$, when a new task $n_t$ arrives, the dataset $\mathcal{D}_t = (X_t, y_t)$ is generated as follows: +> +> 1. **Sample the ground truth vector for the task** +> - Uniformly sample a ground truth vector $w_{n_t}$ from the task pool $\mathcal{W} = \{w_1, \dots, w_N\}$, and set $w_{n_t}$ as the ground truth for the current task. +> 2. **Generate scaling coefficient** +> - Independently sample a random variable $\beta_t \in (0, C)$, where $C = \mathcal{O}(1)$. +> 3. **Construct input feature matrix $X_t$** +> - Generate from $s_t$ samples: +> - **One sample** is defined as $\beta_t v_{n_t}$, where $v_{n_t}$ is the feature signal of task $n_t$. +> - The remaining $s_t - 1$ samples come from a Gaussian distribution: $\mathcal{N}(0, \sigma_t^2 I_d)$, where $\sigma_t \ge 0$ is the noise level. +> 4. **Generate output labels $y_t$** +> - Using linear regression: +> $$ +> y_t = X_t^\top w_{n_t} +> $$ +> +> **Result**: +> Dataset $\mathcal{D}_t = (X_t, y_t)$, corresponding to a linear regression task. + +3. This paper uses Top-1 expert selection only. + +## Formula Theory: + +Expert parameter update: +When the router selects a particular expert, all other experts remain unchanged; only the selected expert is updated, according to the following formula: + +$$ +w_t^{(m_t)} = w_{t-1}^{(m_t)} + X_t (X_t^\top X_t)^{-1}(y_t - X_t^\top w_{t-1}^{(m_t)}) +$$ + +> ### Derivation of the Expert Parameter Update Formula +> +> **Objective**: At round $t$, expert $m_t$ must fit the task dataset $(X_t, y_t)$ +> $$ \min\_{w}\ \|X_t^\top w - y_t\|\_2^2 $$ +> +> **Problem**: Under overparameterization ($s_t < d$), the solution is non-unique; directly computing the least-squares solution discards historical information. +> > Therefore, the paper reformulates it as a **constrained optimization**: +> +> $$ +> \min_w \ \|w - w_{t-1}^{(m_t)}\|_2^2 \quad +> s.t.\ \ X_t^\top w = y_t +> $$ +> +> **Solution**: Using Lagrange multipliers or residual projection, the update is: +> +> $$ +> w_t^{(m_t)} = w_{t-1}^{(m_t)} + X_t (X_t^\top X_t)^{-1}\,(y_t - X_t^\top w_{t-1}^{(m_t)}) +> $$ +> +> **Interpretation**: +> +> - $(y_t - X_t^\top w_{t-1})$ = **residual** = true output − old prediction +> - $X_t (X_t^\top X_t)^{-1}$ = the correction term that projects the residual back into parameter space +> - The entire expression = a least-squares correction near the old parameters +> +> **Properties**: +> +> - Guarantees $X_t^\top w_t = y_t$ → the new parameters perfectly fit the current task +> - Stays as close as possible to $w_{t-1}$ → minimizes catastrophic forgetting + +Auxiliary loss (also commonly referred to as load balance): + +$$ +L_t^{\text{aux}}(\Theta_t, \mathcal{D}_t) = \alpha \cdot M \cdot \sum_{m \in [M]} f_t^{(m)} \cdot P_t^{(m)} +$$ + +> ### Auxiliary Loss +> +> **Parameter explanation** +> +> - $\alpha$: weighting coefficient, controls the proportion of auxiliary loss in the total loss +> - $M$: number of experts +> - $f_t^{(m)}$: frequency with which expert $m$ has been selected in the first $t$ rounds (historical usage) +> - $P_t^{(m)}$: average routing probability assigned to expert $m$ by the router at round $t$ +> +> **Purpose** +> +> - Penalizes experts that have been frequently used historically and are still assigned high probability in the current round +> - Encourages the router to make greater use of underutilized experts +> - Achieves **load balancing** to prevent experts from being over- or under-used +> - The trailing term is intuitively clear: when an expert $m$ has been used many times historically and is still assigned large logits in the current round, this loss term becomes very large, suppressing the router's preference for a few experts and thus preventing routing collapse + +Locality loss: + +$$ +L_t^{\text{loc}}(\Theta_t, \mathcal{D}_t) = \sum_{m \in [M]} \pi_m(X_t,\Theta_t)\, \|w_t^{(m)} - w_{t-1}^{(m)}\|_2 +$$ + +> ### Locality Loss +> +> **Parameter explanation** +> +> - $\pi_m(X_t,\Theta_t)$: probability assigned to expert $m$ by the router (softmax output) +> - $w_t^{(m)}$: parameters of expert $m$ under the current task +> - $w_{t-1}^{(m)}$: parameters of expert $m$ from the previous round +> +> **Purpose** +> +> - Constrains expert parameter updates from deviating too far from historical values +> - Encourages similar tasks to be routed to the same expert, thereby reducing loss +> - Reduces forgetting (updates for new tasks do not completely overwrite old knowledge) +> - Improves expert **specialization**: each expert gradually stabilizes on a particular type of task + +Training loss: + +$$ +L_t^{\text{tr}}(w_t^{(m_t)}, \mathcal{D}_t) = \frac{1}{s_t}\,\|X_t^\top w_t^{(m_t)} - y_t\|_2^2 +$$ + +> ### Training Loss +> +> **Parameter explanation** +> +> - $s_t$: number of data samples for the current task +> - $X_t$: feature matrix +> - $y_t$: output label vector +> - $w_t^{(m_t)}$: parameters of the expert selected at round $t$ +> +> **Purpose** +> +> - Essentially the mean squared error (MSE) of least-squares regression +> - Makes the selected expert fit the current task data +> - Ensures the expert can capture the true signal (ground truth) of the task + +Total loss: + +$$ +L_t^{\text{task}} = L_t^{\text{tr}} + L_t^{\text{loc}} + L_t^{\text{aux}} +$$ + +With the above total loss function, router parameter updates can be performed during training. + +Router update formula: + +$$ +\theta_{t+1}^{(m)} = \theta_t^{(m)} - \eta \cdot \nabla_{\theta^{(m)}} L_t^{\text{task}}(\Theta_t, w_t^{(m_t)}, \mathcal{D}_t), \quad \forall m \in [M] +$$ + +### Tricks: + +#### Early Termination + +In continual learning (CL) scenarios, if the gating network continues to update indefinitely, the allocation probabilities across different experts may gradually converge as more tasks arrive, eventually causing **expert differentiation to collapse** and **routing errors**. To address this, an **Early Termination** mechanism must be introduced. + +- **Core Idea** + After sufficient rounds of task exploration ($T_1$ rounds), the expert assignments in MoE should gradually converge. Continuing to train the gating network at this point no longer yields benefits and instead leads to overfitting and blurring of task boundaries. Therefore, at an appropriate time, **updates to the router parameters $\Theta_t$ should be terminated** to maintain the stability of expert assignments. + +- **Convergence Criterion** + Define a convergence indicator $I^{(m)}$ to measure whether expert $m$ has converged: + + $I^{(m)} = \big| h_m(X_t, \theta_t) - h_{m_t}(X_t, \theta_t) \big|$ + + where $h_m(X_t,\theta_t)$ denotes the gating output of expert $m$ on the current input, and $h_{m_t}(X_t,\theta_t)$ denotes the output of the expert actually selected by the router. + - If this gap is **larger than threshold $\Gamma$**, expert $m$ has not yet converged and $\Theta_t$ should continue to be updated. + - If this gap is **smaller than threshold $\Gamma$**, the gating network is considered converged and updates to $\Theta_t$ are stopped. + - This prevents the router from continuing to update after convergence, which would otherwise destroy expert assignments. It also ensures that different experts stably serve their respective task clusters. Combined with the constraints of $L^{loc}$ and $L^{aux}$, the early termination mechanism enables the system to maintain balance and low forgetting in CL environments over the long term. + +#### Multiple Variants of Locality Loss + +- **Parameter Locality** + +$$ + L^{loc}_{param} = \sum_{m \in [M]} \pi_m(X_t,\Theta_t)\,\|w_t^{(m)} - w_{t-1}^{(m)}\|_2 +$$ + + - The method used in the preceding sections. + - Ensures that the parameter differences for the same expert across adjacent tasks are not too large. + +- **Representation Locality** — Constraints can be applied directly to the representations (hidden states) output by each expert. + + - For example: + + $$ + L^{loc}_{repr} = \sum_{m \in [M]} \pi_m(X_t,\Theta_t)\,\|f_m(X_t) - f_m(X_{t-1})\|_2 + $$ + + - Keeps similar inputs stable on the same expert. + +- **Routing Locality** — Constrains the router's assignment probabilities from jumping too drastically between tasks. + + - Of the form: + + $$ + L^{loc}_{route} = \sum_{m \in [M]} \|\pi_m(X_t,\Theta_t) - \pi_m(X_{t-1},\Theta_{t-1})\|_2 + $$ + +- **Task Embedding Locality** + - If task embeddings can be constructed (e.g., via meta-learning or contrastive learning), one can define: + - Similar tasks → routed to the same expert + - Dissimilar tasks → differentiated as much as possible diff --git a/app/docs/ai/Multi-agents-system-on-Code-Translation/code-translation-intro.en.mdx b/app/docs/ai/Multi-agents-system-on-Code-Translation/code-translation-intro.en.mdx new file mode 100644 index 00000000..59e3f5e8 --- /dev/null +++ b/app/docs/ai/Multi-agents-system-on-Code-Translation/code-translation-intro.en.mdx @@ -0,0 +1,24 @@ +--- +title: Essential Reading for Getting Started with Code Translation +description: "" +date: "2025-09-19" +tags: + - tag-one +docId: qaezsrj15sudk796r5otne36 +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +Recommended introductory papers: + +1. ExeCoder: Empowering Large Language Models with Executability Representation for Code Translation — Work from Microsoft, proposing some fairly creative ideas. +2. Repository-Level Compositional Code Translation and Validation — File-level approach (the longer the code, the harder the task). +3. CoTran: An LLM-based Code Translator using Reinforcement Learning with Feedback from Compiler and Symbolic Execution — A common method combining compiler feedback and RL, with a well-designed architecture. +4. Lost in Translation: A Study of Bugs Introduced by Large Language Models while Translating Code — A survey-like paper that helps you get up to speed quickly. +5. A Systematic Literature Review on Neural Code Translation — A comprehensive literature review; after reading it, you'll feel like you have a handle on everything. +6. IMPROVING COMPLEX REASONING WITH DYNAMIC PROMPT CORRUPTION: A SOFT PROMPT OPTIMIZATION APPROACH — A paper on prompt engineering; reading it may help you write better prompts. +7. Enhancing LLM-based Code Translation in Repository Context via Triple Knowledge-Augmented — Combines knowledge augmentation with code translation from an interesting angle, and also handles file-level processing. + +Feel free to share any great papers you find with everyone! diff --git a/app/docs/ai/agents-todo/agent-ecosystem.en.mdx b/app/docs/ai/agents-todo/agent-ecosystem.en.mdx new file mode 100644 index 00000000..0d7e635b --- /dev/null +++ b/app/docs/ai/agents-todo/agent-ecosystem.en.mdx @@ -0,0 +1,63 @@ +--- +title: Agent +description: "LLM Agents: CS294/194-196 course, ReAct, FireAct, and more" +status: todo +docId: ue27z7z95yzw3lhhfj7nit1c +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +This section aggregates courses and technical frameworks related to LLM agents. + +--- + +## OpenHands (formerly OpenDevin) + +- **Links**: + - [GitHub Repository](https://github.com/All-Hands-AI/OpenHands) + - [Paper arXiv:2407.16741](https://arxiv.org/pdf/2407.16741) +- **Overview**: + OpenHands is an open-source Agent platform capable of executing code / shell / web / API tasks, suited for developers' daily automation and toolchain integration. + +--- + +## Kimi-Researcher + +- **Links**: + - [Official Introduction](https://moonshotai.github.io/Kimi-Researcher/) + - [Zhihu Article](https://zhuanlan.zhihu.com/p/1921119537757140195) +- **Overview**: + A reinforcement learning (RL)-driven research Agent that can self-plan, perform multi-round retrieval and reasoning, for complex tasks such as technical research and report writing. + +--- + +## OpenAI Deep Research + +- **Links**: + - [Introducing Deep Research — OpenAI](https://openai.com/index/introducing-deep-research/) + - [ChatGPT Deep Research — Wikipedia](https://en.wikipedia.org/wiki/ChatGPT_Deep_Research) +- **Overview**: + Deep Research is an Agent mode launched by OpenAI in 2025, supporting multi-step internet research tasks with the ability to browse web pages, PDFs, and image materials to synthesize reports. Suited for professional research in fields such as science, policy, and engineering. + +--- + +## KAG: Knowledge Augmented Generation + +- **Links**: + - [arXiv:2409.13731 (KAG)](https://arxiv.org/abs/2409.13731) +- **Overview**: + KAG is an LLM enhancement framework for professional domains, designed to combine **knowledge graphs + vector retrieval** to address the limitations of RAG in specialized scenarios. It places particular emphasis on the integration of knowledge logic (such as numerical values, temporal information, and expert rules) to deliver more reliable performance on question-answering and reasoning tasks. + +--- + +## DeepResearchAgent / DeepResearcher & DR Agent Survey + +- **Links**: + - [Deep Research Agents: A Systematic Examination and Roadmap (arXiv)](https://arxiv.org/abs/2506.18096) + - [DeepResearcher: Scaling Deep Research via Reinforcement Learning (arXiv:2504.03160)](https://arxiv.org/abs/2504.03160) +- **Overview**: + These works study the architectural components, retrieval methods, tool usage, and planning processes for Agents performing deep research in real-world environments. + - The DR Survey provides a taxonomy of Agent systems, key challenges, and future directions. + - DeepResearcher trains Agents using RL + web interaction, outperforming traditional retrieval- or prompt-based methods on open-domain research tasks. diff --git a/app/docs/ai/agents-todo/cs294-194-196/index.en.mdx b/app/docs/ai/agents-todo/cs294-194-196/index.en.mdx new file mode 100644 index 00000000..0d89429c --- /dev/null +++ b/app/docs/ai/agents-todo/cs294-194-196/index.en.mdx @@ -0,0 +1,15 @@ +--- +title: CS294/194-196 Large Language Model Agents +description: Course materials for CS294/194-196 +docId: eo5rwumxkh7twfdvlp5po9rc +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +## Course Information + +- Official site: https://rdi.berkeley.edu/llm-agents/f24 +- Institution: UC Berkeley +- Highlights: Systematic introduction to the theory and practice of LLM agents diff --git a/app/docs/ai/ai-math-basics/calculus-optimization/index.en.mdx b/app/docs/ai/ai-math-basics/calculus-optimization/index.en.mdx new file mode 100644 index 00000000..4130a800 --- /dev/null +++ b/app/docs/ai/ai-math-basics/calculus-optimization/index.en.mdx @@ -0,0 +1,49 @@ +--- +title: Calculus & Optimization +description: Core concepts in calculus and optimization and their applications in large models +date: "2024-01-12" +tags: + - calculus + - optimization + - derivative + - gradient + - chain-rule + - convex-optimization + - backpropagation + - sgd + - adam + - rmsprop +docId: v8m8kdjzzx7uhiz69r5m3m9o +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +## Core Concepts + +- Derivative +- Partial derivative +- Gradient +- Chain rule +- Taylor expansion +- Lagrange multipliers +- Convex optimization + +## Applications in Large Models + +### Backpropagation + +- A perfect embodiment of gradient computation and the chain rule. + +### Model Training + +- The core of minimizing the loss function (an optimization problem); all optimizers (SGD, Adam, RMSProp) are variants of gradient descent. + +### Activation Functions + +- Their derivative properties are critical for gradient propagation. + +### Model Convergence Analysis + +- Involves convergence theory from calculus. diff --git a/app/docs/ai/ai-math-basics/information-theory/index.en.mdx b/app/docs/ai/ai-math-basics/information-theory/index.en.mdx new file mode 100644 index 00000000..ad6cd522 --- /dev/null +++ b/app/docs/ai/ai-math-basics/information-theory/index.en.mdx @@ -0,0 +1,47 @@ +--- +title: Information Theory +description: Core concepts in information theory and their applications in large models +date: "2024-01-13" +tags: + - information-theory + - entropy + - cross-entropy + - kl-divergence + - mutual-information + - rl + - model-compression +docId: gpoh50befguf7zgsetzkvbi3 +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +## Core Concepts + +- Information content +- Entropy +- Joint entropy +- Conditional entropy +- Mutual information +- Cross-entropy +- KL divergence (Kullback-Leibler Divergence) + +## Applications in Large Models + +### Loss Function + +- Cross-entropy loss is a measure of the difference between the predicted distribution and the true distribution. + +### Attention Mechanism + +- When computing attention weights, the softmax operation relates to probability distributions and entropy. + +### Reinforcement Learning + +- The optimization objective in policy gradient may include an entropy regularization term to encourage exploration. +- The core of TRPO / PPO algorithms is a KL divergence constraint. + +### Model Compression and Quantization + +- Evaluating quantization information loss. diff --git a/app/docs/ai/ai-math-basics/linear-algebra/index.en.mdx b/app/docs/ai/ai-math-basics/linear-algebra/index.en.mdx new file mode 100644 index 00000000..3e8a2f4f --- /dev/null +++ b/app/docs/ai/ai-math-basics/linear-algebra/index.en.mdx @@ -0,0 +1,54 @@ +--- +title: Linear Algebra +description: Core concepts in linear algebra and their applications in large models +date: "2024-01-10" +tags: + - linear-algebra + - matrix + - tensor + - embedding + - attention + - transformer + - pca + - svd +docId: l1kvojw2gvggxflrmzc7j7sm +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +## Core Concepts + +- Vector +- Matrix +- Tensor +- Eigenvalue / eigenvector +- SVD (Singular Value Decomposition) +- PCA (Principal Component Analysis) + +## Applications in Large Models + +### Embedding + +- Word vectors and Token embeddings are fundamentally high-dimensional vectors. + +### Attention Mechanism + +- QKV matrix multiplication +- Core computation in self-attention (dot product) + +### Transformer Architecture + +- Various layers (Linear Layer) +- Residual connections +- Feed-Forward Network + → All involve matrix operations + +### Model Parameters + +- The entire model's parameter count can be represented using matrices and tensors. + +### Dimensionality Reduction and Visualization + +- Reducing the dimensionality of embedding spaces (t-SNE, UMAP, PCA) for analysis. diff --git a/app/docs/ai/ai-math-basics/linear-algebra/resources/index.en.mdx b/app/docs/ai/ai-math-basics/linear-algebra/resources/index.en.mdx new file mode 100644 index 00000000..cac27ae8 --- /dev/null +++ b/app/docs/ai/ai-math-basics/linear-algebra/resources/index.en.mdx @@ -0,0 +1,20 @@ +--- +title: References +description: Linear algebra reference materials. +date: "2024-01-10" +tags: + - linear-algebra + - resources +docId: ba5lqs2zg1jqc30qzw3osm9v +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +- [Immersive Linear Algebra](https://textbooks.math.gatech.edu/ila/index2.html) +- _The Geometric Meaning of Linear Algebra_ (Ren Guangqian, Xie Cong, Hu Cuifang) — PDF +- [3Blue1Brown (YouTube Channel)](https://www.3blue1brown.com/) + - "Essence of Linear Algebra" video series + - "Essence of Calculus" video series + → Exceptional visualization that helps build strong geometric intuition. diff --git a/app/docs/ai/ai-math-basics/math-foundations.en.mdx b/app/docs/ai/ai-math-basics/math-foundations.en.mdx new file mode 100644 index 00000000..c4360865 --- /dev/null +++ b/app/docs/ai/ai-math-basics/math-foundations.en.mdx @@ -0,0 +1,89 @@ +--- +title: Mathematical Foundations for AI +description: "Core mathematics for AI: linear algebra, probability and statistics, calculus and optimization, information theory, numerical analysis" +date: "2025-01-27" +tags: + - mathematics + - linear-algebra + - probability + - calculus + - information-theory +docId: vcfer8dvlt80se4kmbnshx7x +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +AI and large models require a solid mathematical foundation. This section covers the core mathematical concepts needed for deep learning and large model development. + +## Core Mathematical Areas + +### 1. Linear Algebra + +**Core concepts**: vectors, matrices, tensors, eigenvalues/eigenvectors, SVD (Singular Value Decomposition), PCA (Principal Component Analysis) + +**Applications in large models**: + +- **Embedding**: word vectors and Token embeddings are fundamentally high-dimensional vectors +- **Attention Mechanism**: QKV matrix multiplication; core computation in self-attention (dot product) +- **Transformer architecture**: various layers (Linear Layer), residual connections, Feed-Forward Network — all involve matrix operations +- **Model parameters**: the entire model's parameter count can be represented using matrices and tensors +- **Dimensionality reduction and visualization**: reducing embedding spaces (t-SNE, UMAP, PCA) for analysis + +**References**: + +- [Immersive Linear Algebra](https://textbooks.math.gatech.edu/ila/index2.html) +- [3Blue1Brown - Essence of Linear Algebra](https://www.youtube.com/@3blue1brown) — exceptional visualization that helps build geometric intuition +- _The Geometric Meaning of Linear Algebra_ (Ren Guangqian, Xie Cong, Hu Cuifang) + +### 2. Probability and Statistics + +**Core concepts**: random variables, probability distributions (Gaussian, Bernoulli, multinomial), expectation, variance, covariance, conditional probability, Bayes' theorem, Maximum Likelihood Estimation (MLE), Maximum A Posteriori (MAP) + +**Applications in large models**: + +- **Language modeling**: P(next token | context) is conditional probability +- **Loss function**: cross-entropy loss originates from information theory and measures differences between probability distributions +- **Sampling and generation**: Top-k and Top-p (nucleus) sampling are both based on probability distributions +- **Uncertainty quantification**: confidence estimation for model predictions +- **Reinforcement learning**: optimization based on probabilistic policies + +### 3. Calculus and Optimization + +**Core concepts**: derivative, partial derivative, gradient, chain rule, Taylor expansion, Lagrange multipliers, convex optimization + +**Applications in large models**: + +- **Backpropagation**: a perfect embodiment of gradient computation and the chain rule +- **Model training**: the core of minimizing the loss function; all optimizers (SGD, Adam, RMSProp) are variants of gradient descent +- **Activation functions**: their derivative properties are critical for gradient propagation +- **Model convergence analysis**: involves convergence theory from calculus + +### 4. Information Theory + +**Core concepts**: information content, entropy, joint entropy, conditional entropy, mutual information, cross-entropy, KL divergence + +**Applications in large models**: + +- **Loss function**: cross-entropy loss measures the difference between predicted and true distributions +- **Attention mechanism**: the softmax operation relates to probability distributions and entropy when computing attention weights +- **Reinforcement learning**: entropy regularization terms in policy gradient objectives; KL divergence constraints in TRPO/PPO algorithms +- **Model compression and quantization**: evaluating quantization information loss + +### 5. Numerical Analysis + +**Core concepts**: floating-point precision, numerical stability, gradient clipping, learning rate scheduling + +**Applications in large models**: + +- **Preventing gradient explosion/vanishing**: large models are deep and computationally intensive, making numerical stability particularly critical +- **BFloat16/FP16 training**: understanding how different floating-point precisions affect model training +- **Optimizer selection**: some optimizers are numerically more stable + +## Study Recommendations + +1. **Combine theory with practice**: don't just derive formulas — understand how these mathematical concepts apply concretely in AI +2. **Build visual intuition**: use resources like 3Blue1Brown to develop geometric understanding +3. **Implement in code**: try implementing basic mathematical operations yourself to deepen understanding +4. **Build progressively**: start from foundational concepts and gradually move to advanced applications diff --git a/app/docs/ai/ai-math-basics/math_books.en.md b/app/docs/ai/ai-math-basics/math_books.en.md new file mode 100644 index 00000000..1d91e241 --- /dev/null +++ b/app/docs/ai/ai-math-basics/math_books.en.md @@ -0,0 +1,123 @@ +--- +title: Recommended Books on Mathematics and Data Science +description: "" +date: "2025-10-06" +tags: + - tag-one +docId: kzi6k1yg1sehlxidnxdsf59a +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +# Recommended Books on Mathematics and Data Science + +## Reader's Guide + +### A. Machine Learning / Data Science (Beginners – Engineers) + +- _Data with Dao, Programming Made Easy_ (数据有道，编程不难) +- Iris Book Series ③④⑤⑦ (Mathematical Elements / Power of Matrices / Statistics Simplified / Machine Learning) +- _Math with Bad Drawings_ +- _Numbers Don't Lie_ +- _The Ten Equations That Rule the World_ + +### B. Probability / Statistics / Bayesian (Researchers – Advanced Enthusiasts) + +- _Probability Theory: The Logic of Science_ (概率论沉思录) +- _The Mathematics of Selfishness_ (利己主义的数学解析) + +### C. Mathematical Foundations & General Knowledge (High School – Undergraduate & General Public) + +- _What Is Mathematics?_ (什么是数学) +- _Mathematics and Life_ 1 / 2 / 3 (数学与生活) +- _Strange Curves, Counting Rabbits…_ (怪曲线、数兔子…) + +### D. Competitions / Problem Sets / Reference Handbooks (Students – Teachers) + +- _Berkeley Problems in Mathematics_ (伯克利数学问题集) +- _Mathematical Handbook_ (数学手册) + +### E. History of Mathematics & Biographies & Philosophy (Those Interested in Cultural Background) + +- _From Mathematics to Philosophy_ (从数学到哲学) +- _Logical Dilemmas: The Life and Work of Kurt Gödel_ (哥德尔传) +- _A Mathematician's Apology_ (一个数学家的辩白) + +--- + +## 1. Iris Book Series + +- [Iris Book Series 01 · Programming Made Easy](https://space.bilibili.com/513194466?spm_id_from=333.337.search-card.all.click) + — **Dr Ginger (Jiang Weisheng)**. A Python introduction for those with no programming experience, covering variables, control flow, functions, and object-oriented programming. + +- _Iris Book Series · Data with Dao_ + Using Python as the main thread, this book introduces data acquisition, cleaning, visualization, and basic analysis workflows for complete beginners — an entry-level data science read. + +- _Iris Book Series · Mathematical Elements (Volume 3)_ + A quick refresher on the linear algebra, calculus, and probability and statistics needed for machine learning. + +- _Iris Book Series · Power of Matrices (Volume 4)_ + Focuses on linear algebra and matrix decomposition (SVD, eigenvalues, singular values) and their applications in ML. + +- _Iris Book Series · Statistics Simplified (Volume 5)_ + Uses intuitive examples to explain core statistical concepts including Bayes, hypothesis testing, and regression analysis. + +- _Iris Book Series · Machine Learning (Volume 7)_ + Threads through the iris dataset to explain common supervised and unsupervised learning algorithms and hands-on scikit-learn usage. + +- [Math with Bad Drawings](https://www.goodreads.com/book/show/36205393-math-with-bad-drawings) + — **Ben Orlin**. Explains functions, statistics, and game theory using cartoons with deliberately bad drawings — lighthearted and humorous. + +- [Numbers Don't Lie: 71 Things You Need to Know About the World](https://www.goodreads.com/book/show/50705179-numbers-don-t-lie) + Interprets global topics such as energy, population, and technology through the lens of data and orders of magnitude. + +- [The Ten Equations That Rule the World and How You Can Use Them](https://www.goodreads.com/book/show/55607293-the-ten-equations-that-rule-the-world) + Ten equations from linear regression to power laws, demonstrating their predictive power in finance, social networks, and sports. + +--- + +## 2. Competitions / Reference Books + +- [Berkeley Problems in Mathematics (3rd Edition)](https://book.douban.com/subject/2066460/) — P. N. Sousa et al. + A collection of classic problems from UC Berkeley entrance exams and competitions, focusing on analysis, algebra, and combinatorics; complete solutions included. + +- [Mathematical Handbook](https://book.douban.com/subject/20418732/) — Compiled by Sichuan Mining Institute + A formula reference book covering common algebra, trigonometry, calculus, and statistical tables. + +--- + +## 3. Probability / Bayesian + +- [Probability Theory: The Logic of Science](https://wap.sciencenet.cn/blog-1319915-1449152.html?mobile=1) — Edwin T. Jaynes + Advocates unifying probability theory using maximum entropy and a Bayesian perspective; often called the "Bayesian Bible." + +- [The Mathematics of Selfishness](https://book.douban.com/subject/27150468/) — Karl Sigmund + An introduction to evolutionary game theory, using the Prisoner's Dilemma to explain mathematical models of cooperation and altruism. + +--- + +## 4. Popular Science + +- [Strange Curves, Counting Rabbits, & Other Mathematical Explorations](https://book.douban.com/subject/6985480/) + A collection of popular mathematics essays for general audiences, covering fractals, the Fibonacci sequence, topological paradoxes, and more. + +- [What Is Mathematics?](https://book.douban.com/subject/10455982/) — Courant & Robbins (Classic New Edition) + A general mathematics culture book covering number theory, geometry, calculus, and the concept of infinity. + +- [Mathematics and Life](https://book.douban.com/subject/26148739/) 1 / 2 / 3 — Tohyama Hiraku + A classic Japanese popular science series that uses everyday scenarios to explain mathematical ideas, methods, and fascinations. + +--- + +## 5. History of Mathematics · Biographies · Philosophy + +- [A Mathematician's Apology](https://book.douban.com/subject/2135227/) — G. H. Hardy + A confession on mathematical aesthetics and the life of research, written with great literary quality. + +- [Logical Dilemmas: The Life and Work of Kurt Gödel](https://book.douban.com/subject/36073022/) — John Dawson + A systematic account of Gödel's life, academic trajectory, and the background and impact of the incompleteness theorems. + +- [From Mathematics to Philosophy](https://book.douban.com/subject/36532721/) — Hao Wang + Explores the relationships among formal logic, Gödel's incompleteness, mathematical foundations, and philosophical propositions. diff --git a/app/docs/ai/ai-math-basics/numerical-analysis/index.en.mdx b/app/docs/ai/ai-math-basics/numerical-analysis/index.en.mdx new file mode 100644 index 00000000..8dbb6b7e --- /dev/null +++ b/app/docs/ai/ai-math-basics/numerical-analysis/index.en.mdx @@ -0,0 +1,40 @@ +--- +title: Numerical Analysis +description: Core concepts in numerical analysis and their applications in large models +date: "2024-01-14" +tags: + - numerical-analysis + - floating-point + - stability + - gradient-clipping + - learning-rate + - bfloat16 + - fp16 + - optimizer +docId: ebgss2sa91drisxswsh6iu8x +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +## Core Concepts + +- Floating-point precision +- Numerical stability +- Gradient clipping +- Learning rate scheduling + +## Applications in Large Models + +### Preventing Gradient Explosion / Vanishing + +- Large models are deep and computationally intensive, making numerical stability particularly critical. + +### BFloat16 / FP16 Training + +- Understanding how different floating-point precisions affect model training. + +### Optimizer Selection + +- Some optimizers are numerically more stable. diff --git a/app/docs/ai/ai-math-basics/probability-statistics/index.en.mdx b/app/docs/ai/ai-math-basics/probability-statistics/index.en.mdx new file mode 100644 index 00000000..393a1d53 --- /dev/null +++ b/app/docs/ai/ai-math-basics/probability-statistics/index.en.mdx @@ -0,0 +1,58 @@ +--- +title: Probability and Statistics +description: Core concepts in probability and statistics and their applications in large models +date: "2024-01-11" +tags: + - probability + - statistics + - random-variable + - distribution + - bayes + - mle + - map + - hypothesis-testing + - confidence-interval + - cross-entropy + - rlhf +docId: d5fya0gd1w8vblv8qeqgnqtu +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +## Core Concepts + +- Random variable +- Probability distributions (Gaussian, Bernoulli, multinomial) +- Expectation +- Variance +- Covariance +- Conditional probability +- Bayes' theorem +- Maximum Likelihood Estimation (MLE) +- Maximum A Posteriori (MAP) +- Hypothesis testing +- Confidence interval + +## Applications in Large Models + +### Language Modeling + +- P(next token | context) is conditional probability. + +### Loss Function + +- Cross-entropy loss originates from information theory and measures differences between probability distributions. + +### Sampling and Generation + +- Top-k and Top-p (nucleus) sampling are both based on probability distributions. + +### Uncertainty Quantification + +- Confidence estimation for model predictions. + +### Reinforcement Learning (RLHF) + +- Optimization based on probabilistic policies. diff --git a/app/docs/ai/ai-math-basics/probability-statistics/resources/index.en.mdx b/app/docs/ai/ai-math-basics/probability-statistics/resources/index.en.mdx new file mode 100644 index 00000000..d667ea60 --- /dev/null +++ b/app/docs/ai/ai-math-basics/probability-statistics/resources/index.en.mdx @@ -0,0 +1,16 @@ +--- +title: References +description: Probability and statistics reference materials. +date: "2024-01-11" +tags: + - probability + - statistics + - resources +docId: q7kagbrpnek7b89axvssn4bo +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +- Fang Hao diff --git a/app/docs/ai/compute-platforms/model-compuational-resource-demand.en.md b/app/docs/ai/compute-platforms/model-compuational-resource-demand.en.md new file mode 100644 index 00000000..293659a3 --- /dev/null +++ b/app/docs/ai/compute-platforms/model-compuational-resource-demand.en.md @@ -0,0 +1,198 @@ +--- +title: Compute Requirements Guide +description: How to calculate the GPU memory required to train large models +date: "2025-09-20" +tags: + - compute-platforms +docId: ns7q5ehuje6oiua7as6rtnyf +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +# Notes on Multi-GPU Training of Large Models + +## 1. Unit Conventions + +1 GB = 1024 MB = 1024×1024 KB = 1024×1024×1024 Bytes = 1024×1024×1024×8 Bits + +**Parameter types and sizes:** + +| Parameter Type | Bytes | +| -------------- | ----- | +| FP32 | 4 | +| FP16 / BF16 | 2 | +| INT8 | 1 | +| INT4 | 0.5 | + +--- + +## 2. GPU Memory Calculation for Training Large Models + +Even with 80 GB of GPU memory, a single card cannot handle full training of models with billions of parameters. +Weights are only one part — gradients, optimizer states, and activations all consume GPU memory. + +Assuming the model has **N parameters** (e.g., 2B = 2 billion): + +1. **Weights W** + - Storage format: BF16 (2 bytes) + - Memory: $W = N \times 2$ bytes + - Example: 2B parameters → ≈ 4 GB + +2. **Gradients G** + - Storage format: BF16 (2 bytes) + - Memory: $G = N \times 2$ bytes + - Example: 2B parameters → ≈ 4 GB + +3. **Optimizer States (Adam)** + - Contains momentum V and squared gradient S, each in FP32 (4 bytes) + - Each ≈ 8 GB, total ≈ 16 GB + Note: optimizer memory also includes additional overhead (weight gradients may be copied many times during training) + 1. **Weights W** + - Model parameters stored in BF16/FP16 at 2 bytes. + + 2. **Gradients G** + - Temporary storage during backpropagation, BF16/FP16 at 2 bytes. + + 3. **Optimizer states** (vary significantly by optimizer): + - **SGD**: typically only needs the gradients themselves — **0 copies**. + - **SGDM (SGD with momentum)**: requires one momentum vector (FP32, 4 bytes). **One copy** + - **Adam/AdamW**: + - **First-order momentum (V)**: FP32 (4 bytes). + - **Second-order momentum (S)**: FP32 (4 bytes). + - So **2 state copies**. + + 4. **Master weights ($W^A$)** + - Common in mixed-precision training: although forward/backward passes use BF16, the optimizer update requires FP32 precision → so an additional FP32 copy of the weights is stored. + +4. **Activations** + - Depends on batch size, seq_len, and implementation details + - Rough estimate: ≈ 0.7–1.0 × weight size + +--- + +## 3. Formula Summary + +- **Standard Adam mode:** + + $W + G + W^A + V + S + 0.7W ≈ 24.8$ GB (using 2B parameters as an example) + +- **DeepSpeed ZeRO-3 mode:** + + $W + G + W^A + G^A + V + S + 0.7W ≈ 32.8$ GB + +Note: ZeRO-3 uses less memory but incurs higher communication and I/O overhead. + +--- + +## 4. Real-World Case: Mixtral-8×7B + +### Setup and Constants + +- Architecture: `d_model ≈ 4096`, `ffn_dim ≈ 14336`, **8 experts** per layer, **32 layers**, SwiGLU (gate/up/down three linear layers). +- Parameters per single expert: + $4096×14336×2 + 14336×4096 = 176,160,768$ ≈ **1.76×10^8** + → BF16 weights ≈ **352 MB/expert** +- 8 experts per layer ≈ **2.82 GB/layer** +- 32 layers total ≈ **90 GB (expert weights only)** + → Full-expert full-parameter training is impossible on **44 GB GPU memory**. + +--- + +### Case A: Router-only + +- Router parameters: `d_model × n_experts ≈ 4k × 8 = 32k` +- All 32 layers ≈ **millions of parameters** +- Extremely low overhead (MB level); memory is mainly consumed by **activations** +- Fully feasible on 44 GB, but improvement is limited + +--- + +### Case B: Partial Layers × Partial Experts + +Example: train only the bottom **6 layers**, **2 experts** per layer, plus the router. + +- Trainable parameter count: + $6 × 2 × 176,160,768 = 2.114B$ +- Weights (BF16): ≈ 4.23 GB +- Gradients (BF16): ≈ 4.23 GB +- Adam states (V+S, FP32): ≈ 16.9 GB +- Master weights (FP32): ≈ 8.46 GB +- **Total (persistent memory + gradients)**: ≈ 33.8 GB +- Adding frozen weight footprint and activation overhead, a 44 GB card requires: + - `batch=1–2` + - `seq_len ≤ 1024` + - `use_cache=False` + - `gradient_checkpointing=True` +- Feasible, but requires strict control. + +--- + +### Case C: 4-bit Full Model + LoRA + +Attach LoRA (r=16) to experts / router. + +- LoRA parameters per expert: + $r × (4096+14336 + 4096+14336 + 14336+4096) = r × 55296$ + → r=16 → 0.885M/expert +- 8 experts per layer: 7.08M +- 32 layers: 226.6M LoRA parameters +- Memory breakdown: + - Weights ≈ 0.45 GB + - Gradients ≈ 0.45 GB + - Adam + Master ≈ 2.72 GB + - Total ≈ 3.6 GB +- Well within the 44 GB memory budget + +--- + +## 5. Parallelism Strategies + +### Data Parallelism (DP) + +- Each GPU holds a full model copy, processes different batches, gradients are aggregated +- Advantage: simple +- Disadvantage: high memory redundancy + +### Distributed Data Parallelism (DDP) + +- One process per GPU, gradients synchronized in buckets +- Advantage: mainstream, stable +- Disadvantage: still requires full model on each GPU + +### ZeRO Optimization (DeepSpeed) + +- ZeRO-1: shard optimizer states +- ZeRO-2: also shard gradients +- ZeRO-3: shard parameters as well +- Advantage: memory-efficient +- Disadvantage: complex communication + +### Model Parallelism + +- **Tensor Parallelism (TP)**: split matrices across devices +- **Pipeline Parallelism (PP)**: split layers, like an assembly line +- **MoE Parallelism**: experts distributed across different devices, tokens activate a subset of experts + +--- + +## 6. Lessons Learned + +- Memory fragmentation: + `PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True` + (must be set before `import torch`) +- Communication library: + NCCL > Gloo > MPI (unless in special environments) +- DDP: must synchronize random seed +- Evaluation: + - eval_mb_size can be larger + - small training batch + gradient accumulation + - disable `model.config.use_cache` +- device = "auto": HuggingFace will automatically distribute different parts of the model based on your GPU memory. For large models like 7B on a single 44 GB card: attention + embedding + some FFN layers typically go on GPU, while frozen modules or inactive MoE experts can offload to CPU. This is great for inference but **requires caution during training** — parameters that need gradients must reside on GPU at all times, otherwise each forward/backward pass involves moving parameters back and forth, causing catastrophic communication overhead. Therefore, **device_map=auto is not always safe for training** as it may place trainable layers on CPU, leading to slow or non-functional training. + +--- + +Author: **Yang Lewis** +Non-commercial reproduction must credit the source. +For commercial use, contact the author: **840691168ly@gmail.com** diff --git a/app/docs/ai/foundation-models/datasets/index.en.mdx b/app/docs/ai/foundation-models/datasets/index.en.mdx new file mode 100644 index 00000000..e7f7cf59 --- /dev/null +++ b/app/docs/ai/foundation-models/datasets/index.en.mdx @@ -0,0 +1,327 @@ +--- +title: Dataset Construction +description: "Large model dataset construction: data sources, processing pipelines, quality control" +date: "2025-01-27" +tags: + - dataset-construction + - data-processing + - data-quality + - common-crawl + - data-cleaning +docId: egpawb1yui58yprrsgxn9qj2 +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +Data is the foundation of large models, and high-quality datasets directly impact model performance. This section provides a detailed introduction to methods and techniques for constructing large model datasets. + +## Data Sources + +### Web Data + +- **Common Crawl**: large-scale web crawl data + - Covers billions of web pages worldwide + - Rich multilingual content + - Regularly updated data snapshots + +- **Wikipedia**: high-quality encyclopedia data + - Available in multiple languages + - Structured knowledge content + - Continuously updated and maintained + +### Specialized Data + +- **Book corpora**: high-quality text data + - Project Gutenberg open-source books + - Academic publications + - Technical documentation and manuals + +- **Code data**: repositories such as GitHub + - Open-source project code + - Multiple programming languages + - Code comments and documentation + +- **Academic papers**: sources such as arXiv and PubMed + - Latest research findings + - Specialized domain knowledge + - Citation networks + +## Data Processing Pipeline + +### 1. Data Cleaning + +**Text quality filtering**: + +- Remove low-quality content (garbled text, duplicate content) +- Language detection and filtering +- Format standardization +- Encoding normalization + +**Content filtering**: + +- Remove advertisements and spam +- Filter harmful and inappropriate content +- Remove privacy-sensitive information +- Copyright content identification + +### 2. Format Standardization + +**Text normalization**: + +- Unify encoding format (UTF-8) +- Standardize punctuation +- Handle special characters +- Paragraph and line-break conventions + +**Structured processing**: + +- Extract main body text +- Remove HTML tags +- Preserve meaningful formatting information +- Unify document structure + +### 3. Deduplication + +**Exact deduplication**: + +- MD5 hash matching +- Identification of perfectly identical content +- Batch deduplication processing + +**Fuzzy deduplication**: + +- MinHash algorithm +- Similarity threshold settings +- Near-duplicate detection +- SimHash fingerprint matching + +**Cross-document deduplication**: + +- Paragraph-level deduplication +- Sentence-level deduplication +- n-gram overlap detection + +### 4. Quality Filtering + +**Statistical metric filtering**: + +- Document length limits +- Vocabulary richness checks +- Linguistic complexity assessment +- Punctuation ratio + +**Language model scoring**: + +- Perplexity evaluation +- Language model scoring +- Readability assessment +- Grammar correctness checking + +### 5. Privacy Protection + +**Personally Identifiable Information (PII) detection**: + +- Email address detection +- Phone number identification +- ID number filtering +- Address information handling + +**Data de-identification**: + +- Sensitive information substitution +- Anonymization processing +- Differential privacy techniques +- Encrypted data storage + +## Data Quality Control + +### Quality Assessment Metrics + +**Content quality**: + +- Information accuracy +- Logical coherence +- Linguistic fluency +- Knowledge depth + +**Diversity metrics**: + +- Topic coverage range +- Linguistic style diversity +- Source diversity +- Temporal span coverage + +**Balance considerations**: + +- Language distribution balance +- Domain knowledge balance +- Viewpoint and stance balance +- Cultural background diversity + +### Quality Assurance Process + +**Automated checks**: + +- Batch quality assessment +- Anomaly detection algorithms +- Statistical analysis reports +- Quality trend monitoring + +**Manual review**: + +- Random sampling inspection +- Expert domain review +- Annotation quality control +- Feedback loop mechanism + +## Special Data Processing + +### Multilingual Data + +**Language detection**: + +- Automatic language identification +- Mixed multilingual processing +- Dialect and variant identification +- Code-switching handling + +**Cross-lingual alignment**: + +- Parallel corpus construction +- Translation quality assessment +- Cultural adaptation + +### Multimodal Data + +**Image-text alignment**: + +- Image-text pairing +- Caption accuracy verification +- Visual content understanding +- Multimodal consistency + +**Structured data**: + +- Tabular data processing +- Knowledge graph integration +- Database content extraction + +## Data Pipeline Technologies + +### Distributed Processing + +**Big data frameworks**: + +- Apache Spark processing +- Hadoop ecosystem +- Distributed storage (HDFS) +- Streaming data processing + +**Parallelization strategies**: + +- Data sharding +- Task scheduling optimization +- Dynamic resource allocation +- Fault recovery mechanisms + +### Data Version Management + +**Version control**: + +- Dataset version tracking +- Change log management +- Rollback mechanism design +- Incremental update support + +**Metadata management**: + +- Data source information logging +- Processing pipeline tracking +- Quality metric monitoring +- Usage statistics analysis + +## Compliance Considerations + +### Laws and Regulations + +**Data compliance**: + +- GDPR privacy protection +- Copyright law requirements +- Regional regulatory compliance +- Industry standard alignment + +**Usage licenses**: + +- Understanding open-source licenses +- Commercial use restrictions +- Derivative work rules +- Attribution requirements + +### Ethical Considerations + +**Bias and fairness**: + +- Data bias identification +- Representativeness analysis +- Fairness evaluation metrics +- Bias mitigation strategies + +**Social impact**: + +- Content values review +- Cultural sensitivity considerations +- Social responsibility +- Negative impact assessment + +## Best Practices + +### Data Management + +1. **Establish clear data standards** +2. **Implement automated quality checks** +3. **Maintain data processing transparency** +4. **Regularly update and maintain datasets** +5. **Maintain comprehensive documentation** + +### Recommended Tools + +**Data processing tools**: + +- pandas: Python data processing +- Apache Beam: batch and stream processing +- Dask: parallel computing framework +- Ray: distributed computing platform + +**Quality checking tools**: + +- Great Expectations: data quality framework +- Apache Griffin: data quality monitoring +- Deequ: data quality testing + +## Future Trends + +1. **Increased automation**: smarter data processing pipelines +2. **Real-time data integration**: dynamic data updates and integration +3. **Privacy-preserving techniques**: federated learning and differential privacy +4. **Multimodal fusion**: more complex multimodal data processing +5. **Personalized data**: customized datasets for specific tasks + +## Study Recommendations + +1. **Theory foundation**: master data science and statistics fundamentals +2. **Engineering skills**: proficiency with big data processing tools +3. **Quality awareness**: develop sensitivity to data quality +4. **Compliance awareness**: understand relevant laws and regulations +5. **Practical experience**: participate in real dataset construction projects + +## From UNSW IT-AI Involution Hell Documentation + +- https://huggingface.co/ +- AK https://hf.co/akhaliq +- https://www.modelscope.cn/home +- https://www.kaggle.com/datasets +- UCI Machine Learning Repository: https://archive.ics.uci.edu/ml/index.php +- ImageNet diff --git a/app/docs/ai/foundation-models/deploy-infer/index.en.mdx b/app/docs/ai/foundation-models/deploy-infer/index.en.mdx new file mode 100644 index 00000000..fcd791f4 --- /dev/null +++ b/app/docs/ai/foundation-models/deploy-infer/index.en.mdx @@ -0,0 +1,443 @@ +--- +title: Deployment and Inference +description: "Large model deployment and inference optimization: KV Cache, Flash Attention, quantization, inference frameworks" +date: "2025-01-27" +tags: + - model-deployment + - inference-optimization + - kv-cache + - flash-attention + - quantization + - vllm +docId: z157s85hnz1y37tr28y2a8h2 +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +Deployment and inference of large models is the critical step of putting trained models into production. It involves inference optimization, deployment frameworks, service architecture, and more. + +## Inference Optimization Techniques + +### KV Cache + +**Core principle**: cache key-value pairs to avoid redundant computation and accelerate generation. + +**Implementation**: + +- Store Keys and Values from historical sequences +- New tokens only need to compute the current Query +- Significantly reduces computational complexity +- From O(n²·d) down to O(n·d) + +**Memory management**: + +- Dynamic memory allocation +- Batch processing optimization +- Memory defragmentation +- OOM prevention mechanisms + +### Flash Attention + +**Technical characteristics**: a memory-efficient attention computation algorithm + +**Core optimizations**: + +- Tiled computation strategy +- Memory access optimization +- Reduced IO complexity +- Numerical stability guarantees + +**Performance improvements**: + +- Reduced memory usage +- Faster computation +- Support for longer sequences +- Hardware-friendly design + +### Quantization + +**Quantization methods**: + +- **INT8 quantization**: 8-bit integer representation +- **INT4 quantization**: 4-bit integer representation +- **Mixed precision**: different precision for different layers +- **Dynamic quantization**: runtime quantization + +**Quantization strategies**: + +- Weight quantization +- Activation quantization +- KV Cache quantization +- Gradient quantization + +**Tooling**: + +- PyTorch quantization +- TensorRT quantization +- ONNX quantization +- Custom quantization kernels + +### Parallel Inference + +**Model parallelism**: + +- Tensor parallelism: intra-layer parameter splitting +- Pipeline parallelism: inter-layer pipelining +- Expert parallelism: MoE model expert distribution +- Hybrid parallelism: combining multiple strategies + +**Data parallelism**: + +- Batch parallelism +- Sequence parallelism +- Dynamic batching +- Continuous batching + +## Deployment Frameworks + +### vLLM + +**Highlights**: high-throughput inference engine + +- **PagedAttention**: efficient memory management +- **Continuous batching**: dynamic batch optimization +- **Streaming output**: real-time response support +- **Multi-GPU support**: distributed inference for large models + +**Core technologies**: + +- Memory pool management +- Request scheduling optimization +- KV Cache sharing +- Inference concurrency control + +### TensorRT-LLM + +**Highlights**: NVIDIA-optimized inference framework + +- **Deep optimization**: optimized for NVIDIA GPUs +- **Operator fusion**: automatic operator fusion +- **Multi-precision**: supports FP16/INT8/INT4 +- **Plugin ecosystem**: rich plugin support + +**Optimization techniques**: + +- Graph optimization +- Memory optimization +- Kernel fusion +- Dynamic shape support + +### Text Generation Inference (TGI) + +**Highlights**: HuggingFace inference service + +- **Ease of use**: simple deployment and usage +- **Model support**: broad model compatibility +- **API standard**: standardized API interface +- **Monitoring**: built-in monitoring and logging + +**Features**: + +- Automatic batching +- Streaming responses +- Safety filtering +- Load balancing + +### FastChat + +**Highlights**: chat model deployment framework + +- **Multi-model**: supports various chat models +- **Web interface**: user-friendly UI +- **API service**: RESTful API support +- **Distributed**: multi-node deployment support + +## Service Architecture Design + +### Inference Service Architecture + +**Component design**: + +- Model loader +- Request handler +- Batch scheduler +- Response generator +- Monitoring component + +**Performance optimization**: + +- Asynchronous processing +- Connection pool management +- Caching strategies +- Resource scheduling + +### Load Balancing + +**Strategies**: + +- Round-robin scheduling +- Least connections +- Weighted distribution +- Health checks + +**Implementation**: + +- Nginx load balancing +- HAProxy configuration +- Kubernetes Service +- Custom load balancers + +### Scaling Strategies + +**Horizontal scaling**: + +- Instance count adjustment +- Dynamic auto-scaling +- Resource monitoring triggers +- Warm-up mechanisms + +**Vertical scaling**: + +- Resource specification adjustment +- GPU memory expansion +- CPU core increases +- Storage capacity expansion + +## Memory Optimization + +### Memory Management Strategies + +**KV Cache optimization**: + +- Paged storage +- Memory sharing +- Garbage collection +- Defragmentation + +**Model weight optimization**: + +- Weight sharing +- Lazy loading +- Memory mapping +- Compressed storage + +### Memory Monitoring + +**Monitoring metrics**: + +- Memory utilization +- OOM frequency +- Memory fragmentation rate +- GC time statistics + +**Alerting mechanisms**: + +- Threshold alerts +- Trend warnings +- Automated handling +- Failover + +## Inference Performance Optimization + +### Latency Optimization + +**Latency reduction strategies**: + +- Model warm-up +- Batch processing optimization +- Operator fusion +- Hardware acceleration + +**Time to First Token (TTFT)**: + +- Prefill optimization +- Memory pre-allocation +- Model pre-loading +- Cache warm-up + +### Throughput Optimization + +**Increasing throughput**: + +- Batch size tuning +- Concurrent request handling +- Pipeline processing +- Resource utilization improvement + +**Continuous batching**: + +- Dynamic batch adjustment +- Request priority management +- Latency sensitivity tuning +- Fairness guarantees + +### Cost Optimization + +**Compute costs**: + +- Maximize GPU utilization +- Mixed instance usage +- On-demand scaling +- Spot instance usage + +**Storage costs**: + +- Model compression +- Hot/cold data separation +- Cache strategy optimization +- Data lifecycle management + +## Quality Assurance + +### Model Validation + +**Functional testing**: + +- Output quality validation +- Boundary condition testing +- Stress testing +- Regression testing + +**Performance testing**: + +- Latency benchmarking +- Throughput testing +- Concurrency capacity testing +- Stability testing + +### Monitoring System + +**Core metrics**: + +- QPS (queries per second) +- Average response time +- P99 latency +- Error rate +- Resource utilization + +**Monitoring tools**: + +- Prometheus monitoring +- Grafana visualization +- Custom monitoring +- Alert systems + +### A/B Testing + +**Test design**: + +- Traffic splitting +- Metric comparison +- Statistical significance +- Effect evaluation + +**Implementation approaches**: + +- Canary releases +- Blue-green deployments +- Shadow testing +- Progressive rollout + +## Security and Compliance + +### Security Protection + +**Input validation**: + +- Content filtering +- Length limits +- Format checks +- Malicious input detection + +**Output control**: + +- Content moderation +- Sensitive information filtering +- Copyright protection +- Harmful content blocking + +### Privacy Protection + +**Data protection**: + +- Request log desensitization +- User information anonymization +- Encrypted data transmission +- Storage encryption + +**Compliance requirements**: + +- GDPR compliance +- Data localization +- Audit logs +- Access control + +## Fault Handling + +### Common Issues + +**Performance issues**: + +- Out-of-Memory (OOM) +- Low GPU utilization +- Latency spikes +- Throughput drops + +**Stability issues**: + +- Service crashes +- Memory leaks +- Network timeouts +- Model anomalies + +### Recovery Strategies + +**Automatic recovery**: + +- Health checks +- Auto-restart +- Failover +- Service degradation + +**Monitoring and alerting**: + +- Real-time monitoring +- Early warning mechanisms +- Automated handling +- Manual intervention + +## Best Practices + +### Deployment Recommendations + +1. **Incremental deployment**: start small and scale gradually +2. **Performance baselines**: establish performance benchmarks and monitoring +3. **Resource planning**: plan compute and storage resources appropriately +4. **Security first**: prioritize security and privacy protection +5. **Complete documentation**: maintain comprehensive deployment documentation + +### Operations Strategies + +1. **Automated operations**: automate as much of the operations pipeline as possible +2. **Monitoring and alerting**: build a comprehensive monitoring and alerting system +3. **Backup and recovery**: establish data backup and recovery strategies +4. **Version management**: standardize the version release process +5. **Incident response**: develop detailed incident handling procedures + +## Future Trends + +1. **Hardware co-design**: deep software-hardware co-optimization +2. **Edge deployment**: model deployment on edge computing devices +3. **Federated inference**: distributed privacy-preserving inference +4. **Adaptive optimization**: intelligent adaptive inference optimization +5. **Green computing**: low-power, environmentally friendly inference techniques + +## Study Recommendations + +1. **Systematic learning**: comprehensive understanding of the inference optimization stack +2. **Hands-on practice**: deploy and optimize inference services yourself +3. **Performance tuning**: deep dive into performance tuning techniques +4. **Framework proficiency**: become proficient with mainstream inference frameworks +5. **Stay current**: track the latest developments in optimization techniques diff --git a/app/docs/ai/foundation-models/evaluation/index.en.mdx b/app/docs/ai/foundation-models/evaluation/index.en.mdx new file mode 100644 index 00000000..42b42c17 --- /dev/null +++ b/app/docs/ai/foundation-models/evaluation/index.en.mdx @@ -0,0 +1,287 @@ +--- +title: Model Evaluation +description: "Large model evaluation system: benchmarks, evaluation metrics, Chinese and English evaluation standards" +date: "2025-01-27" +tags: + - model-evaluation + - benchmark + - mmlu + - c-eval + - evaluation-metrics +docId: lndxpf7luoeqwwde4in23xr1 +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +Model evaluation is an essential means of measuring large model performance and capabilities, providing a scientific basis for model improvement and application selection. + +## Benchmark Evaluation System + +### General Capability Benchmarks + +#### MMLU (Massive Multitask Language Understanding) + +- **Scope**: multitask language understanding +- **Question count**: 15,908 multiple-choice questions +- **Subjects**: 57 disciplines from mathematics to history +- **Difficulty range**: from high school to professional level +- **Metric**: accuracy + +#### HellaSwag + +- **Goal**: commonsense reasoning ability +- **Task type**: sentence completion +- **Data source**: real-world scenario descriptions +- **Metric**: accuracy + +#### ARC (AI2 Reasoning Challenge) + +- **Focus**: scientific reasoning ability +- **Question type**: elementary school science multiple-choice +- **Difficulty levels**: Easy and Challenge +- **Characteristic**: requires multi-step reasoning + +#### GSM8K + +- **Scope**: mathematical problem solving +- **Question type**: elementary school math word problems +- **Answer format**: numerical answers +- **Focus**: mathematical reasoning chains + +### Chinese Evaluation Benchmarks + +#### C-Eval + +- **Goal**: comprehensive Chinese evaluation +- **Question count**: 13,948 questions +- **Subject coverage**: 52 subject areas +- **Difficulty range**: from middle school to professional level +- **Characteristic**: aligned with the Chinese education system + +#### CMMLU (Chinese Massive Multitask Language Understanding) + +- **Scope**: Chinese multitask evaluation +- **Question sources**: Chinese exams and textbooks +- **Subject categories**: humanities, social science, STEM, medicine, etc. +- **Metric**: multi-dimensional assessment + +#### AGIEval + +- **Characteristic**: human exam evaluation +- **Data source**: real exam questions +- **Exam types**: college entrance exam, civil service exam, bar exam, etc. +- **Value**: direct comparison with human performance + +### Domain-Specific Benchmarks + +#### HumanEval + +- **Goal**: code generation ability +- **Task type**: function implementation +- **Programming language**: primarily Python +- **Evaluation method**: unit test pass rate + +#### MATH + +- **Scope**: math competition problems +- **Difficulty level**: high school math competition level +- **Question types**: proofs, calculations +- **Evaluation method**: answer correctness + +#### BBH (Big-Bench Hard) + +- **Characteristic**: LLM challenge benchmark +- **Task source**: hard subset of Big-Bench +- **Focus**: reasoning and comprehension +- **Characteristic**: challenging for large models + +## Evaluation Methodology + +### Evaluation Design Principles + +1. **Comprehensiveness**: covers multiple model capabilities +2. **Objectivity**: avoids subjective bias and preference +3. **Reproducibility**: results can be reproduced and verified +4. **Fairness**: fair comparison across different models +5. **Practicality**: relevant to real-world application scenarios + +### Evaluation Dimensions + +#### Knowledge Ability + +- **Factual knowledge**: mastery of foundational facts +- **Conceptual understanding**: understanding of abstract concepts +- **Knowledge reasoning**: reasoning based on knowledge +- **Knowledge currency**: awareness of recent developments + +#### Reasoning Ability + +- **Logical reasoning**: deductive and inductive reasoning +- **Mathematical reasoning**: numerical computation and proofs +- **Commonsense reasoning**: everyday common sense +- **Causal reasoning**: understanding causal relationships + +#### Language Ability + +- **Language comprehension**: text understanding and parsing +- **Language generation**: fluent and accurate generation +- **Multilingual**: cross-lingual capabilities +- **Style adaptation**: adapting to different writing styles + +#### Safety Evaluation + +- **Harmful content**: avoidance of harmful content generation +- **Bias detection**: social bias identification +- **Privacy protection**: handling of private information +- **Adversarial robustness**: resistance to adversarial attacks + +## Evaluation Implementation + +### Evaluation Process + +1. **Benchmark selection**: choose appropriate benchmarks based on evaluation goals +2. **Environment setup**: configure evaluation environment and dependencies +3. **Model preparation**: load and configure the model under evaluation +4. **Execute evaluation**: run evaluation scripts and programs +5. **Result analysis**: compile and analyze evaluation results + +### Evaluation Frameworks + +#### OpenCompass + +- **Characteristic**: open-source evaluation framework +- **Support**: multiple models and benchmarks +- **Features**: automated evaluation pipeline +- **Visualization**: result display and comparison + +#### lm-evaluation-harness + +- **Source**: open-sourced by EleutherAI +- **Characteristic**: standardized evaluation interface +- **Support**: wide range of evaluation tasks +- **Ease of use**: simple command-line interface + +#### FlagEval + +- **Source**: BAAI (Beijing Academy of AI) +- **Characteristic**: friendly to Chinese evaluation +- **Coverage**: comprehensive evaluation dimensions +- **Standards**: rigorous scientific evaluation standards + +### Evaluation Environment + +#### Hardware Requirements + +- **GPU**: select based on model size +- **Memory**: sufficient system RAM +- **Storage**: fast SSD storage +- **Network**: stable network connection + +#### Software Environment + +- **Python**: primary programming language +- **PyTorch/TensorFlow**: deep learning frameworks +- **transformers**: model loading library +- **Evaluation tools**: specific evaluation frameworks + +## Result Analysis + +### Performance Metrics + +#### Accuracy Metrics + +- **Accuracy**: overall accuracy rate +- **Top-k accuracy**: accuracy within the top-k predictions +- **F1 score**: harmonic mean of precision and recall +- **BLEU/ROUGE**: text generation quality + +#### Efficiency Metrics + +- **Inference speed**: token generation speed +- **Memory usage**: memory footprint during inference +- **Energy consumption**: inference energy statistics +- **Cost-effectiveness**: performance-to-cost ratio + +### Comparative Analysis + +#### Model Comparison + +- **Same-scale models**: comparison of models with similar parameter counts +- **Different architectures**: comparison across different architectures +- **Development trends**: trends in model capability development +- **Pros and cons analysis**: strengths and weaknesses of each model + +#### Capability Analysis + +- **Strength identification**: task areas where the model excels +- **Weakness analysis**: areas where the model falls short +- **Improvement directions**: model optimization suggestions +- **Application recommendations**: suitable use-case scenarios + +### Visualization + +#### Radar Charts + +- Multi-dimensional capability display +- Comparison across different models +- Intuitive capability distribution +- Balance analysis + +#### Heatmaps + +- Fine-grained performance display +- Task dimension analysis +- Performance difference visualization +- Pattern recognition + +## Evaluation Challenges + +### Technical Challenges + +1. **Evaluation cost**: large model evaluation consumes significant resources +2. **Benchmark limitations**: existing benchmarks may lack comprehensiveness +3. **Benchmark contamination detection**: preventing models from gaming benchmarks +4. **Dynamic updates**: benchmarks require continuous updates + +### Methodological Challenges + +1. **Evaluation bias**: benchmarks themselves may contain biases +2. **Cultural differences**: fairness in cross-cultural evaluation +3. **Capability definition**: how to scientifically define and measure capabilities +4. **Ecosystem effects**: the influence of evaluation on model development + +## Future Developments + +### Evaluation Innovation + +1. **Dynamic evaluation**: real-time updated evaluation benchmarks +2. **Interactive evaluation**: multi-turn interactive evaluation modes +3. **Human-AI collaboration**: evaluation involving human experts +4. **Automation**: smarter automated evaluation systems + +### Evaluation Standards + +1. **International standards**: establishing internationally recognized evaluation standards +2. **Industry norms**: developing industry-wide evaluation specifications +3. **Certification systems**: establishing model capability certification +4. **Regulatory alignment**: aligning with regulatory requirements + +## Best Practices + +### Evaluation Strategy + +1. **Multi-dimensional evaluation**: assess comprehensively from multiple dimensions +2. **Benchmark combination**: use multiple benchmarks for cross-validation +3. **Regular evaluation**: establish periodic evaluation mechanisms +4. **Result verification**: multiple evaluation rounds to ensure reliability +5. **Transparency**: publicly disclose evaluation methods and results + +### Applying Results + +1. **Model improvement**: improve models based on evaluation findings +2. **Application guidance**: guide model selection for specific scenarios +3. **Capability matching**: match tasks to model capabilities +4. **Risk assessment**: identify model application risks +5. **Continuous monitoring**: continuously monitor model performance diff --git a/app/docs/ai/foundation-models/finetune/index.en.mdx b/app/docs/ai/foundation-models/finetune/index.en.mdx new file mode 100644 index 00000000..07a02560 --- /dev/null +++ b/app/docs/ai/foundation-models/finetune/index.en.mdx @@ -0,0 +1,348 @@ +--- +title: Model Fine-Tuning +description: "Large model fine-tuning techniques: LoRA, PEFT, fine-tuning frameworks, and other parameter-efficient methods" +date: "2025-01-27" +tags: + - fine-tuning + - lora + - peft + - parameter-efficient + - unsloth +docId: l5nes88zd54y6ao64ufkylz2 +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +Model fine-tuning is the key technique for adapting pre-trained large models to specific tasks. This section introduces various efficient fine-tuning methods and practical tips. + +## Fine-Tuning Overview + +### Types of Fine-Tuning + +1. **Full fine-tuning**: updates all model parameters +2. **Parameter-Efficient Fine-Tuning (PEFT)**: trains only a small number of parameters +3. **Instruction tuning**: fine-tuning on instruction-following data +4. **Alignment fine-tuning**: fine-tuning for human preference alignment + +### Fine-Tuning Challenges + +- **Compute resources**: full fine-tuning of large models is expensive +- **Catastrophic forgetting**: fine-tuning may degrade original capabilities +- **Data quality**: high-quality task data is difficult to obtain +- **Hyperparameter sensitivity**: fine-tuning hyperparameter selection is critical + +## Parameter-Efficient Fine-Tuning (PEFT) + +### Core Idea + +Achieve results comparable to full fine-tuning by training only a small number of parameters, dramatically reducing compute and storage costs. + +### Main Methods + +#### LoRA (Low-Rank Adaptation) + +**Principle**: decompose weight updates into the product of low-rank matrices + +``` +W_new = W_original + ΔW = W_original + BA +``` + +where B and A are trainable low-rank matrices. + +**Advantages**: + +- Dramatically reduces the number of trainable parameters +- Keeps pre-trained weights unchanged +- Supports multi-task LoRA merging +- Can be merged back into the original weights at inference time + +#### AdaLoRA (Adaptive LoRA) + +**Improvement**: adaptively adjusts the rank size for different layers + +- Allocates parameter budget based on importance +- Dynamically prunes less important parameters +- Further improves parameter efficiency + +#### Prefix Tuning + +**Principle**: prepends trainable prefix tokens to the input sequence + +- Only trains the prefix portion's parameters +- Keeps the model backbone unchanged +- Suited for generation tasks + +#### P-Tuning v2 + +**Improvement**: a deeper version of Prefix Tuning + +- Adds trainable parameters at every layer +- Better task adaptation capability +- Suitable for both understanding and generation tasks + +#### BitFit + +**Principle**: fine-tunes only bias parameters + +- Extremely few parameters (less than 0.1%) +- Suited for small-scale task fine-tuning +- Extremely low compute cost + +### Method Comparison + +| Method | Parameter Count | Use Case | Advantages | Disadvantages | +| ------------- | --------------- | ---------------- | ----------------------- | ---------------------- | +| LoRA | 0.1–1% | General tasks | Good results, easy impl | Need to choose rank | +| Prefix Tuning | 0.1–3% | Generation tasks | Stable results | Sequence length limits | +| P-Tuning v2 | 0.1–5% | Understanding | Strong adaptability | Slightly more params | +| BitFit | < 0.1% | Simple tasks | Minimal parameters | Limited expressiveness | + +## Fine-Tuning Frameworks and Tools + +### Recommended Frameworks + +#### LLaMA-Factory + +- **Highlights**: comprehensive fine-tuning toolkit +- **Support**: multiple models and fine-tuning methods +- **Ease of use**: web interface and configuration-driven +- **Documentation**: detailed usage tutorials + +#### Hugging Face TRL + +- **Highlights**: officially recommended framework +- **Support**: RL fine-tuning, SFT, DPO +- **Ecosystem**: deeply integrated with transformers +- **Updates**: continuously updated with latest techniques + +#### Swift Framework + +- **Source**: open-sourced by Alibaba +- **Highlights**: Chinese-friendly, supports multimodal +- **Performance**: optimized for domestic hardware +- **Community**: active Chinese-language community + +#### X-Tuner Framework + +- **Source**: MMDetection team +- **Highlights**: lightweight, easy to extend +- **Performance**: excellent memory optimization +- **Integration**: integrated with MMX toolset + +### Unsloth — Efficient Fine-Tuning Framework + +- **Project**: [GitHub link](https://github.com/unslothai/unsloth) +- **Highlights**: significant speed improvements (2–5x) +- **Optimization**: 80% reduction in memory usage +- **Support**: mainstream models and methods +- **Ease of use**: simple API interface + +## Fine-Tuning Practical Tips + +### Key Learning Points + +**Understand the underlying principles**: + +- Don't just run scripts — learn the underlying implementation +- Understand the KV Cache mechanism and memory management +- Master the role and implementation of Causal Mask +- Understand gradient computation and backpropagation + +### Data Preparation + +**Data formats**: + +- Instruction-response pair format +- Conversational data format +- Task-specific formats +- Multi-turn dialogue handling + +**Data quality**: + +- Data cleaning and deduplication +- Quality assessment and filtering +- Data balancing and augmentation +- Domain data collection + +### Hyperparameter Tuning + +**Key parameters**: + +- Learning rate: typically smaller than in pre-training +- LoRA rank (r): balance performance and efficiency +- LoRA alpha: controls adaptation strength +- Batch size: adjust based on hardware + +**Training strategies**: + +- Progressive learning rate scheduling +- Early stopping to prevent overfitting +- Gradient accumulation to simulate large batches +- Periodic evaluation and checkpointing + +## Multi-Task Fine-Tuning + +### Task Routing + +**Methods**: + +- Task-specific LoRA modules +- Mixture of Experts (MoE) architecture +- Conditional generation control +- Multi-head output design + +### Modular Design + +**LoRA combinations**: + +- Task-specific LoRA +- Domain-general LoRA +- Capability-enhancement LoRA +- Dynamic combination strategies + +## Advanced Fine-Tuning Techniques + +### Instruction Tuning + +**Data construction**: + +- Diverse instruction templates +- Task description variants +- Few-shot examples +- Negative sample construction + +**Training strategies**: + +- Multi-task mixed training +- Curriculum learning +- Contrastive learning enhancement +- Meta-learning methods + +### Reinforcement Learning Fine-Tuning (RLHF) + +**Process**: + +1. Supervised Fine-Tuning (SFT) +2. Reward model training +3. Reinforcement learning optimization +4. Iterative improvement + +**Key techniques**: + +- PPO algorithm optimization +- Reward model design +- Value function estimation +- Policy gradient computation + +### Alignment Fine-Tuning + +**Methods**: + +- Constitutional AI +- DPO (Direct Preference Optimization) +- Learning from human feedback +- Value alignment + +## Evaluation and Analysis + +### Evaluation Metrics + +**Task performance**: + +- Accuracy, F1 score +- BLEU, ROUGE scores +- Human evaluation quality +- Task-specific metrics + +**Model capabilities**: + +- Preservation of original capabilities +- Adaptation to new tasks +- Generalization performance testing +- Robustness analysis + +### Analysis Tools + +**Visualization**: + +- Loss curve analysis +- Attention weight visualization +- Parameter change tracking +- Performance comparison charts + +**Diagnostics**: + +- Overfitting detection +- Catastrophic forgetting analysis +- Parameter importance analysis +- Activation pattern analysis + +## Deployment and Inference + +### Model Merging + +**LoRA merging**: + +```python +# Merge LoRA weights back into the base model +merged_model = base_model + lora_model.merge() +``` + +**Multi-LoRA switching**: + +- Dynamic loading of different LoRAs +- Task-specific routing +- Memory-efficient switching +- Batch processing optimization + +### Inference Optimization + +**Memory optimization**: + +- Quantization techniques +- Gradient checkpointing +- Dynamic batching +- KV Cache optimization + +**Speed optimization**: + +- Model parallel inference +- Batch processing optimization +- Hardware acceleration +- Compilation optimization + +## Best Practices + +### Experiment Design + +1. **Establish baselines**: start with simple methods +2. **Ablation studies**: validate the contribution of each component +3. **Hyperparameter search**: systematic tuning +4. **Multiple runs**: ensure reproducibility +5. **Detailed logging**: record all experimental details + +### Engineering Tips + +1. **Progressive training**: from small data to large data +2. **Checkpoint management**: save and restore regularly +3. **Monitoring mechanisms**: real-time training state monitoring +4. **Error handling**: gracefully handle training exceptions +5. **Resource management**: allocate compute resources appropriately + +## Future Trends + +1. **Automated fine-tuning**: automatic selection of fine-tuning strategies and hyperparameters +2. **Multimodal fine-tuning**: unified fine-tuning for cross-modal tasks +3. **Personalized fine-tuning**: model adaptation to individual users +4. **Federated fine-tuning**: privacy-preserving distributed fine-tuning +5. **Continual learning**: continual adaptation without forgetting + +## Study Recommendations + +1. **Theory foundation**: deeply understand the mathematical principles of fine-tuning +2. **Hands-on practice**: start with simple tasks +3. **Code reading**: read the source code of excellent frameworks +4. **Experimental comparison**: compare the effectiveness of different methods +5. **Community participation**: be active in open-source communities and forums diff --git a/app/docs/ai/foundation-models/foundation-models-lifecycle.en.mdx b/app/docs/ai/foundation-models/foundation-models-lifecycle.en.mdx new file mode 100644 index 00000000..6cffba98 --- /dev/null +++ b/app/docs/ai/foundation-models/foundation-models-lifecycle.en.mdx @@ -0,0 +1,119 @@ +--- +title: Foundation Models +description: "The full lifecycle of foundation models: dataset construction, training, fine-tuning, deployment, and evaluation" +date: "2025-01-27" +tags: + - foundation-models + - llm-lifecycle + - model-development +docId: i88bna4sg5pr4ekhg32drv2i +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +Foundation models are the core of modern AI systems. This section covers the complete technology stack and lifecycle management from dataset construction to deployment and evaluation. + +## Core Components + +### Dataset Construction + +- See: [Dataset Construction](./datasets/) +- Data sourcing and acquisition strategies +- Data cleaning and quality control +- Privacy protection and compliance +- Multimodal data processing techniques + +### Model Training + +- See: [Model Training](./training/) +- Distributed training techniques +- MoE (Mixture of Experts) models +- Model weight merging strategies +- Training optimization and stability + +### Model Fine-Tuning + +- See: [Model Fine-Tuning](./finetune/) +- LoRA (Low-Rank Adaptation) +- PEFT (Parameter-Efficient Fine-Tuning) +- Instruction tuning and alignment +- Fine-tuning frameworks and tools + +### Deployment and Inference + +- See: [Deployment and Inference](./deploy-infer/) +- KV Cache optimization +- Flash Attention acceleration +- Quantization and parallel inference +- Inference framework comparison + +### Model Evaluation + +- See: [Model Evaluation](./evaluation/) +- Benchmark evaluation systems +- Chinese and English evaluation benchmarks +- Evaluation methods and metrics +- Result analysis and application + +### Classic QKV Interview Questions + +- See: [QKV Interview Questions](./qkv-interview/) +- KV Cache working principles +- Attention mechanism details +- Classic interview question breakdowns +- In-depth technical analysis + +## Learning Paths + +### Beginner Track + +1. Theory foundations: Transformer architecture and attention mechanism +2. Data processing: understanding the dataset construction pipeline +3. Fine-tuning practice: mastering LoRA and other parameter-efficient fine-tuning methods +4. Evaluation understanding: familiarity with mainstream benchmarks and metrics + +### Advanced Development + +1. Training optimization: distributed training and MoE +2. Inference acceleration: KV Cache, Flash Attention, etc. +3. Deployment engineering: vLLM, TensorRT, and other inference frameworks +4. Performance tuning: system-level performance analysis and optimization + +### Architecture Design + +1. Architecture trade-offs: pros and cons of different architectures and their scenarios +2. System integration: end-to-end application system design +3. Cost optimization: balancing performance, cost, and resources +4. Technology selection: scenario-driven technical solutions + +## Key Concepts + +### Decoder-only Architecture Advantages + +- Attention fit: causal attention naturally suits generation tasks +- Generation adaptation: natively suited for autoregressive language modeling +- Unified framework: multiple tasks unified under text generation + +### KV Cache Core Principles + +- Reuse: reusing historical KV pairs reduces computation +- Complexity reduction: O(n²) → O(n) +- Memory trade-off: trading space for time + +## Technology Trends + +1. Model efficiency: parameter-efficient training and inference optimization +2. Multimodal fusion: unified text/image/audio +3. Long-context handling: support for longer contexts +4. Edge deployment: compression for edge devices +5. Green AI: compute techniques that reduce energy consumption + +## References + +- _Hands-on Large Models_ (Zhihu column) +- _Attention is All You Need_ +- _Language Models are Few-Shot Learners_ + +> Learning tip: The stack is broad and fast-moving — choose your path based on your role and goals; balance theory with practice, and keep up with the frontier. diff --git a/app/docs/ai/foundation-models/qkv-interview/index.en.mdx b/app/docs/ai/foundation-models/qkv-interview/index.en.mdx new file mode 100644 index 00000000..8f249ffc --- /dev/null +++ b/app/docs/ai/foundation-models/qkv-interview/index.en.mdx @@ -0,0 +1,110 @@ +--- +title: Classic QKV Interview Questions +description: In-depth analysis of classic interview questions on the QKV mechanism and KV Cache in Transformers +date: "2025-01-27" +tags: + - interview-questions + - qkv + - kv-cache + - attention-mechanism + - transformer +docId: h7s6nm7h5oqnhhdq9m1mgwwo +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +# Classic QKV Interview Questions + +The QKV mechanism in Transformers is a hot topic in large model interviews. This section provides an in-depth breakdown of the classic interview questions. + +## Core Interview Questions + +### 1. Why Can KV Be Cached During LLM Inference? + +**Core reason**: the autoregressive generation property means KV pairs can be reused. + +**Detailed breakdown**: + +1. **Eliminate redundant computation**: the Keys and Values of historical sequences would need to be recomputed on every generation step — caching avoids this +2. **Speed up inference**: when generating a new token, only the current token's Query needs to be computed, then attended to the cached KV pairs +3. **Reduce computational complexity**: from O(n²·d) down to O(n·d), where n is sequence length and d is vector dimension +4. **Cross-request reuse**: multiple requests sharing the same prefix can share KV Cache, improving overall system throughput + +### 2. Why Can't Q Be Cached? + +**Key insight**: Q doesn't need to be cached — it's not that it can't be. + +**Reasoning**: + +1. **Dependency difference**: the output for each newly generated token only depends on that token's Q, and that Q is never needed again in subsequent inference steps +2. **No efficiency gain**: caching Q brings no efficiency improvement; each Q is generated based on the preceding sequence and has temporal dependencies +3. **Autoregressive property**: each token generation depends only on all previous tokens — computing Q is itself inherently based on the historical sequence + +### 3. Why Are Three Different Matrices WQ, WK, WV Needed? + +**Function separation**: decompose the attention mechanism into three distinct roles: + +- **Query generation (WQ)**: generates "what I'm looking for" +- **Key generation (WK)**: generates "what I am" +- **Value generation (WV)**: generates "what information I contain" + +**Mathematical principle**: different linear transformations learn different representation spaces, increasing the model's expressive power and flexibility. + +### 4. What Is the Purpose of Multi-Head Attention? + +**Core idea**: parallel specialization — different heads learn different types of attention patterns. + +**Specific roles**: + +1. **Information subspaces**: each head attends to different feature subspaces +2. **Attention diversity**: simultaneously captures multiple types of attention patterns +3. **Positional information**: different heads may focus on different positional relationships +4. **Semantic levels**: different heads attend to different levels of semantic information + +### 5. How Is KV Cache Memory Usage Calculated? + +**Formula**: + +``` +KV Cache memory = 2 × sequence length × num layers × hidden dim × num heads × bytes per element +``` + +**Optimization strategies**: + +- Quantization: use INT8 or INT4 quantization for KV Cache +- Paging: PagedAttention's paged storage +- Compression: dynamically compress inactive cache entries +- Sharing: KV Cache sharing across multiple requests + +## Advanced Technical Questions + +### Flash Attention Optimization Principle + +- **Memory access optimization**: tiled attention computation reduces data transfer between HBM and SRAM +- **Algorithm improvement**: IO complexity reduced from O(N²) to O(N²d²/M), enabling support for longer sequences + +### Impact of Different Precisions on KV Cache + +| Precision | Memory Usage | Compute Speed | Precision Loss | +| --------- | ------------ | ------------- | -------------- | +| FP16 | 50% | 1.5–2x | Minimal | +| INT8 | 25% | 2–3x | Small | +| INT4 | 12.5% | 3–4x | Moderate | + +## Interview Preparation Tips + +### Technical Depth + +1. **Understand the principles**: deeply understand the mathematical foundations of the attention mechanism +2. **Implementation details**: understand the concrete implementation of KV Cache +3. **Optimization techniques**: master related optimization techniques +4. **Performance analysis**: be able to analyze memory and compute overhead + +### Communication Skills + +1. **Structured answers**: follow the order of principle → implementation → optimization +2. **Use examples**: explain abstract concepts with concrete examples +3. **Back with data**: support optimization claims with specific numbers +4. **Comparative analysis**: compare the pros and cons of different approaches diff --git a/app/docs/ai/foundation-models/training/index.en.mdx b/app/docs/ai/foundation-models/training/index.en.mdx new file mode 100644 index 00000000..82b82087 --- /dev/null +++ b/app/docs/ai/foundation-models/training/index.en.mdx @@ -0,0 +1,293 @@ +--- +title: Model Training +description: "Large model training techniques: MoE, distributed training, model weight merging, and more" +date: "2025-01-27" +tags: + - model-training + - distributed-training + - moe + - model-merging + - training-optimization +docId: jgz0nl0cbd4frj2dg98mdv0x +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +Large model training is a complex engineering problem involving distributed computing, memory optimization, training stability, and more. + +## Training Fundamentals + +### Training Pipeline + +1. **Data preparation**: tokenization, batching, data loading +2. **Model initialization**: weight initialization, architecture configuration +3. **Forward pass**: compute the loss function +4. **Backward pass**: gradient computation and updates +5. **Model saving**: checkpoint saving and recovery + +### Key Techniques + +- **Gradient accumulation**: simulating large-batch training +- **Mixed precision**: FP16/BF16 training acceleration +- **Gradient clipping**: preventing gradient explosion +- **Learning rate scheduling**: optimizing the convergence process + +## Distributed Training + +### Data Parallelism + +- **Principle**: different GPUs process different data batches +- **Implementation**: PyTorch DDP, DeepSpeed +- **Suited for**: scenarios with small models but large data volumes + +### Model Parallelism + +- **Tensor parallelism**: split model layers across different GPUs +- **Pipeline parallelism**: assign model layers sequentially across GPUs +- **Expert parallelism**: expert assignment for MoE models + +### Hybrid Parallelism + +- **3D parallelism**: data + tensor + pipeline parallelism +- **Zero Redundancy Optimizer**: optimizer state sharding +- **Activation recomputation**: trade compute for memory + +## MoE (Mixture of Experts) + +### Core Concepts + +**Sparse activation**: only a subset of expert networks is activated each time, achieving a balance between computational efficiency and model capacity. + +**Routing mechanism**: intelligently routes inputs to appropriate experts + +- Top-k routing: selects the k most relevant experts +- Load balancing: ensures balanced expert utilization +- Noise injection: improves routing robustness + +**Architecture design**: + +- Expert network structure +- Gating network design +- Residual connection strategies + +### Technical Challenges + +1. **Load balancing**: preventing uneven expert utilization +2. **Communication overhead**: cross-device expert calls +3. **Training stability**: routing learning convergence +4. **Inference optimization**: sparse model inference acceleration + +## Model Weight Merging + +**Weight merging techniques**: see the full comparison table for weight merging methods + +### Merging Strategies + +**Linear interpolation merging**: + +```python +merged_weight = alpha * weight_1 + (1 - alpha) * weight_2 +``` + +**SLERP (Spherical Linear Interpolation)**: + +- Suited for normalized weights +- Preserves the angular relationship of weight vectors +- Particularly effective for embedding layers + +**Task Arithmetic**: + +- Merging based on task vectors +- Supports combining capabilities from multiple tasks +- Controllable capability transfer + +### Use Cases + +- **Multi-task model fusion**: combining capabilities from different tasks +- **Different training stage integration**: merging weights from different training stages +- **Capability combination optimization**: balancing different capability dimensions + +## Training Optimization Techniques + +### Memory Optimization + +**Gradient checkpointing**: + +- Recompute activation values +- Reduce memory footprint +- Trade-off with computation time + +**Zero Redundancy Optimizer**: + +- ZeRO-1: optimizer state sharding +- ZeRO-2: add gradient sharding +- ZeRO-3: parameter sharding + +**CPU offloading**: + +- Store parameters on CPU +- Dynamically load to GPU +- Extended memory capacity + +### Compute Optimization + +**Operator fusion**: + +- LayerNorm fusion +- Attention operator optimization +- Custom CUDA kernels + +**Compilation optimization**: + +- TorchScript compilation +- TensorRT optimization +- ONNX conversion + +## Training Stability + +### Numerical Stability + +**Loss scaling**: + +- Automatic mixed precision +- Dynamic loss scaling +- Gradient overflow detection + +**Weight initialization**: + +- Xavier/Kaiming initialization +- Hierarchical initialization +- Pre-trained weight loading + +### Training Monitoring + +**Metric monitoring**: + +- Loss curve tracking +- Gradient norm monitoring +- Learning rate changes +- Memory usage + +**Anomaly detection**: + +- NaN/Inf detection +- Gradient explosion monitoring +- Model divergence warnings + +## Large-Scale Training Engineering + +### Hardware Configuration + +**Compute resources**: + +- GPU cluster configuration +- Memory and bandwidth requirements +- Storage system design +- Network topology optimization + +**Environment management**: + +- Docker containerization +- Environment consistency guarantees +- Dependency management +- Version control + +### Experiment Management + +**Hyperparameter search**: + +- Grid search +- Bayesian optimization +- Early stopping +- Resource budget management + +**Experiment tracking**: + +- MLflow experiment logging +- Weights & Biases monitoring +- Experiment result comparison +- Reproducibility guarantees + +## Fault Handling + +### Common Issues + +1. **Out-of-memory**: batch size adjustment, gradient accumulation +2. **Convergence issues**: learning rate adjustment, architecture optimization +3. **Communication failures**: network configuration, node recovery +4. **Data issues**: data validation, anomaly handling + +### Recovery Strategies + +**Checkpoint mechanism**: + +- Periodically save model state +- Save optimizer state +- Record random seed +- Resume training progress + +**Fault-tolerant design**: + +- Node failure detection +- Automatic task restart +- Elastic training architecture +- Data integrity checks + +## Performance Optimization + +### System Level + +**I/O optimization**: + +- Data prefetching +- Multi-process data loading +- Memory-mapped files +- SSD storage optimization + +**Communication optimization**: + +- AllReduce algorithm optimization +- Communication topology design +- Bandwidth utilization improvement +- Latency reduction techniques + +### Algorithm Level + +**Training strategies**: + +- Progressive training +- Curriculum learning +- Adversarial training +- Multi-stage training + +**Regularization techniques**: + +- Dropout variants +- Weight decay +- Label smoothing +- Data augmentation + +## Best Practices + +1. **Thorough experiment design**: control variables, ensure reproducibility +2. **Progressive scaling**: validate from small models to large models step by step +3. **Monitoring-driven**: real-time monitoring of training state and resource usage +4. **Documentation**: detailed recording of experiment configurations and results +5. **Team collaboration**: establish good experiment sharing mechanisms + +## Future Directions + +1. **Automated training**: automatic hyperparameter tuning and architecture search +2. **Efficient architectures**: more efficient model architecture design +3. **Hardware co-design**: software-hardware co-optimization +4. **Green AI**: reducing training energy consumption and carbon emissions +5. **Federated learning**: distributed collaborative training paradigm + +## From UNSW IT-AI Involution Hell Documentation + +- SwanLab - AI model training tracking and visualization tool +- Documentation: https://docs.swanlab.cn/guide_cloud/general/what-is-swanlab.html +- Official site: https://swanlab.cn +- GitHub: https://github.com/swanhubx/swanlab diff --git a/app/docs/ai/generative-todo/generative-models-plan.en.mdx b/app/docs/ai/generative-todo/generative-models-plan.en.mdx new file mode 100644 index 00000000..d8cf5697 --- /dev/null +++ b/app/docs/ai/generative-todo/generative-models-plan.en.mdx @@ -0,0 +1,12 @@ +--- +title: Generative Models +description: "Generative models: Diffusion and other generative techniques" +status: todo +docId: nor5ktairygnt4dorqbddo9n +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T12:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +This section aggregates generative model resources (Diffusion, etc.) — content to be added later. From 07bd538f3d420deeb4dcbc42ff5d2fae6c7ce692 Mon Sep 17 00:00:00 2001 From: longsizhuo Date: Wed, 15 Apr 2026 17:45:42 +0000 Subject: [PATCH 11/19] =?UTF-8?q?feat(docs):=20i18n=20computer-science=20+?= =?UTF-8?q?=20jobs=20+=20all-projects=20=E7=BF=BB=E8=AF=91=E5=AE=8C?= =?UTF-8?q?=E6=88=90=20(23=20=E7=AF=87)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit translator-cs-jobs 产出： - computer-science: 双向翻译 - data-structures (en→zh, 5 篇): index / array / linked-list - frontend (zh→en, 2 篇) - cpp_backend (zh→en, 8 篇): mempool / threadpool / 编译系列 - index.mdx (zh→en) - jobs (zh→en, 5 篇): event-keynote 2 / interview-prep 3 - all-projects (zh→en, 2 篇): ai-town / multimodal-rl 代码块内变量名和 API 原样保留，仅翻译注释。 frontmatter 继承原文 docId，带 translatedFrom 标记。 --- app/docs/all-projects/ai-town.en.mdx | 74 +++ app/docs/all-projects/multimodal-rl.en.mdx | 94 ++++ .../1_Handwritten_threadpool.en.md | 423 ++++++++++++++++++ .../2_Handwritten_mempool1.en.md | 136 ++++++ .../cpp_backend/easy_compile/1_cpp_libs.en.md | 186 ++++++++ .../cpp_backend/easy_compile/2_base_gcc.en.md | 154 +++++++ .../cpp_backend/easy_compile/3_Make.en.md | 69 +++ .../cpp_backend/easy_compile/4_CMake.en.md | 187 ++++++++ .../cpp_backend/easy_compile/5_vcpkg.en.md | 150 +++++++ .../cpp_backend/mempool_simple.en.mdx | 120 +++++ .../array/01-static-array.zh.mdx | 173 +++++++ .../array/02-dynamic-array.zh.mdx | 327 ++++++++++++++ .../data-structures/array/index.zh.mdx | 84 ++++ .../data-structures/index.zh.mdx | 49 ++ .../data-structures/linked-list/index.zh.mdx | 258 +++++++++++ .../frontend/frontend-learning/index.en.mdx | 93 ++++ .../computer-science/frontend/index.en.mdx | 30 ++ app/docs/computer-science/index.en.mdx | 75 ++++ app/docs/jobs/event-keynote/coffee-chat.en.md | 137 ++++++ .../jobs/event-keynote/event-takeway.en.md | 30 ++ .../jobs/interview-prep/interview-tips.en.mdx | 94 ++++ .../jobs/interview-prep/pre-interview.en.md | 51 +++ ...ations-to-get-an-offer-as-a-student.en.mdx | 137 ++++++ 23 files changed, 3131 insertions(+) create mode 100644 app/docs/all-projects/ai-town.en.mdx create mode 100644 app/docs/all-projects/multimodal-rl.en.mdx create mode 100644 app/docs/computer-science/cpp_backend/Handwritten_pool_components/1_Handwritten_threadpool.en.md create mode 100644 app/docs/computer-science/cpp_backend/Handwritten_pool_components/2_Handwritten_mempool1.en.md create mode 100644 app/docs/computer-science/cpp_backend/easy_compile/1_cpp_libs.en.md create mode 100644 app/docs/computer-science/cpp_backend/easy_compile/2_base_gcc.en.md create mode 100644 app/docs/computer-science/cpp_backend/easy_compile/3_Make.en.md create mode 100644 app/docs/computer-science/cpp_backend/easy_compile/4_CMake.en.md create mode 100644 app/docs/computer-science/cpp_backend/easy_compile/5_vcpkg.en.md create mode 100644 app/docs/computer-science/cpp_backend/mempool_simple.en.mdx create mode 100644 app/docs/computer-science/data-structures/array/01-static-array.zh.mdx create mode 100644 app/docs/computer-science/data-structures/array/02-dynamic-array.zh.mdx create mode 100644 app/docs/computer-science/data-structures/array/index.zh.mdx create mode 100644 app/docs/computer-science/data-structures/index.zh.mdx create mode 100644 app/docs/computer-science/data-structures/linked-list/index.zh.mdx create mode 100644 app/docs/computer-science/frontend/frontend-learning/index.en.mdx create mode 100644 app/docs/computer-science/frontend/index.en.mdx create mode 100644 app/docs/computer-science/index.en.mdx create mode 100644 app/docs/jobs/event-keynote/coffee-chat.en.md create mode 100644 app/docs/jobs/event-keynote/event-takeway.en.md create mode 100644 app/docs/jobs/interview-prep/interview-tips.en.mdx create mode 100644 app/docs/jobs/interview-prep/pre-interview.en.md create mode 100644 app/docs/jobs/interview-prep/preparations-to-get-an-offer-as-a-student.en.mdx diff --git a/app/docs/all-projects/ai-town.en.mdx b/app/docs/all-projects/ai-town.en.mdx new file mode 100644 index 00000000..eda40423 --- /dev/null +++ b/app/docs/all-projects/ai-town.en.mdx @@ -0,0 +1,74 @@ +--- +title: AI Town Design Document +description: "" +date: "2025-10-18" +tags: + - ai-project +docId: bkxwg1m9p9rnm8062wsm020w +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T08:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +# AI Town Design Document + +## 1. Project Overview + +- **Type**: A lightweight simulation + social + quest-based mini-game driven by multi-agent NPCs +- **Core selling points**: NPCs "remember you" and collaborate with each other through dialogue; players can use **community contribution points** (earned by posting, submitting PRs, etc.) to obtain in-game currency and abilities, driving town events +- **Technology foundation**: Godot 4 (Microverse-style) + multi-agent framework (O-R-P-A: Observe → Retrieve → Plan → Act) + local model first (with template fallback) + +## 2. Goals (MVP) + +1. Single map + 3 NPCs (merchant / messenger / editor) + quest board (fetch / relay / check-in) +2. Dialogue with **short-term memory + end-of-day summary** +3. **Minimal community integration**: support entering a "redemption code" to receive coins / action points (future: automatic issuance via webhook) +4. Use points (or redeemed coins) to trigger 2–3 **visible world changes** (discount day / extra quests / expanded dialogue budget) + +## 3. Core Gameplay (Version 1) + +- **Loop**: Accept quest → Dialogue / collaborate with NPCs → Complete to earn coins / AP → Nightly summary generated → Events refresh next day +- **Uses of points / coins** (pick 2–3 to implement first) + - Unlock a **discount day** at the shop (all prices -10%) + - Purchase **action points** (one extra quest per day) + - Purchase **dialogue budget** (3 additional conversation turns with an NPC that day) + - Trigger a **theme-day announcement** (published by the editor NPC; NPC dialogue becomes more active) + +## 4. Open-Source Community Integration (Two Phases) + +### Phase A (MVP) — Redemption Code Verification + +- The community backend issues one-time **redemption codes** (containing point value and expiry); player enters code in-game → server verifies and voids → returns coins / AP +- **Advantage**: No login or account binding required; maximally stable and ready to ship + +### Phase B (Mid-term) — Automatic Issuance via Webhook + +- Posts, PR merges on GitHub / the site trigger a Webhook → write to `pending_rewards` +- Game launch or clicking "Sync" → fetch pending rewards → automatically credited +- Optional: bind Steam / GitHub account for stronger identity verification + +## 5. System Architecture (Minimal Modules) + +- **Client (Godot)** + - `Wallet` (authoritative entry point for coins / AP) + - `TaskManager`, `DialogManager`, `MemoryManager`, `CharacterManager` + - `TownEventBus` (broadcasts shop open / midday break / close / theme day) + - `RedeemPanel` (redemption code UI) + +- **Services (can be merged into community backend)** + - `/api/v1/redeem` (one-time verification and voiding) + - (Reserved) `/api/v1/rewards/pending`, `/webhooks/github` + +**Data Flow (MVP)** +Community issues code → Player enters it in-game → `redeem` verifies → Returns coins / AP → `Wallet` credits → `TownEventBus` triggers discount / quest refresh + +## 6. Scoring and Spending (Initial Draft) + +| Action | Community Points Earned | In-game Conversion (Example) | +| ---------------- | ----------------------: | ---------------------------- | +| Post approved | +80 | 80 pts = 400 coins | +| PR merged | +80 | 80 pts = 400 coins | +| Article featured | +50 | 50 pts = 1 "theme day" item | + +> Conversion rates are stored in a config file; events can apply temporary bonuses (e.g., 1.2× on weekends) diff --git a/app/docs/all-projects/multimodal-rl.en.mdx b/app/docs/all-projects/multimodal-rl.en.mdx new file mode 100644 index 00000000..cf0b5b2f --- /dev/null +++ b/app/docs/all-projects/multimodal-rl.en.mdx @@ -0,0 +1,94 @@ +--- +title: Multimodal Reinforcement Learning Project (MVP Goals) +description: Build a lightweight multimodal understanding and generation system that closes the loop from visual perception to language expression, incorporating reinforcement learning and answer-to-image generation. +date: "2025-10-17" +tags: + - projects + - multimodal + - reinforcement-learning + - RLHF +docId: ifwz8sqxqsgjrafa79pycrcm +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T08:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +# Multimodal Group – MVP Specification + +**Project version:** v0.1 +**Repository:** [involutionhell](https://github.com/InvolutionHell/involutionhell) + +--- + + +## 1. Vision + +Build a lightweight multimodal understanding and generation system that enables the model to interpret images, retrieve relevant information, and produce logically coherent text output. +The goal is to close the full loop from visual perception to language expression, and further develop the ability to explain answers through generated images. + + +## 2. MVP Phase Goals + + +### Phase 1: Basic Multimodal Pipeline + +- Image content recognition (objects, scenes, semantic labels). +- Semantic retrieval (image → text / text → image). +- Generative understanding and text output. +- Model references: CLIP / SigLIP / BLIP-2 / LLaVA / Qwen-VL. + + +### Phase 2: Multimodal Reinforcement Learning + +- Incorporate user feedback and reward signals to optimise model generation and retrieval performance. +- Main directions: + 1. RLHF / DPO fine-tuning to learn user preferences. + 2. Retrieval strategy optimisation based on behavioural data. + 3. Generation quality control and consistency improvement. + +- Goal: give the system the ability to self-improve and adapt to user preferences. + + +### Phase 2.5: Answer-to-Image Generation + +- Automatically generate illustrative images from the model's text answers to aid comprehension. +- Implementation: use Stable Diffusion / SDXL to convert answer text into image prompts. +- Application examples: + - Answer "the process of black hole formation" → generate a structural diagram. + - Explain a scene from a novel → generate a conceptual illustration. + +- Goal: enable the system not only to understand images and answer questions, but also to explain answers through generated images. + + +## 3. System Architecture + +``` +[Frontend] → Upload image / Display results + ↓ +[Backend API] → FastAPI + LangChain + Vector Search + ↓ +[Multimodal Models] → CLIP / BLIP / LLaVA / Qwen-VL + ↓ +[RL Module + Answer-to-Image] (Phase 2 and 2.5) +``` + + +## 4. Milestones + +| Phase | Goal | Deliverables | +| --------- | ------------------------------------- | --------------------------------------------- | +| Phase 1 | Multimodal recognition and generation | Image recognition, retrieval, text generation | +| Phase 2 | Reinforcement learning optimisation | RLHF / DPO, retrieval strategy optimisation | +| Phase 2.5 | Answer-to-image generation | Automatic illustration generation | +| Phase 3 | Scaling and deployment | Web demo and API interface | + + +## 5. Team Responsibilities + +| Module | Owner | +| ----------------------------------------------- | -------- | +| Image recognition and encoding | Member A | +| Semantic retrieval and data processing | Member B | +| Generation module and model integration | Member C | +| Reinforcement learning and visualisation output | Member D | diff --git a/app/docs/computer-science/cpp_backend/Handwritten_pool_components/1_Handwritten_threadpool.en.md b/app/docs/computer-science/cpp_backend/Handwritten_pool_components/1_Handwritten_threadpool.en.md new file mode 100644 index 00000000..79f12d08 --- /dev/null +++ b/app/docs/computer-science/cpp_backend/Handwritten_pool_components/1_Handwritten_threadpool.en.md @@ -0,0 +1,423 @@ +--- +title: Handwritten Thread Pool +description: "" +date: "2025-09-29" +tags: + - tag-one +docId: mnjkrtrs7xk3fq538eqreuge +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T08:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +# Thread Pool + +> The code below uses the single-queue `BlockingQueue`. When using the double-queue version, replace all `BlockingQueue` with `BlockingQueuePro`. + +## Interface + +**Constructor**: Initializes the blocking queue with a `unique_ptr`, creates `threads_num` threads, and binds each to the `Worker` function. (All Workers block inside `Pop()` at this point.) + +**Post**: Submits a task to the thread pool. + +**Destructor**: Wakes up all consumer threads. Workers that find no tasks exit. This has nothing to do with producers. + +```cpp +#pragma once + +#include +#include +#include + +template +class BlockingQueue; + +class ThreadPool { +public: + // initialize the thread pool + explicit ThreadPool(int threads_num); + + // stop the thread pool + ~ThreadPool(); + + // submit a task to the thread pool + void Post(std::function task); + +private: + // the loop function each thread runs + void Worker(); + // task queue + std::unique_ptr>> task_queue_; + // holds all thread objects; each thread is bound to Worker() + std::vector workers_; +}; +``` + +```cpp +#include "blockingqueue.h" +#include +#include "threadpool.h" + +// Creates a blocking queue and spawns threads_num threads, each bound to Worker(). +ThreadPool::ThreadPool(int threads_num) { + task_queue_ = std::make_unique>>(); + for (size_t i = 0; i < threads_num; ++i) { + workers_.emplace_back([this] {Worker();}); +// The lambda [this] { Worker(); } captures this, so each thread can call the current object's Worker. + } +} + +// Stop the thread pool +// Cancel the queue and wake up all blocked threads +ThreadPool::~ThreadPool() { + task_queue_->Cancel(); + for(auto &worker : workers_) { + if (worker.joinable()) + worker.join(); + } +} + +// Submit a task +void ThreadPool::Post(std::function task) { + task_queue_->Push(task); +} + +// Fetch a task from task_queue_ and execute it +void ThreadPool::Worker() { + while (true) { + std::function task; + // blocking is implemented inside Pop + if (!task_queue_->Pop(task)) { + break; + } + task(); + } +} +``` + +# Blocking Queue (Single-Queue Version) + +![Design](https://cdn.nlark.com/yuque/0/2025/jpeg/43055607/1758722093302-3845f815-ddbc-4bee-a789-de63daa92cd1.jpeg) + +**Single-queue maintenance:** + +1. `nonblock_` (bool): non-blocking flag. When `true`, the queue does not block. Default is blocking (false) at construction. +2. `queue_` (`std::queue`): the underlying storage container. +3. `mutex_` (`std::mutex`): mutex for thread-safe access. +4. `not_empty_` (`std::condition_variable`): used for producer-consumer synchronization. + +```cpp +template +class BlockingQueue { +public: + BlockingQueue(bool nonblock = false) : nonblock_(nonblock) { } + // enqueue + void Push(const T &value) { + // lock_guard automatically locks/unlocks (locks on construction, unlocks on destruction) + std::lock_guard lock(mutex_); + // push element into queue + queue_.push(value); + // notify one waiting thread: the queue is not empty, a task is available + not_empty_.notify_one(); + } + // normal pop: element is returned + // exceptional pop: no element returned + bool Pop(T &value) { + // lock — if already locked by another thread, this thread blocks here + // condition_variable::wait requires unique_lock + std::unique_lock lock(mutex_); + + // The main purpose of this line is to ensure safe dequeue. + // We only proceed when the queue is non-empty. + // But what if we only check queue emptiness and Cancel() is called? + // After Cancel(), we want consumers to exit, but they'd still block here + // because they don't know whether to stop. + // So we use nonblock_ to signal consumers to stop. + // + // Proceed when: queue is non-empty OR Cancel() has been called + // -> predicate is true -> continue + // Block when: queue is empty AND Cancel() has NOT been called + // -> predicate is false -> auto-unlock mutex, yield CPU, block here + not_empty_.wait(lock, [this]{ return !queue_.empty() || nonblock_; }); + if (queue_.empty()) return false; // consumer thread exits + + value = queue_.front(); + queue_.pop(); + return true; + } + + // unblock all threads waiting on this queue + void Cancel() { + // auto lock/unlock + std::lock_guard lock(mutex_); + // tell consumers to stop + nonblock_ = true; + // wake up all threads blocked in wait (each woken thread re-evaluates the predicate) + not_empty_.notify_all(); + } + +private: + bool nonblock_; + std::queue queue_; + std::mutex mutex_; + std::condition_variable not_empty_; +}; +``` + +# Blocking Queue (Double-Queue Version) + +![Design](https://cdn.nlark.com/yuque/0/2025/jpeg/43055607/1759131100901-946e59ae-cd19-4546-aa9d-a9ee658f0b5a.jpeg) + +In the single-queue version, both producers and consumers compete for the same lock. + +Double-queue design: + +- `prod_queue_`: the queue producers write to (protected by `prod_mutex_`). +- `cons_queue_`: the queue consumers read from (protected by `cons_mutex_`). +- When the consumer queue is empty, `SwapQueue_()` **swaps** the two queues, enabling batch transfer. + +Benefits: + +- **Reduced lock contention**: producers and consumers mostly do not compete for the same lock. +- **Higher throughput**: a single swap lets consumers retrieve data in bulk, reducing frequent locking. + +```cpp +template +class BlockingQueuePro { +public: + BlockingQueuePro(bool nonblock = false) : nonblock_(nonblock) {} + + void Push(const T &value) { + std::lock_guard lock(prod_mutex_); + prod_queue_.push(value); + not_empty_.notify_one(); + } + + bool Pop(T &value) { + std::unique_lock lock(cons_mutex_); + // auto-trigger swap when consumer queue is empty; if still empty after swap, return false + // no need for .wait here + if (cons_queue_.empty() && SwapQueue_() == 0) { + return false; + } + value = cons_queue_.front(); + cons_queue_.pop(); + return true; + } + + void Cancel() { + std::lock_guard lock(prod_mutex_); + nonblock_ = true; + not_empty_.notify_all(); + } + +private: + int SwapQueue_() { + std::unique_lock lock(prod_mutex_); + // block when producer queue is empty and not cancelled + // proceed when cancelled regardless of queue state + not_empty_.wait(lock, [this] {return !prod_queue_.empty() || nonblock_; }); + std::swap(prod_queue_, cons_queue_); + // return the number of items now in the consumer queue + // =0 only when producer queue was empty and cancellation was signalled + return cons_queue_.size(); + } + + bool nonblock_; + std::queue prod_queue_; + std::queue cons_queue_; + std::mutex prod_mutex_; + std::mutex cons_mutex_; + std::condition_variable not_empty_; +}; +``` + +# Benchmark + +The double-queue version is theoretically faster due to reduced lock contention. The experiment below confirms this. + +Setup: 4 producer threads, each submitting 25,000 tasks (each task runs a 1,000-iteration loop). The consumer thread count equals the optimal thread count for the current CPU (`std::thread::hardware_concurrency()`). In this WSL environment, that is 16. + +Both `single` (single-queue) and `double` (double-queue) versions are benchmarked, outputting total elapsed time and QPS. + +```cpp +std::atomic task_counter{0}; +int main() { + const int num_producers = 4; + const int num_tasks_per_producer = 25000; // 100,000 tasks total + const int num_threads_in_pool = std::thread::hardware_concurrency(); + + // For testing, the single and double versions are subclasses of ThreadPoolBase + // See appendix for implementation details + ThreadPoolSingle pool(num_threads_in_pool); + + auto start = std::chrono::high_resolution_clock::now(); + + std::vector producers; + // 4 producers start working + for (int i = 0; i < num_producers; ++i) { + producers.emplace_back(Producer, std::ref(pool), i, num_tasks_per_producer); + } + for (auto& p : producers) { + p.join(); + } + + // main thread busy-waits; using std::condition_variable to wait for wakeup would be more efficient + while (task_counter < num_producers * num_tasks_per_producer) { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + + auto end = std::chrono::high_resolution_clock::now(); + double elapsed = std::chrono::duration(end - start).count(); + + int total_tasks = num_producers * num_tasks_per_producer; + std::cout << "[Single Queue] Total: " << total_tasks + << " tasks. Time: " << elapsed + << " seconds. QPS = " << total_tasks / elapsed << std::endl; +} +``` + +```cpp +void Task(int id) { + volatile long sum = 0; + for (int i = 0; i < 1000; ++i) { + sum += i; + } + task_counter++; +} + +void Producer(ThreadPoolSingle& pool, int producer_id, int num_tasks) { + for (int i = 0; i < num_tasks; ++i) { + int task_id = producer_id * 100000 + i; + pool.Post([task_id]() { Task(task_id); }); + } +} +``` + +Results: +![](https://cdn.nlark.com/yuque/0/2025/png/43055607/1758958199070-807c1517-f594-4617-bf88-3f8228a66594.png) + +After reducing task count and main-thread poll interval by 10x (tasks per producer: 2,500, poll interval: 5 ms) — the original completed too fast and the main thread wait time distorted results: + +![](https://cdn.nlark.com/yuque/0/2025/png/43055607/1758958396516-f9017fa6-3732-4c52-b585-ed44e7c8b3ef.png) + +Averaging across runs would be more rigorous, but the gap is large enough that measurement error does not affect the conclusion. + +# Appendix + +Modified single-queue and double-queue versions + +```cpp +#pragma once + +#include +#include +#include +#include "blockingqueue.h" +#include "blockingqueuepro.h" + +// forward declarations +// blockingqueue can only be used as pointer or reference + +// template +// class BlockingQueue; +// template +// class BlockingQueuePro; + +// class ThreadPool { +// public: +// // initialize the thread pool +// explicit ThreadPool(int threads_num); + +// // stop the thread pool +// ~ThreadPool(); + +// // submit a task to the thread pool +// void Post(std::function task); + +// private: +// void Worker(); +// std::unique_ptr>> task_queue_; +// std::vector workers_; +// }; + + +class ThreadPoolBase { +public: + explicit ThreadPoolBase(int threads_num) : threads_num_(threads_num) {} + virtual ~ThreadPoolBase() = default; + + virtual void Post(std::function task) = 0; + +protected: + int threads_num_; + std::vector workers_; +}; + +class ThreadPoolSingle : public ThreadPoolBase { +public: + explicit ThreadPoolSingle(int threads_num) + : ThreadPoolBase(threads_num), + task_queue_(std::make_unique>>()) { + for (int i = 0; i < threads_num_; ++i) { + workers_.emplace_back([this] { Worker(); }); + } + } + + ~ThreadPoolSingle() { + task_queue_->Cancel(); + for (auto &w : workers_) { + if (w.joinable()) w.join(); + } + } + + void Post(std::function task) override { + task_queue_->Push(task); + } + +private: + void Worker() { + while (true) { + std::function task; + if (!task_queue_->Pop(task)) break; + task(); + } + } + + std::unique_ptr>> task_queue_; +}; + +class ThreadPoolDouble : public ThreadPoolBase { +public: + explicit ThreadPoolDouble(int threads_num) + : ThreadPoolBase(threads_num), + task_queue_(std::make_unique>>()) { + for (int i = 0; i < threads_num_; ++i) { + workers_.emplace_back([this] { Worker(); }); + } + } + + ~ThreadPoolDouble() { + task_queue_->Cancel(); + for (auto &w : workers_) { + if (w.joinable()) w.join(); + } + } + + void Post(std::function task) override { + task_queue_->Push(task); + } + +private: + void Worker() { + while (true) { + std::function task; + if (!task_queue_->Pop(task)) break; + task(); + } + } + + std::unique_ptr>> task_queue_; +}; +``` diff --git a/app/docs/computer-science/cpp_backend/Handwritten_pool_components/2_Handwritten_mempool1.en.md b/app/docs/computer-science/cpp_backend/Handwritten_pool_components/2_Handwritten_mempool1.en.md new file mode 100644 index 00000000..8e0db091 --- /dev/null +++ b/app/docs/computer-science/cpp_backend/Handwritten_pool_components/2_Handwritten_mempool1.en.md @@ -0,0 +1,136 @@ +--- +title: Handwritten Fixed-Size Memory Pool +description: "" +date: "2025-09-29" +tags: + - tag-one +docId: xgxqqvglxyauoeh8eye7lzu6 +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T08:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +# Handwritten Fixed-Size Memory Pool + +## Design Diagram + +![Design](https://cdn.nlark.com/yuque/0/2025/jpeg/43055607/1758718719250-e6f52459-0f73-493b-8294-7b8f931da054.jpeg) + +## Code + +### Struct Definitions + +#### `mempool_s` — struct that manages the memory pool + +```c +typedef struct mempool_s { + int blocksize; // size of each memory block + int freecount; // number of remaining free blocks + char *free_ptr; // pointer to the next free block + char *mem; // head pointer of the entire memory pool +} mempool_t; +``` + +### Public Interface + +#### `memp_create`: Create the memory pool + +```c +int memp_create(mempool_t *m, int block_size) { + + if (!m) return -1; + + // 1. initialize these two simple ints + m->blocksize = block_size; + m->freecount = MEM_PAGE_SIZE / block_size; + + // 2. allocate space for the entire pool and initialize m->mem + m->mem = (char *)malloc(MEM_PAGE_SIZE); + if (!m->mem) { // allocation failed (not enough free memory) + return -2; + } + // zero-initialize the allocated space + memset(m->mem, 0, MEM_PAGE_SIZE); + + // 3. initialize free_ptr + m->free_ptr = m->mem; + + // initialize the "next pointer" inside each block + int i = 0; + char *ptr = m->mem; + for (i = 0;i < m->freecount;i ++) { + + *(char **)ptr = ptr + block_size; + ptr = ptr + block_size; + } + // the last block's "next_ptr" points to NULL + *(char **)ptr = NULL; + return 0; +} +``` + +#### `memp_alloc`: Allocate a block + +```c +void *memp_alloc(mempool_t *m) { + // pool is full + if (!m || m->freecount == 0) return NULL; + // 1. get the next free block as the return value + void *ptr = m->free_ptr; + // 2. update free_ptr + m->free_ptr = *(char **)ptr; + // 3. update freecount + m->freecount --; + + return ptr; +} +``` + +#### `memp_free`: Free a specific block + +```c +void memp_free(mempool_t *m, void *ptr) { + // equivalent to: ptr->next = m->free_ptr + // insert the freed block at the head of the free list (head insertion) + *(char **)ptr = m->free_ptr; + // update free_ptr (the head of the free block linked list) + m->free_ptr = (char *)ptr; + // update freecount + m->freecount ++; +} +``` + +#### `memp_destory`: Destroy the entire memory pool + +```c +void memp_destory(mempool_t *m) { + if (!m) return ; + // free the entire pool in one call — the pool was malloc'd as a whole, not block by block + free(m->mem); +} +``` + +## Usage Example + +```c +int main() { + mempool_t m; + memp_create(&m, 32); + + void *p1 = memp_alloc(&m); + printf("memp_alloc : %p\n", p1); + + void *p2 = memp_alloc(&m); + printf("memp_alloc : %p\n", p2); + + void *p3 = memp_alloc(&m); + printf("memp_alloc : %p\n", p3); + + memp_free(&m, p2); +} +``` + +Output: each block is exactly 32 bytes apart, as expected. + +![](https://cdn.nlark.com/yuque/0/2025/png/43055607/1759069995143-4548da88-8c23-463e-b9e7-0f7d8978f03b.png) diff --git a/app/docs/computer-science/cpp_backend/easy_compile/1_cpp_libs.en.md b/app/docs/computer-science/cpp_backend/easy_compile/1_cpp_libs.en.md new file mode 100644 index 00000000..366ae5c5 --- /dev/null +++ b/app/docs/computer-science/cpp_backend/easy_compile/1_cpp_libs.en.md @@ -0,0 +1,186 @@ +--- +title: C++ Libraries on Linux/Windows +description: "" +date: "2025-09-29" +tags: + - tag-one +docId: totx4pej5lhyt1nl4anwhakj +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T08:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +# C++ Libraries on Linux/Windows + +## 1. What Does a Library Look Like? + +Every library has its own source format, but all libraries share: + +1. `.hpp`/`.h` (located under `include/` or the root directory): header files required for linking — these are mandatory. +2. `.cpp`: implements the logic declared in the headers. Some libraries are called **header-only libraries** (logic implemented directly in headers — no `.cpp`, no need to link a static or shared library). + +- On Linux: `XXX.so` (shared object / dynamic library) or `XXX.a` (static library) +- On Windows: `XXX.lib` (static library) or `XXX.dll` (dynamic library) + +## 2. How to Obtain a Library? + +> Unless otherwise noted, the environment is Linux (specifically Ubuntu). + +### Downloading a Third-Party Library + +#### Windows + +1. **Manually compile and install from source**: download or `git clone` a zip/7z archive, extract the source, and compile to produce the required library. +2. **Use a language-specific package manager** (C++: vcpkg; Python: pip; Java: Maven). For example, vcpkg downloads packages to `vcpkg/installed/` by default. + +> A brief introduction to vcpkg: +> +> - **vcpkg** is Microsoft's open-source cross-platform C/C++ package manager. +> - Supports **Windows / Linux / macOS**, though it was originally developed for the Windows ecosystem. +> - Its main purpose is to simplify obtaining third-party C/C++ libraries and their complex build configurations. + +#### Linux + +1. **Compile and install from source** + +> Some libraries can only be installed from source, such as the latest versions of gRPC and Protobuf. +> +> The typical workflow is: clone → run `make` or a one-click script like `build.sh`. + +2. **Use the system package manager** + +> `apt`, `yum`, `dnf`, etc. + +3. **Use a language-specific package manager** + +> vcpkg, conan, etc. + +### Writing Your Own Library + +After writing your `.cpp` and `.h` files, you can compile them into a static or dynamic library. + +#### Package as a Static Library + +```bash +g++ -c mylib.cpp -o mylib.o +ar rcs my_static_lib.a mylib.o // not recommended +// In practice, prefix the output name with "lib". +// This is more than a convention. +// When using -lmylib, the linker automatically prepends "lib" when searching, +// looking for libmylib.a or .so. +// Without the "lib" prefix, you must specify the full path (the linker can't find it automatically). +ar rcs libmy_static_lib.a mylib.o // recommended +``` + +#### Package as a Dynamic Library + +```bash +// Simple project: one-step compilation +g++ -shared -o mylib.dll mylib.cpp +// Complex project: first compile to .o, then convert to .dll +g++ -c -fPIC mylib.cpp -o mylib.o +g++ -shared -o my_dynamic_lib.dll mylib.o +``` + +`-fPIC` generates Position Independent Code, which is required for dynamic libraries. + +Following the steps above, you get `my_dynamic_lib.dll` (dynamic library) and `my_static_lib.a` (static library). + +## 3. Where Are Downloaded Libraries Stored? + +#### Python libraries may be located at + +- `venv/` in the project root (a virtual environment created by `venv`; the folder name is user-defined) +- A specific environment folder under the conda directory (the conda base path is set when conda is installed; all conda environments are stored there, with the environment name as a subdirectory) + +#### Frontend libraries may be located at + +- `node_modules/` in the project root (created and managed by npm) + +#### Java Spring projects (Maven) may be located at + +- Cached at `~/.m2/repository/` (the local Maven repository, same on Windows and Linux) + +#### C++ libraries (Linux) + +- System built-in libraries: `/lib`, `/lib64`, `/usr/lib`, `/usr/lib64` +- apt/yum/dnf/pacman package manager libraries: `/usr/lib/x86_64-linux-gnu` (library files), `/usr/include` (headers) +- Self-compiled: `/usr/local/lib`, `/usr/local/include` +- Package managers (vcpkg/conan): dedicated paths under the user's home directory + - vcpkg: `~/vcpkg/installed//lib` + - conan: `~/.conan/data///...` + - Custom placement: `~/lib`, `~/include` + +## 4. How to Use a Library? + +> Using a library requires two steps: **link the headers** and **link the library implementation**. + +### Link Headers + +> Without `-I` (uppercase i), the compiler searches the current file's directory by default. + +1. Keep library headers in place and hard-code each header path individually: + +```bash +g++ -I path/to/s_lib1.h -I path/to/s_lib2.h ... -o output main.cpp +``` + +2. Consolidate headers into an `include/` directory under the project, then specify that directory once (recommended for your own libraries; avoid moving third-party headers since they may have other dependencies): + +```bash +g++ -I include/ -o output main.cpp +``` + +(Note: the include directory specified with `-I` is combined with `#include "path/to/lib.h"` in code to form the full header path.) + +> If you find pure command-line compilation tedious, writing this into `CMakeLists.txt` with CMake is much more convenient. + +### Link Library Implementation + +> **For non-header-only libraries, you also need to link the library implementation — either dynamic or static.** + +#### Link a Dynamic Library + +```bash +g++ myapp.cpp -L /path/to/library -l mylib +``` + +At runtime, it is recommended to place `.dll` files in the same directory as the executable. The dynamic library search order is: + +1. Windows (highest to lowest priority): + - The program's current directory (usually where the `.exe` resides). + - Windows system directories (e.g., `C:\Windows\System32`). + - The current user's `AppData` folder. + +2. Linux (highest to lowest priority): + - `LD_LIBRARY_PATH` environment variable (set at runtime) + - `rpath` / `runpath` embedded in the executable + - System cache `/etc/ld.so.cache` + - **System default directories (most common)**: `/lib`, `/usr/lib`, `/usr/local/lib`, `/lib64` (64-bit systems). These are hard-coded into the dynamic linker. + +> You can also: 1. Set the `PATH` environment variable. 2. Use `LoadLibrary("D:\\libs\\mylib.dll")` in code to load a DLL explicitly. + +#### Link a Static Library + +```bash +g++ main.cpp -L /path/to/lib -l +``` + +Note: during compilation of a static library, only the availability of header files is checked — the compiler does not verify whether third-party libraries exist or are correctly linked. This is because the compilation stage only generates object files (`.o`) and packages them into a `.a` file; linking does not happen yet. A C++ program build can therefore be broken into three stages: + +- **Compiling the static library**: + - Only include headers (`include_directories`) — no need to specify third-party `.a` or `.so` files. +- **Compiling the main program**: + - Only include the static library's headers. + - No need to specify third-party `.a` or `.so` files. +- **Linking the main program**: + - Must explicitly bring in the static library and any third-party libraries it depends on (e.g., `-lthirdparty`). + +**Common compiler flags** (command-line compilation): + +1. `-I` (uppercase i) + header directory: tells the compiler where to find headers (required) +2. `-L` + library directory: specifies where library files are located +3. `-l` (lowercase L) + library name: specifies the library to link; searches for `.so` or `.a` (Windows: `.dll` and `.lib`); no suffix needed; prefers dynamic libraries + +The difference between `-L` and `-l`: `-L` specifies the path where the library lives; `-l` specifies the library name within that path. diff --git a/app/docs/computer-science/cpp_backend/easy_compile/2_base_gcc.en.md b/app/docs/computer-science/cpp_backend/easy_compile/2_base_gcc.en.md new file mode 100644 index 00000000..c3ff693d --- /dev/null +++ b/app/docs/computer-science/cpp_backend/easy_compile/2_base_gcc.en.md @@ -0,0 +1,154 @@ +--- +title: GCC/G++ Basics +description: "" +date: "2025-09-29" +tags: + - tag-one +docId: kyu85av71b4n07hbdycbhvj9 +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T08:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +# GCC/G++ Basics + +## Getting Started with g++ + +### Installing GCC/G++ (Linux) + +Install using your distribution's package manager. + +```cpp +sudo apt update +sudo apt install build-essential -y +// check versions +gcc --version +g++ --version +``` + +```cpp +// CentOS / RHEL 7: +sudo yum groupinstall "Development Tools" -y +// CentOS Stream 8 / RHEL 8+ / Fedora: +sudo dnf groupinstall "Development Tools" -y +``` + +### Installing GCC/G++ (Windows) + +Download (URLs below) and add to your `PATH` environment variable so `gcc`/`g++` commands work from any directory. + +1. MinGW + +[https://osdn.net/projects/mingw/downloads/68260/mingw-get-setup.exe/](https://www.mingw-w64.org/downloads/) + +[https://www.mingw-w64.org/downloads/](https://www.mingw-w64.org/downloads/) (recommended) + +**A gotcha with MinGW and the `` standard library:** + +Using threads with plain MinGW 9.2.0 requires these extra steps: + +1. Download additional header files from this repository: [https://github.com/meganz/mingw-std-threads](https://github.com/meganz/mingw-std-threads) + +![](https://cdn.nlark.com/yuque/0/2025/png/43055607/1759049314232-221dc93b-c560-4036-b049-db786935066f.png) + +Place those header files into MinGW's `include/` directory. + +2. In your code, change `#include ` to `#include ` (as documented in the repository's README). +3. If compiling from the command line, add the flag `-D_WIN32_WINNT=0x0501` to tell the compiler you are targeting Windows XP or later. (This may only be necessary for the win32 variant — the mingw-win64 version might not require it.) + +The problematic version I used: + +![](https://cdn.nlark.com/yuque/0/2025/png/43055607/1759049360158-4e44b580-0b41-4c64-8266-3a2bf893aa12.png) + +After switching to w64devkit: + +![](https://cdn.nlark.com/yuque/0/2025/png/43055607/1759049365783-54107cad-7f77-497a-a679-1cc80f2c5095.png) + +Also note: plain MinGW requires you to separately install `mingw32-make` via the MinGW Installer (`mingw-get.exe`). + +### Basic g++ Usage Example + +Create a text file (`test.txt`), rename its extension to `.cpp` (marking it as a C++ source file), open it with Notepad or VS Code, and write: + +```cpp +int main(){ + return 0; +} +``` + +Save and close the file. + +Open a terminal (cmd) in the directory containing `test.cpp` and run: + +```bash +g++ test.cpp +``` + +An `a.exe` executable appears in the current directory. Since no output name was specified, the compiler uses the default name `a`. + +To specify the output file name with `-o`: + +```bash +g++ -o b test.cpp +``` + +This produces `b.exe` in the current directory. + +`-o` stands for output; the token immediately following it is the output filename. You can also reorder the arguments: + +```bash +g++ test.cpp -o b +``` + +(Note: the `-o` flag and its value must stay together — do not separate them.) + +C/C++ compilation consists of four stages (the commands above show the all-in-one shortcut): + +Preprocessing → Compilation → Assembly → Linking + +| Stage | Input | Output & Extension | Flag (abbreviation meaning) | +| ------------- | ----------- | ------------------------------------- | --------------------------- | +| Preprocessing | `.cpp`/`.h` | Preprocessed file `.i` (Intermediate) | `-E` (Expansion) | +| Compilation | `.i` | Assembly code `.s` | `-S` (Source) | +| Assembly | `.s` | Object file `.o` | `-c` (Compile) | +| Linking | `.o` | Executable `.exe` or no extension | plain `g++` | + +**Preprocessing stage: `.cpp` → `.i`** +Handles `#include` (header inclusion), `#define` (macro expansion), `#ifdef` (conditional compilation), and similar directives. + +```bash +g++ -E test.cpp // print preprocessed output (expanded macros and included headers) to the terminal +g++ -E test.cpp -o preprocess.i // write output to preprocess.i +``` + +**Compilation stage: `.i` → `.s`** + +Translates preprocessed code into assembly. + +```bash +g++ -S preprocess.i -o assemble.s +``` + +**Assembly stage: `.s` → `.o`** + +Converts assembly into machine code, producing an object file (not directly executable on its own). + +```bash +g++ -c assemble.s -o machine.o +g++ -c test.cpp // you can also pass .cpp directly to generate the same-named .o +``` + +**Linking stage: `.o` → `.exe`** + +Links one or more object files with libraries to produce an executable. + +```bash +g++ machine.o -o test +``` + +**Generating debug information:** + +```bash +g++ -g test.cpp -o test // includes debug symbols compared to plain g++ test.cpp -o test +``` diff --git a/app/docs/computer-science/cpp_backend/easy_compile/3_Make.en.md b/app/docs/computer-science/cpp_backend/easy_compile/3_Make.en.md new file mode 100644 index 00000000..ffc97465 --- /dev/null +++ b/app/docs/computer-science/cpp_backend/easy_compile/3_Make.en.md @@ -0,0 +1,69 @@ +--- +title: Building with Make +description: "" +date: "2025-09-29" +tags: + - tag-one +docId: g6wucmr69lamd9xyxm7uunnd +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T08:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +# Building with Make + +### 1. How `make` Works + +`make` manages the build process based on **file dependencies** and **timestamps**, following these steps: + +1. Read the `Makefile`: `make` parses the `Makefile` to obtain build rules. +2. **Check target modification times**: `make` compares timestamps to decide whether to rebuild. For example, if `source.c` is newer than `source.o`, `make` considers `source.o` stale and re-runs the relevant compile command. +3. **Execute build rules**: if a target needs rebuilding, `make` runs the compilation and linking commands according to the dependency graph until the final target (executable or library) is produced. + +### 2. Basic `Makefile` Structure + +A `Makefile` is the configuration file `make` reads. It defines build rules, targets, dependencies, and commands. A typical `Makefile` contains: + +### Basic Syntax + +- **Target**: the file to build (usually an object file or the final executable). +- **Dependency**: files the target depends on. If a dependency is updated, the target must be rebuilt. +- **Command**: the shell command that produces the target (e.g., a compile command). Commands **must be indented with a TAB character**. + +```makefile +target: dependencies + command +``` + +### 3. `Makefile` Example + +Suppose we have a simple C++ project with two source files, `main.cpp` and `utils.cpp`, which produce object files `main.o` and `utils.o`, and are ultimately linked into the executable `myapp`. + +```makefile +CC = g++ # compiler: g++ +CFLAGS = -Wall -g # compile flags: -Wall enables all warnings, -g includes debug info + +# object files +OBJS = main.o utils.o + +# executable +TARGET = myapp + +# default target +all: $(TARGET) + +$(TARGET): $(OBJS) # executable depends on object files + $(CC) $(OBJS) -o $(TARGET) # link command: produces the executable + +main.o: main.cpp utils.h + $(CC) $(CFLAGS) -c main.cpp # compile command: produces main.o + +utils.o: utils.cpp utils.h + $(CC) $(CFLAGS) -c utils.cpp # compile command: produces utils.o + +clean: + rm -f $(OBJS) $(TARGET) # remove intermediate and target files +``` + +Targets serve as the entry points for execution. diff --git a/app/docs/computer-science/cpp_backend/easy_compile/4_CMake.en.md b/app/docs/computer-science/cpp_backend/easy_compile/4_CMake.en.md new file mode 100644 index 00000000..e8cf0b52 --- /dev/null +++ b/app/docs/computer-science/cpp_backend/easy_compile/4_CMake.en.md @@ -0,0 +1,187 @@ +--- +title: CMake +description: "" +date: "2025-09-29" +tags: + - tag-one +docId: xk44lx4q1gpcm1uqk8nnbg7q +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T08:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +# CMake + +[https://juejin.cn/post/6844903557183832078](https://juejin.cn/post/6844903557183832078) — A CMake tutorial on Juejin + +[https://zhuanlan.zhihu.com/p/97369704](https://zhuanlan.zhihu.com/p/97369704) — A CMake tutorial on Zhihu + +CMake is used to compile a C++ project into an executable or a static/dynamic library. + +## CMake Commands + +Run `cmake --help` for more information. + +1. CMake requires a `CMakeLists.txt`, or you can pass parameters manually when invoking CMake. +2. It is recommended to create a `build/` directory inside the project directory, then run CMake from there (you can also specify the build directory with `--build`). + +```bash +cmake .. // append any extra parameters not declared in CMakeLists.txt (most parameters are usually declared there) +``` + +Running CMake from the `build/` directory keeps generated files (intermediate and final) separate from the project source. `..` tells CMake that `CMakeLists.txt` and the source code are in the parent directory (the project root). + +CMake works in two steps: first it generates build system files, then it uses those files to perform the actual build. The key commands are: + +1. Generate using a preset + +```bash +cmake --preset=default +``` + +This generates: + +- `build.ninja` +- `CMakeCache.txt` +- `CMakeFiles/` +- `cmake_install.cmake` +- `vcpkg_installed/` +- `vcpkg-manifest-install.log` + +Besides using a preset, you can also specify options manually: + +Use `-S` to set the source directory and `-B` to set the build directory. For example: + +`cmake -S .` sets the current directory as the source directory (the CMake variable **`CMAKE_SOURCE_DIR`** holds this path). + +2. Build + +```bash +cmake --build vcpkg-build // build into vcpkg-build; no need to cd into build/ and run cmake .. separately +``` + +This generates (in addition to the files from step 1): + +- `build.ninja` +- `CMakeCache.txt` +- `CMakeFiles/` +- `cmake_install.cmake` +- `libfeature-extraction-lib.a` (the compiled static library, new compared to step 1) +- `vcpkg_installed/` +- `vcpkg-manifest-install.log` + +## CMakeLists.txt + +### Understanding `find_package` in Depth + +``` +find_package( [optional args: CONFIG REQUIRED]) +``` + +`find_package` does **not** eagerly load all components of a library. Components are loaded lazily when referenced by `target_link_libraries` or other commands that need them. It merely declares a global target. + +`CONFIG` — use the library's own config file (e.g., `spdlogConfig.cmake`) rather than CMake's built-in find module. + +`REQUIRED` — abort with an error if the package is not found. + +### `.cmake` Files + +`CMakeLists.txt` and `.cmake` files share exactly the same syntax. A `.cmake` file is essentially a reusable module for `CMakeLists.txt`. Include one as follows: + +```makefile +include(cmakes/gtests_main.cmake) +include(cmakes/gtests_fe.cmake) +``` + +Typically used with conditionals, for example to enable unit tests optionally: + +```makefile +option(BUILD_TESTS "Build tests" OFF) +if (BUILD_TESTS) + include(cmakes/gtests_main.cmake) + include(cmakes/gtests_fe.cmake) +endif () +``` + +### CMake Parameters and Their g++ Equivalents + +Every CMake command parameter has a corresponding g++ flag, except for internal-only parameters (helpers used within the CMake script itself). + +**Required configuration items in `CMakeLists.txt` and their g++ equivalents:** + +1. **Define the build target** — executable or library + +```cmake +add_executable(my_program main.cpp) +// or +add_library(my_library STATIC my_library.cpp) +``` + +`add_executable` corresponds to: + +```bash +g++ -o my_program main.cpp +``` + +`add_library` corresponds to: + +```bash +g++ -c my_library.cpp -o my_library.o +ar rcs libmy_library.a my_library.o +``` + +2. **Link dependency libraries** + +```cmake +target_link_libraries(my_program gtest gtest_main) +``` + +3. **Include directories** (for your own headers and third-party headers) + +```cmake +include_directories(my_program PRIVATE /path/to/include) +// or +target_include_directories(my_program PRIVATE /path/to/include) +``` + +4. **Set build type** + +```cmake +CMAKE_BUILD_TYPE +``` + +**CMake-internal parameters (no g++ equivalent):** + +1. Specify the minimum CMake version: `cmake_minimum_required(VERSION 3.10)` +2. Set project name and version: `project(ProjectName VERSION 1.0 LANGUAGES CXX)` +3. Find a package: `find_package(PackageName CONFIG REQUIRED)` + +**Package search order** (from ChatGPT — may need revision): + +CMake searches for a package's config file in this order: + +- Paths specified in the `CMAKE_PREFIX_PATH` environment variable. +- System default install paths (e.g., `/usr/lib/cmake`, `/usr/local/lib/cmake`). +- The path specified by `CMAKE_INSTALL_PREFIX`. +- If using a package manager like vcpkg, the path specified in its toolchain file. + +(Default vcpkg path: `[vcpkg-root]/installed/[triplet]/share/[package]/[package]Config.cmake`) + +For reference: + +- Package headers install to: `[vcpkg-root]/installed/[triplet]/include/` +- Library files install to: `[vcpkg-root]/installed/[triplet]/lib/` + +### Running Without `CMakeLists.txt` + +Required command-line parameters when not using a `CMakeLists.txt`: + +1. Specify the source directory (required) +2. Set build type: `-D CMAKE_BUILD_TYPE=Release` or `Debug` +3. Choose a build generator: + - `-G "Unix Makefiles"` — Unix make generator + - `-G "MinGW Makefiles"` — Windows make generator + - `-G "Ninja"` — Ninja generator (requires separate installation) + - `-G "Visual Studio 16 2019"` — Visual Studio 2019 generator + - `-G "Xcode"` — Apple Xcode generator diff --git a/app/docs/computer-science/cpp_backend/easy_compile/5_vcpkg.en.md b/app/docs/computer-science/cpp_backend/easy_compile/5_vcpkg.en.md new file mode 100644 index 00000000..32444e07 --- /dev/null +++ b/app/docs/computer-science/cpp_backend/easy_compile/5_vcpkg.en.md @@ -0,0 +1,150 @@ +--- +title: vcpkg Package Manager +description: "" +date: "2025-09-29" +tags: + - tag-one +docId: gtqamuq3tftmvzstbunkgbo5 +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T08:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +# vcpkg Package Manager + +vcpkg has two modes: **Classic mode** and **Manifest mode**. + +Classic mode: run `vcpkg install` to download packages, then specify the include and library directories in `CMakeLists.txt`. + +Manifest mode: write a `vcpkg.json` manifest file. + +## Configuration + +Download and set up the environment: + +```bash +git clone +cd vcpkg/ +./bootstrap-vcpkg.sh +``` + +Recommended: add vcpkg to your environment variables: + +```bash +echo 'export VCPKG_ROOT="$HOME/vcpkg"' >> ~/.bashrc +// adjust the path if you cloned vcpkg somewhere other than ~ +``` + +1. Add vcpkg to your project: + +```bash +// create and initialize the manifest files vcpkg.json and vcpkg-configuration.json +vcpkg new --application +// add a dependency (e.g. fmt); this appends "fmt" to the dependencies list in vcpkg.json +vcpkg add port fmt // only modifies vcpkg.json without validation — same as editing the file manually +``` + +vcpkg reads the manifest (`vcpkg.json`) to determine which dependencies to install and integrate with CMake, providing the packages the project needs. + +2. Add library information to `CMakeLists.txt`: + +```bash +find_package(fmt CONFIG REQUIRED) +target_link_libraries(HelloWorld PRIVATE fmt::fmt) +``` + +3. Run the CMake configuration: + +Create a `CMakePresets.json` file and set the toolchain (`CMAKE_TOOLCHAIN_FILE`) to point to vcpkg's built-in CMake toolchain. When this toolchain is active, CMake automatically links libraries installed by vcpkg. + +```json +{ + "version": 2, + "configurePresets": [ + { + "name": "vcpkg", + "generator": "Ninja", // (or "MinGW Makefiles") — equivalent to the -G "" flag shown earlier + "binaryDir": "${sourceDir}/build", + "cacheVariables": { + "CMAKE_TOOLCHAIN_FILE": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" + // {VCPKG_ROOT} can be set in CMakeUserPresets.json + // or configured as an environment variable + } + } + ] +} +``` + +```json +{ + "version": 2, + "configurePresets": [ + { + "name": "default", + "inherits": "vcpkg", + "environment": { + "VCPKG_ROOT": "" + } + } + ] +} +``` + +## Usage + +1. Package your own library and manage it with vcpkg (register it in a vcpkg registry). +2. Download and use third-party libraries managed by vcpkg. + +**Creating Your Own vcpkg Registry** + +In addition to the official vcpkg git registry at https://github.com/microsoft/vcpkg, you can create your own git registry (there is also a filesystem registry type, but this section focuses on git registries). + +**What is a vcpkg registry?** + +A registry stores metadata about libraries (not the library source code itself — storing source code would make it bloated). + +**What is it used for?** + +It is referenced in `vcpkg-configuration.json`, as shown below: + +```bash +{ + "default-registry": { + "kind": "git", + "repository": "https://github.com/microsoft/vcpkg.git", + "baseline": "234534dfvbsdvw43434f" + }, // the default registry is required — it loads the official registry (thousands of packages) + "registries": [ // custom registries + { + "kind": "git", + "repository": "https://github.com/xxx/xxx.git", // repository URL + "baseline": "d3e4723c1224t34fsdsvd0e4c2615f6d75", // version baseline + "reference": "main", // branch name + "packages": [ + "datastax-cpp-driver", // library names included in this registry + "cpp-common", + "ppconsul", + "leveldb", + "grpc", + "polaris-cpp" + ] + }, + { + "kind": "git", + "repository": "git@gitlab.xxxxx/xxxx.git", + "baseline": "15efa5017d9a3esdvsdvsdvwecs1d316", + "reference": "main", + "packages": [ + "feature-generation-lib", + "nps-client-brpc", + "brpc", + "cybercore-sdk-cpp", + "opentelemetry-cpp", + "mv-protocols-cpp", + "feature-extraction-lib" + ] + } + ] +} +``` diff --git a/app/docs/computer-science/cpp_backend/mempool_simple.en.mdx b/app/docs/computer-science/cpp_backend/mempool_simple.en.mdx new file mode 100644 index 00000000..b5384a7d --- /dev/null +++ b/app/docs/computer-science/cpp_backend/mempool_simple.en.mdx @@ -0,0 +1,120 @@ +--- +title: Handwritten Memory Pool (Simple Fixed-Size) +description: "" +date: "2025-09-27" +tags: + - tag-one +docId: q8290wmhyofuiskzn1ph63ta +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T08:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +# Handwritten Memory Pool (Simple Fixed-Size) + +# Simple Version (Fixed-Size Blocks) + +## Design Diagram + +![Design](https://cdn.nlark.com/yuque/0/2025/jpeg/43055607/1758718719250-e6f52459-0f73-493b-8294-7b8f931da054.jpeg) + +## Code Structure + +```c +typedef struct mempool_s { + int blocksize; // size of each memory block + int freecount; // number of remaining free blocks + char *free_ptr; // pointer to the next free block + char *mem; // head pointer of the entire memory pool +} mempool_t; +``` + +```c +int memp_create(mempool_t *m, int block_size) { + + if (!m) return -1; + + // 1. initialize these two simple ints + m->blocksize = block_size; + m->freecount = MEM_PAGE_SIZE / block_size; + + // 2. allocate space for the entire pool and initialize m->mem + m->mem = (char *)malloc(MEM_PAGE_SIZE); + if (!m->mem) { // allocation failed (not enough free memory) + return -2; + } + // zero-initialize the allocated space + memset(m->mem, 0, MEM_PAGE_SIZE); + + // 3. initialize free_ptr + m->free_ptr = m->mem; + + // initialize the "next pointer" inside each block + int i = 0; + char *ptr = m->mem; + for (i = 0;i < m->freecount;i ++) { + + *(char **)ptr = ptr + block_size; + ptr = ptr + block_size; + } + // the last block's "next_ptr" points to NULL + *(char **)ptr = NULL; + return 0; +} +``` + +```c +void *memp_alloc(mempool_t *m) { + // pool is full + if (!m || m->freecount == 0) return NULL; + // 1. get the next free block as the return value + void *ptr = m->free_ptr; + // 2. update free_ptr + m->free_ptr = *(char **)ptr; + // 3. update freecount + m->freecount --; + + return ptr; +} +``` + +```c +void memp_free(mempool_t *m, void *ptr) { + // equivalent to: ptr->next = m->free_ptr + // insert the block to be freed at the head of the free list (head insertion) + *(char **)ptr = m->free_ptr; + // update free_ptr (the head of the free block linked list) + m->free_ptr = (char *)ptr; + // update freecount + m->freecount ++; +} +``` + +```c +void memp_destory(mempool_t *m) { + if (!m) return ; + // free the entire pool in one call, since the pool was malloc'd as a whole, not block by block + free(m->mem); +} +``` + +## Usage Example + +```c +int main() { + mempool_t m; + memp_create(&m, 32); + + void *p1 = memp_alloc(&m); + printf("memp_alloc : %p\n", p1); + + void *p2 = memp_alloc(&m); + printf("memp_alloc : %p\n", p2); + + void *p3 = memp_alloc(&m); + printf("memp_alloc : %p\n", p3); + + memp_free(&m, p2); +} +``` diff --git a/app/docs/computer-science/data-structures/array/01-static-array.zh.mdx b/app/docs/computer-science/data-structures/array/01-static-array.zh.mdx new file mode 100644 index 00000000..fb8c3a7d --- /dev/null +++ b/app/docs/computer-science/data-structures/array/01-static-array.zh.mdx @@ -0,0 +1,173 @@ +--- +title: 静态数组 +description: 静态数组的实现原理、特性与使用场景 +date: "2024-01-04" +tags: + - static-array + - memory-management +docId: gmpls10e2dz0bbizotvhglc8 +lang: zh +translatedFrom: en +translatedAt: 2026-04-15T08:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +# 静态数组 + +静态数组是数组最基本的形式，其大小在编译时确定，在程序运行期间无法改变。 + +## 内存布局 + +静态数组在内存中连续存储： + +``` +内存地址： 1000 1004 1008 1012 1016 +数组： [10] [20] [30] [40] [50] +下标： 0 1 2 3 4 +``` + +假设每个整数占 4 字节，数组元素 `arr[i]` 的内存地址为： + +``` +address = base_address + i * element_size +``` + +## 特性分析 + +### 时间复杂度 + +- **访问**：O(1) —— 通过下标直接计算内存地址 +- **查找**：O(n) —— 需要遍历整个数组 +- **插入**：O(n) —— 需要移动后续元素 +- **删除**：O(n) —— 需要移动后续元素 + +### 空间复杂度 + +- **存储**：O(n) —— n 个元素 +- **额外空间**：O(1) —— 无需额外指针或元数据 + +## 代码实现 + +### C++ 实现 + +```cpp +#include +using namespace std; + +int main() { + // 声明静态数组 + int arr[5] = {10, 20, 30, 40, 50}; + + // 访问元素 + cout << "First element: " << arr[0] << endl; + + // 修改元素 + arr[2] = 35; + + // 遍历数组 + for (int i = 0; i < 5; i++) { + cout << arr[i] << " "; + } + + return 0; +} +``` + +### JavaScript 实现 + +```javascript +// JavaScript 中的数组实际上是动态的，但我们可以模拟静态数组的行为 +class StaticArray { + constructor(size) { + this.size = size; + this.data = new Array(size); + } + + get(index) { + if (index < 0 || index >= this.size) { + throw new Error("Index out of bounds"); + } + return this.data[index]; + } + + set(index, value) { + if (index < 0 || index >= this.size) { + throw new Error("Index out of bounds"); + } + this.data[index] = value; + } + + length() { + return this.size; + } +} + +// 使用示例 +const arr = new StaticArray(5); +arr.set(0, 10); +arr.set(1, 20); +console.log(arr.get(0)); // 10 +``` + +## 优缺点 + +### 优点 + +1. **内存效率高**：无额外元数据开销 +2. **对缓存友好**：连续内存布局提升访问效率 +3. **简单直接**：易于实现和使用 +4. **编译期优化**：编译器可进行更多优化 + +### 缺点 + +1. **大小固定**：运行时无法改变大小 +2. **内存浪费**：若无法充分利用所有空间 +3. **插入/删除效率低**：需要移动大量元素 + +## 应用场景 + +静态数组特别适用于： + +- **嵌入式系统**：内存受限，需要精确控制内存 +- **高性能计算**：需要最大化内存访问效率 +- **系统编程**：底层系统代码，需要可预测的内存布局 +- **固定大小数据集**：如像素数组、音频采样等 + +## 实际示例 + +### 图像处理 + +```cpp +// 处理 640x480 的灰度图像 +unsigned char image[640 * 480]; + +// 访问像素 (x, y) +int getPixel(int x, int y) { + return image[y * 640 + x]; +} + +// 设置像素值 +void setPixel(int x, int y, unsigned char value) { + image[y * 640 + x] = value; +} +``` + +### 查找表 + +```cpp +// 预计算查找表 +const int SQUARE_TABLE[101] = { + 0, 1, 4, 9, 16, 25, 36, 49, 64, 81, 100, + // ... 预计算 0-100 的平方值 +}; + +int getSquare(int n) { + return SQUARE_TABLE[n]; // O(1) 查找 +} +``` + +## 总结 + +静态数组是理解所有数组类型的基础。尽管存在大小固定的局限，其高效性和简洁性在特定场景下不可替代。 + +下一节我们将学习动态数组如何解决静态数组大小固定的问题。 diff --git a/app/docs/computer-science/data-structures/array/02-dynamic-array.zh.mdx b/app/docs/computer-science/data-structures/array/02-dynamic-array.zh.mdx new file mode 100644 index 00000000..45cf5a92 --- /dev/null +++ b/app/docs/computer-science/data-structures/array/02-dynamic-array.zh.mdx @@ -0,0 +1,327 @@ +--- +title: 动态数组 +description: 动态数组的实现原理、扩容机制与性能分析 +date: "2024-01-05" +tags: + - dynamic-array + - resizing + - amortized-analysis +docId: nuojcaq1s6r5nggul0uq3r3j +lang: zh +translatedFrom: en +translatedAt: 2026-04-15T08:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +# 动态数组 + +动态数组解决了静态数组大小固定的问题，可在运行时动态调整大小，是许多高级数据结构和算法的基础。 + +## 核心概念 + +动态数组通过以下机制实现动态扩展： + +1. **容量（Capacity）**：当前已分配内存能容纳的元素数量 +2. **大小（Size）**：当前实际存储的元素数量 +3. **扩容策略**：当 size 超过 capacity 时，重新分配更大的内存空间 + +``` +容量： [_ _ _ _ _ _ _ _] (capacity = 8) +大小： [1 2 3 4 _ _ _ _] (size = 4) +``` + +## 扩容机制 + +### 常见扩容策略 + +1. **倍增扩容**：每次将容量扩大为原来的两倍 +2. **黄金比例扩容**：增长因子为 1.5 或 1.618 +3. **固定增量扩容**：每次增加固定数量的空间 + +```javascript +// 倍增扩容示例 +function resize(oldCapacity) { + return oldCapacity * 2; // 或 oldCapacity + oldCapacity +} +``` + +### 扩容过程 + +``` +初始状态：[1 2 3 4] capacity=4, size=4 + +添加元素 5： +1. 检测到 size == capacity +2. 分配新内存：capacity = 4 * 2 = 8 +3. 复制旧数据：[1 2 3 4 _ _ _ _] +4. 添加新元素：[1 2 3 4 5 _ _ _] +5. 释放旧内存 +``` + +## 代码实现 + +### JavaScript 实现 + +```javascript +class DynamicArray { + constructor() { + this.capacity = 2; + this.size = 0; + this.data = new Array(this.capacity); + } + + // 获取元素 + get(index) { + if (index < 0 || index >= this.size) { + throw new Error("Index out of bounds"); + } + return this.data[index]; + } + + // 设置元素 + set(index, value) { + if (index < 0 || index >= this.size) { + throw new Error("Index out of bounds"); + } + this.data[index] = value; + } + + // 在末尾添加元素 + push(value) { + // 检查是否需要扩容 + if (this.size >= this.capacity) { + this.resize(); + } + + this.data[this.size] = value; + this.size++; + } + + // 移除最后一个元素 + pop() { + if (this.size === 0) { + throw new Error("Array is empty"); + } + + const value = this.data[this.size - 1]; + this.size--; + + // 可选：缩容以节省内存 + if (this.size < this.capacity / 4) { + this.shrink(); + } + + return value; + } + + // 扩容 + resize() { + const oldCapacity = this.capacity; + this.capacity *= 2; + const newData = new Array(this.capacity); + + // 复制旧数据 + for (let i = 0; i < this.size; i++) { + newData[i] = this.data[i]; + } + + this.data = newData; + console.log(`Expanded: ${oldCapacity} -> ${this.capacity}`); + } + + // 缩容 + shrink() { + if (this.capacity <= 2) return; + + const oldCapacity = this.capacity; + this.capacity = Math.floor(this.capacity / 2); + const newData = new Array(this.capacity); + + for (let i = 0; i < this.size; i++) { + newData[i] = this.data[i]; + } + + this.data = newData; + console.log(`Shrunk: ${oldCapacity} -> ${this.capacity}`); + } + + // 在指定位置插入元素 + insert(index, value) { + if (index < 0 || index > this.size) { + throw new Error("Index out of bounds"); + } + + if (this.size >= this.capacity) { + this.resize(); + } + + // 右移元素 + for (let i = this.size; i > index; i--) { + this.data[i] = this.data[i - 1]; + } + + this.data[index] = value; + this.size++; + } + + // 删除指定位置的元素 + remove(index) { + if (index < 0 || index >= this.size) { + throw new Error("Index out of bounds"); + } + + const value = this.data[index]; + + // 向左移动元素 + for (let i = index; i < this.size - 1; i++) { + this.data[i] = this.data[i + 1]; + } + + this.size--; + + if (this.size < this.capacity / 4) { + this.shrink(); + } + + return value; + } + + length() { + return this.size; + } + + toString() { + const elements = []; + for (let i = 0; i < this.size; i++) { + elements.push(this.data[i]); + } + return `[${elements.join(", ")}] (size: ${this.size}, capacity: ${this.capacity})`; + } +} +``` + +### 使用示例 + +```javascript +const arr = new DynamicArray(); + +// 添加元素 +arr.push(1); +arr.push(2); +arr.push(3); // 触发扩容 +console.log(arr.toString()); // [1, 2, 3] (size: 3, capacity: 4) + +// 插入元素 +arr.insert(1, 10); +console.log(arr.toString()); // [1, 10, 2, 3] (size: 4, capacity: 4) + +// 删除元素 +arr.remove(0); +console.log(arr.toString()); // [10, 2, 3] (size: 3, capacity: 4) +``` + +## 性能分析 + +### 时间复杂度 + +| 操作 | 平均情况 | 最坏情况 | 说明 | +| ---------------- | -------- | -------- | ----------------------- | +| 访问 | O(1) | O(1) | 直接索引访问 | +| 搜索 | O(n) | O(n) | 需要遍历 | +| 插入（末尾） | O(1)\* | O(n) | 摊还 O(1)，偶尔需要扩容 | +| 插入（任意位置） | O(n) | O(n) | 需要移动元素 | +| 删除（末尾） | O(1)\* | O(1) | 摊还 O(1) | +| 删除（任意位置） | O(n) | O(n) | 需要移动元素 | + +\*摊还时间复杂度 + +### 摊还分析 + +虽然单次扩容操作需要 O(n) 时间，但通过摊还分析可以证明： + +- 连续进行 n 次 `push` 操作的总时间复杂度为 O(n) +- 因此单次 `push` 操作的摊还时间复杂度为 O(1) + +**证明思路**： + +- 假设从空数组开始，进行 n 次 push 操作 +- 扩容发生在大小为 1, 2, 4, 8, ..., 2^k 时 +- 总的复制操作次数为：1 + 2 + 4 + ... + 2^k < 2n +- 所以平均每次 push 的成本为 (n + 2n) / n = 3 = O(1) + +## 优化策略 + +### 1. 选择合适的增长因子 + +```javascript +// 不同的增长策略 +const GROWTH_STRATEGIES = { + DOUBLE: (capacity) => capacity * 2, // 快速增长，可能浪费内存 + GOLDEN_RATIO: (capacity) => Math.floor(capacity * 1.5), // 平衡增长 + FIBONACCI: (capacity) => capacity + previousCapacity, // 渐进增长 +}; +``` + +### 2. 预分配容量 + +```javascript +// 如果知道大概的数据量，可以预分配容量 +class DynamicArray { + constructor(initialCapacity = 2) { + this.capacity = initialCapacity; + this.size = 0; + this.data = new Array(this.capacity); + } + + // 预留容量 + reserve(minCapacity) { + if (minCapacity > this.capacity) { + this.capacity = minCapacity; + this.resize(); + } + } +} +``` + +### 3. 内存对齐优化 + +```cpp +// C++ 中考虑内存对齐 +template +class DynamicArray { +private: + T* data; + size_t size; + size_t capacity; + + // 确保容量是 2 的幂次，有利于内存对齐 + size_t nextPowerOfTwo(size_t n) { + size_t power = 1; + while (power < n) power <<= 1; + return power; + } +}; +``` + +## 实际应用 + +### 1. 编程语言中的动态数组 + +- **JavaScript**：`Array` +- **Python**：`list` +- **Java**：`ArrayList` +- **C++**：`std::vector` +- **C#**：`List` + +### 2. 应用场景 + +- **缓冲区**：网络数据包缓冲、文件读取缓冲 +- **集合**：实现栈、队列等其他数据结构 +- **图形编程**：顶点数组、像素缓冲 +- **数据处理**：动态添加和处理数据项 + +## 总结 + +动态数组是现代编程中最重要的数据结构之一，它兼具数组高效访问的特性和灵活调整大小的能力。理解其扩容机制和摊还分析，对于编写高效代码至关重要。 + +虽然动态数组解决了静态数组的大小限制问题，但在某些场景下，链表等其他数据结构可能更为合适。下一节我们将学习链表相关知识。 diff --git a/app/docs/computer-science/data-structures/array/index.zh.mdx b/app/docs/computer-science/data-structures/array/index.zh.mdx new file mode 100644 index 00000000..6b0e2aee --- /dev/null +++ b/app/docs/computer-science/data-structures/array/index.zh.mdx @@ -0,0 +1,84 @@ +--- +title: 数组 +description: 数组的基本概念、特性与应用场景 +date: "2024-01-03" +tags: + - array + - linear-data-structure +docId: ai7cmwf4irjaobqf7uokj3b4 +lang: zh +translatedFrom: en +translatedAt: 2026-04-15T08:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +# 数组 + +数组是最基础也最重要的数据结构之一。它是一种有序集合，存储相同类型的元素，这些元素在内存中连续存放。 + +## 数组的特性 + +### 优点 + +- **随机访问**：可通过下标在 O(1) 时间内访问任意元素 +- **内存利用率高**：元素连续存储，内存利用率高 +- **对缓存友好**：连续的内存访问模式对 CPU 缓存非常友好 + +### 缺点 + +- **大小固定**：静态数组的大小在创建时确定，无法动态调整 +- **插入/删除开销大**：在中间位置插入或删除元素需要移动其他元素 + +## 数组类型 + +### 静态数组 + +- [静态数组详解](/computer-science/data-structures/array/static-array) +- 大小在编译时确定 +- 通常分配在栈上 + +### 动态数组 + +- [动态数组详解](/computer-science/data-structures/array/dynamic-array) +- 大小可在运行时调整 +- 通常分配在堆上 + +## 基本操作 + +```javascript +// 创建数组 +const arr = [1, 2, 3, 4, 5]; + +// 访问元素 - O(1) +const element = arr[2]; // 3 + +// 修改元素 - O(1) +arr[1] = 10; + +// 遍历数组 - O(n) +for (let i = 0; i < arr.length; i++) { + console.log(arr[i]); +} +``` + +## 时间复杂度 + +| 操作 | 时间复杂度 | +| ---- | ---------- | +| 访问 | O(1) | +| 查找 | O(n) | +| 插入 | O(n) | +| 删除 | O(n) | + +## 应用场景 + +数组适用于： + +- 需要频繁随机访问元素的场景 +- 数据量相对固定的情况 +- 需要高效遍历的算法 +- 作为其他数据结构的底层基础 + +--- + +继续学习数组的具体实现和高级应用！ diff --git a/app/docs/computer-science/data-structures/index.zh.mdx b/app/docs/computer-science/data-structures/index.zh.mdx new file mode 100644 index 00000000..f10987e1 --- /dev/null +++ b/app/docs/computer-science/data-structures/index.zh.mdx @@ -0,0 +1,49 @@ +--- +title: 数据结构基础 +description: 数据结构的基本概念与核心知识 +date: "2024-01-02" +tags: + - data-structures + - fundamentals +docId: vti0bt2qlnr681msbk6igznc +lang: zh +translatedFrom: en +translatedAt: 2026-04-15T08:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +# 数据结构基础 + +数据结构是计算机科学的基础，研究如何在计算机中组织和存储数据，以实现高效的访问与修改。 + +## 什么是数据结构？ + +数据结构是一种组织、管理和存储数据的格式，使数据的访问和修改更加高效。为算法选择合适的数据结构，是设计高效程序的关键。 + +## 主要分类 + +### 线性数据结构 + +- [数组](/computer-science/data-structures/array) - 最基础的数据结构 +- [链表](/computer-science/data-structures/linked-list) - 动态数据结构 +- 栈 - 后进先出（LIFO） +- 队列 - 先进先出（FIFO） + +### 非线性数据结构 + +- 树 - 层次结构 +- 图 - 复杂关系结构 +- 哈希表 - 快速查找结构 + +## 学习重点 + +学习数据结构时，重点关注以下方面： + +1. **时间复杂度** - 操作效率 +2. **空间复杂度** - 内存占用 +3. **应用场景** - 何时选用何种结构 +4. **实现细节** - 如何编写具体代码 + +## 下一步 + +建议从最基础的数组入手，逐步过渡到更复杂的数据结构。每种结构都有其独特的优势与适用场景。 diff --git a/app/docs/computer-science/data-structures/linked-list/index.zh.mdx b/app/docs/computer-science/data-structures/linked-list/index.zh.mdx new file mode 100644 index 00000000..9c271aad --- /dev/null +++ b/app/docs/computer-science/data-structures/linked-list/index.zh.mdx @@ -0,0 +1,258 @@ +--- +title: 链表 +description: 链表的基本概念、类型与应用场景 +date: "2024-01-06" +tags: + - linked-list + - pointers + - dynamic-data-structure +docId: lt9yrqt0ksl2liabq9ocw0z4 +lang: zh +translatedFrom: en +translatedAt: 2026-04-15T08:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +# 链表 + +链表是一种线性数据结构，其元素不存储在连续的内存位置，而是通过指针相互链接。每个元素（称为节点）包含数据和一个指向下一个节点的指针。 + +## 基本概念 + +### 节点结构 + +``` +┌─────────────┐ ┌─────────────┐ ┌─────────────┐ +│ data | next │───▶│ data | next │───▶│ data | null │ +└─────────────┘ └─────────────┘ └─────────────┘ + 节点 1 节点 2 节点 3 +``` + +每个节点包含： + +- **数据域（data）**：存储实际数据 +- **指针域（next）**：指向下一个节点 + +### 链表与数组的对比 + +| 特性 | 数组 | 链表 | +| ---------- | ------------- | -------------------- | +| 内存布局 | 连续 | 分散 | +| 访问方式 | 随机访问 O(1) | 顺序访问 O(n) | +| 插入/删除 | O(n) | O(1) | +| 内存开销 | 低 | 高（需存储额外指针） | +| 缓存友好性 | 好 | 差 | + +## 链表类型 + +### 单链表 + +- [单链表详解](/computer-science/data-structures/linked-list/singly-linked-list) +- 每个节点只有一个指向下一节点的指针 +- 只能从头到尾单向遍历 + +### 双链表 + +- [双链表详解](/computer-science/data-structures/linked-list/doubly-linked-list) +- 每个节点有两个指针：prev 和 next +- 可双向遍历 + +### 循环链表 + +- 最后一个节点指向第一个节点 +- 形成环形结构 + +## 基本操作 + +### 创建节点 + +```javascript +class ListNode { + constructor(data) { + this.data = data; + this.next = null; + } +} +``` + +### 遍历链表 + +```javascript +function traverse(head) { + let current = head; + while (current !== null) { + console.log(current.data); + current = current.next; + } +} +``` + +### 查找元素 + +```javascript +function search(head, target) { + let current = head; + let index = 0; + + while (current !== null) { + if (current.data === target) { + return index; + } + current = current.next; + index++; + } + + return -1; // 未找到 +} +``` + +## 时间复杂度分析 + +| 操作 | 时间复杂度 | 说明 | +| ---------------- | ---------- | -------------- | +| 访问 | O(n) | 需要从头遍历 | +| 查找 | O(n) | 需要遍历搜索 | +| 插入（头部） | O(1) | 直接修改头指针 | +| 插入（尾部） | O(n) | 需要找到尾节点 | +| 插入（已知位置） | O(1) | 直接修改指针 | +| 删除（头部） | O(1) | 直接修改头指针 | +| 删除（已知节点） | O(1) | 直接修改指针 | +| 删除（按值） | O(n) | 需要先查找 | + +## 优缺点 + +### 优点 + +1. **动态大小**：可在运行时动态增长和缩减 +2. **高效的插入/删除**：在已知位置的插入/删除为 O(1) +3. **内存利用率高**：按需分配内存 +4. **灵活性强**：可实现复杂数据结构 + +### 缺点 + +1. **额外内存开销**：每个节点需要存储指针 +2. **不支持随机访问**：无法直接访问第 i 个元素 +3. **缓存性能差**：节点在内存中不连续 +4. **指针管理复杂**：容易出现内存泄漏或悬空指针 + +## 应用场景 + +### 适合使用的情况 + +- **频繁插入/删除**：尤其是在列表中间操作 +- **大小未知**：运行时数据量变化较大 +- **实现其他结构**：如栈、队列、图的邻接表 +- **撤销操作**：编辑器的撤销功能 + +### 不适合使用的情况 + +- **需要随机访问**：频繁通过下标访问元素 +- **内存敏感**：对内存使用有严格要求 +- **缓存敏感**：需要高性能顺序访问 + +## 实际应用示例 + +### 1. 音乐播放列表 + +```javascript +class Song { + constructor(title, artist) { + this.title = title; + this.artist = artist; + this.next = null; + } +} + +class Playlist { + constructor() { + this.head = null; + this.current = null; + } + + addSong(title, artist) { + const newSong = new Song(title, artist); + if (!this.head) { + this.head = newSong; + this.current = newSong; + } else { + let last = this.head; + while (last.next) { + last = last.next; + } + last.next = newSong; + } + } + + nextSong() { + if (this.current && this.current.next) { + this.current = this.current.next; + return this.current; + } + return null; + } +} +``` + +### 2. 浏览器历史记录 + +```javascript +class HistoryEntry { + constructor(url, title) { + this.url = url; + this.title = title; + this.next = null; + } +} + +class BrowserHistory { + constructor() { + this.head = null; + this.maxSize = 100; + this.size = 0; + } + + visit(url, title) { + const entry = new HistoryEntry(url, title); + entry.next = this.head; + this.head = entry; + this.size++; + + // 限制历史记录大小 + if (this.size > this.maxSize) { + this.removeLast(); + } + } + + getHistory() { + const history = []; + let current = this.head; + while (current) { + history.push({ + url: current.url, + title: current.title, + }); + current = current.next; + } + return history; + } +} +``` + +## 学习建议 + +1. **从基础开始**：先掌握单链表的基本操作 +2. **用图形辅助理解**：用图示理解指针变化 +3. **注意边界情况**：空链表、单节点、头尾节点的特殊处理 +4. **多练习指针操作**：熟练掌握指针的增删改查 +5. **对比学习**：与数组对比，理解各自的适用场景 + +## 下一步 + +建议按以下顺序学习： + +1. [单链表](/computer-science/data-structures/linked-list/singly-linked-list) - 掌握基本概念和操作 +2. [双链表](/computer-science/data-structures/linked-list/doubly-linked-list) - 理解双向指针的优势 +3. 循环链表 - 了解特殊链表变体 +4. 链表高级应用 - 如 LRU 缓存、跳表等 + +掌握链表是理解更复杂数据结构（如树和图）的必要基础！ diff --git a/app/docs/computer-science/frontend/frontend-learning/index.en.mdx b/app/docs/computer-science/frontend/frontend-learning/index.en.mdx new file mode 100644 index 00000000..5b56b912 --- /dev/null +++ b/app/docs/computer-science/frontend/frontend-learning/index.en.mdx @@ -0,0 +1,93 @@ +--- +title: What Frontend Skills Should 2025 New Graduates Learn? +description: "" +date: "2025-09-19" +tags: + - frontend +docId: uzoqs57kwc4tfut4wvgnbjhf +lang: en +translatedFrom: zh +translatedAt: 2026-04-15T08:00:00Z +translatorAgent: claude-sonnet-4-6 +--- + +The author is a fresh graduate who worked domestically for 8 months. The following reflects my understanding of frontend learning for the 2025 autumn recruitment season. Please feel free to point out any errors or omissions. + +#### Prerequisites + +1. The ability to effectively use Google Search and leverage LLMs to find information. + +#### HTML + +1. Understand the purpose of `` tags, such as `viewport` and `keyword`. + +2. Use `` and `