supertensor: shape-aware tensor figure toolkit
Extracted from the tensor-formula-viz skill and rebuilt around the idea that the geometry rules should be enforced by construction rather than restated as prose an agent has to remember. - assets/supertensor.sty: faces, stacks, index faces, shared caption lanes, meaning box, signature. Macros take a declared axis and a declared role, so equal shapes get equal edges, a x a is square, a transpose swaps the face, and contracted axes share an edge length -- without any manual alignment. - scripts/preflight.sh: decide the TikZ/CJK path before drawing. - scripts/build.sh: compile and fail on silent corruption (missing CJK glyphs, overfull boxes, undeclared roles), then export pdf/svg/png/thumb. - scripts/test.sh: build every figure as a regression test for the package. - examples/: three golden figures (TP-FFN, causal MHA, MoE top-k gather) plus an anti-pattern gallery of figures that compile cleanly and still lie. - SKILL.md + references/: lean entry point, details loaded on demand.
This commit is contained in:
@@ -0,0 +1,114 @@
|
||||
% Golden example 3 -- MoE top-k routing and gather.
|
||||
% Shows the three cell semantics side by side in one figure: a SCORE face
|
||||
% (graded lightness), an INDEX face (discrete symbols, no ramp), and a BOOLEAN
|
||||
% support face (one flat level) -- plus a gather whose output heights are data
|
||||
% dependent and must sum back to T*k.
|
||||
% ../scripts/build.sh moe-topk-gather.tex
|
||||
\documentclass[border=10pt]{standalone}
|
||||
\usepackage[cjk]{supertensor}
|
||||
|
||||
\stsetrole{act}{stTeal} % X and the per-expert buffers: same data, regrouped
|
||||
\stsetrole{wg}{stViolet} % learned router weight
|
||||
\stsetrole{s}{stCoral} % scores
|
||||
\stsetrole{idx}{stOrange} % indices / token ids
|
||||
\stsetrole{m}{stGray} % boolean support
|
||||
|
||||
\stdim{T}{6}
|
||||
\stdim{d}{4}
|
||||
\stdim{E}{4}
|
||||
\stdim{k}{2}
|
||||
\stdim{ne}{3} % capacity per expert in this instance: n_e = 3
|
||||
|
||||
\begin{document}
|
||||
\begin{tikzpicture}
|
||||
|
||||
% ================================================================= formula ==
|
||||
\node (F) at (0,0) {\stformula{$\displaystyle
|
||||
\mathbf G=\mathrm{softmax}(\mathbf{XW}_g),\quad
|
||||
\mathcal I_t=\operatorname*{top-}k_{e}\,\mathbf G_{t,e},\quad
|
||||
\mathbf D_{t,e}=\mathbf 1[e\in\mathcal I_t],\quad
|
||||
\mathbf X^{(e)}=\mathrm{gather}(\mathbf X,\mathbf D_{:,e})$}};
|
||||
|
||||
% ============================================================ stage A row ===
|
||||
\node[st stage, below=7mm of F] (SA) {打分:token 对专家};
|
||||
\coordinate (a) at ($(SA)+(-3.6,-1.8)$);
|
||||
|
||||
\stface[role=act, bracket=true]{X}{(a)}{T}{d}
|
||||
\node[st op, right=6mm of X] (mA) {$\times$};
|
||||
\stface[role=wg]{Wg}{($(mA)+(1.6,0)$)}{d}{E}
|
||||
\node[st op, right=6mm of Wg] (eA) {$=$};
|
||||
\stface[role=s]{G}{($(eA)+(1.6,0)$)}{T}{E}
|
||||
|
||||
\node[inner sep=0pt, fit=(X)(Wg)(G)] (rowA) {};
|
||||
\stlane{rowA}
|
||||
\stcaption{X}{$\mathbf X$}{$T\times d$}
|
||||
\stcaption{Wg}{$\mathbf W_g$}{$d\times E$}
|
||||
\stcaption{G}{$\mathbf G$}{$T\times E$}
|
||||
\stnolane
|
||||
|
||||
% ============================================================ stage B row ===
|
||||
\node[st stage, below=9mm of X-shape.south west, anchor=north west] (SB)
|
||||
{取前 $k$:连续分数 $\rightarrow$ 离散选择};
|
||||
\coordinate (b) at ($(SB.west)+(0.6,-1.7)$);
|
||||
|
||||
% Indices are drawn as symbols, not as magnitudes: expert 3 is not "bigger"
|
||||
% than expert 0, so the index face gets no lightness ramp.
|
||||
\stindexface[role=idx]{I}{(b)}{T}{k}{0,1, 1,2, 2,3, 3,0, 0,2, 1,3}
|
||||
% The same routing decision as a boolean support: one flat level, exactly k
|
||||
% cells per row, and every unselected cell left unfilled.
|
||||
\stface[role=m, pattern=data, level=3,
|
||||
data={3300,0330,0033,3003,3030,0303}]{D}{($(I.east)+(3.1,0)$)}{T}{E}
|
||||
\starrowlabel{I.east}{D.west}{one-hot}
|
||||
|
||||
\node[st comm, right=9mm of D] (gz) {Gather};
|
||||
\starrow{D.east}{gz.west}
|
||||
|
||||
\node[inner sep=0pt, fit=(I)(D)(gz)] (rowB) {};
|
||||
\stlane{rowB}
|
||||
\stcaption{I}{$\mathcal I$}{$T\times k$}
|
||||
\stcaption{D}{$\mathbf D$}{$T\times E$}
|
||||
\stnolane
|
||||
|
||||
% ============================================================ stage C row ===
|
||||
% Every stage heading starts on the same left rail; only the vertical position
|
||||
% follows the previous row.
|
||||
\coordinate (cy) at ($(I-shape.south)+(0,-9mm)$);
|
||||
\node[st stage, anchor=north west] (SC) at (SB.west |- cy)
|
||||
{按专家聚合:每个缓冲区的高度是数据决定的};
|
||||
\coordinate (c) at ($(SC.west)+(0.5,-1.9)$);
|
||||
|
||||
% Each buffer keeps X's width d -- gather regroups rows, it never reshapes the
|
||||
% feature axis. The heights are n_e, and they must sum to T*k.
|
||||
\stindexface[role=idx, border=false]{t0}{(c)}{ne}{1}{1,4,5}
|
||||
\stface[role=act]{B0}{($(t0.east)+(2*\stunit,0)$)}{ne}{d}
|
||||
\stindexface[role=idx, border=false]{t1}{($(B0.east)+(1.1,0)$)}{ne}{1}{1,2,6}
|
||||
\stface[role=act]{B1}{($(t1.east)+(2*\stunit,0)$)}{ne}{d}
|
||||
\stindexface[role=idx, border=false]{t2}{($(B1.east)+(1.1,0)$)}{ne}{1}{2,3,5}
|
||||
\stface[role=act]{B2}{($(t2.east)+(2*\stunit,0)$)}{ne}{d}
|
||||
\stindexface[role=idx, border=false]{t3}{($(B2.east)+(1.1,0)$)}{ne}{1}{3,4,6}
|
||||
\stface[role=act]{B3}{($(t3.east)+(2*\stunit,0)$)}{ne}{d}
|
||||
|
||||
\stcaptiontop{t0}{\stshapefont{token}}
|
||||
|
||||
\node[inner sep=0pt, fit=(t0)(B3)] (rowC) {};
|
||||
\stlane{rowC}
|
||||
\stcaption{B0}{$\mathbf X^{(1)}$}{$n_1\times d$}
|
||||
\stcaption{B1}{$\mathbf X^{(2)}$}{$n_2\times d$}
|
||||
\stcaption{B2}{$\mathbf X^{(3)}$}{$n_3\times d$}
|
||||
\stcaption{B3}{$\mathbf X^{(4)}$}{$n_4\times d$}
|
||||
\stnolane
|
||||
|
||||
% ============================================================== meaning box ==
|
||||
\node[inner sep=0pt, fit=(F)(rowA)(rowB)(rowC)(B3-shape)(t0-top)] (all) {};
|
||||
\stmeaningbox{mb}{16.6cm}{all}
|
||||
{$T$ token 数,$d$ 模型宽度,$E$ 专家数(图中 $E=4$),$k$ 每 token 选中的专家数
|
||||
(图中 $k=2$),$n_e$ 落到第 $e$ 个专家的 token 数}
|
||||
{$\mathbf G$ 是分数,深浅可比大小;$\mathcal I$ 是索引,格内是符号不是数值,
|
||||
故不用深浅;$\mathbf D$ 是布尔支撑,只有一档灰、每行恰好 $k$ 格;
|
||||
$\mathbf X^{(e)}$ 与 $\mathbf X$ 同色,因为它是同一批数据换了分组}
|
||||
{top-$k$ 把连续分数截成离散选择,这一步不可微;gather 只重排行、不动特征轴,
|
||||
故每个缓冲区仍是 $d$ 宽;$\sum_e n_e=Tk$,图中 $4\times3=6\times2$}
|
||||
\stsignature{MoE 路由:top-$k$ 选择与按专家 gather}{mb}
|
||||
|
||||
\end{tikzpicture}
|
||||
\end{document}
|
||||
Reference in New Issue
Block a user