diff --git a/.env.example b/.env.example index 825f562..79edc7c 100644 --- a/.env.example +++ b/.env.example @@ -17,6 +17,11 @@ APP_ENV=production # ---- Binance ---- BINANCE_BASE_URL=https://fapi.binance.com +# ---- CoinGlass（默认关闭；开启后必须填 TOTP）---- +COINGLASS_ENABLED=false +COINGLASS_BASE_URL=https://capi.coinglass.com +COINGLASS_TOTP_SECRET= + # ---- Go module build ---- # 国内构建推荐 goproxy.cn；海外可改为 https://proxy.golang.org,direct。 # 注意：这不是 HTTP 代理，不要填 http://127.0.0.1:7890。 diff --git a/.gitignore b/.gitignore index 63f1481..a6583dd 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,7 @@ logs/* # OS .DS_Store Thumbs.db + +.understand-anything + +docs/coinglass/ diff --git a/Makefile b/Makefile index a62e4f7..8e9c484 100644 --- a/Makefile +++ b/Makefile @@ -31,11 +31,14 @@ guard: @echo "G1 (no signing)..." && ! grep -rqi --include="*.go" "apikey\|x-mbx-apikey\|hmac\|secret_key\|api_secret" . @echo "G6 (boundary)..." && ! grep -rq "repo/webapi/binance\|repo/persistent" internal/usecase internal/controller @echo "G11 (ctrl derivs)..." && ! grep -rqn "FundingRepo\|OIRepo\|LSRepo\|TakerRepo" internal/controller - @echo "G12.1 (usecase float)..." && ! { grep -rn --include="*.go" "float64\|float32" internal/usecase | grep -vE "indicator|derivatives_signal" | grep -q . ; } + @echo "G11.2 (ctrl coinglass)..." && ! grep -rqn "CoinglassRepo\|coinglassRepo" internal/controller + @echo "G12.1 (usecase float)..." && ! { grep -rn --include="*.go" "float64\|float32" internal/usecase | grep -vE "indicator|derivatives_signal|coinglass" | grep -q . ; } @echo "G12.2 (indicator no repo)..." && ! grep -q "internal/repo" internal/usecase/indicator.go @echo "G12.3 (indicator no DB)..." && ! grep -qE "binance|postgres|pgx" internal/usecase/indicator.go @echo "G12.5 (derivatives_signal no repo)..." && ! grep -q "internal/repo" internal/usecase/derivatives_signal.go @echo "G12.6 (derivatives_signal no DB)..." && ! grep -qE "binance|postgres|pgx" internal/usecase/derivatives_signal.go + @echo "G12.7 (coinglass_signal no repo)..." && ! { test -f internal/usecase/coinglass_signal.go && grep -q "internal/repo" internal/usecase/coinglass_signal.go ; } + @echo "G12.8 (coinglass_signal no DB)..." && ! { test -f internal/usecase/coinglass_signal.go && grep -qE "binance|postgres|pgx" internal/usecase/coinglass_signal.go ; } @echo "G12.4 (entity no float)..." && ! grep -rqn --include="*.go" "float64\|float32" internal/entity @echo "G13.1 (compose requires password)..." && grep -qx 'POSTGRES_PASSWORD=' .env.example @echo "G13.2 (compose fixed app port)..." && grep -q 'APP_HOST_PORT' docker-compose.yml && grep -q 'APP_PORT: "8080"' docker-compose.yml && ! grep -q '$${APP_PORT' docker-compose.yml diff --git a/ai/adr/0004-coinglass-numeric-boundary-extension.md b/ai/adr/0004-coinglass-numeric-boundary-extension.md new file mode 100644 index 0000000..1aca378 --- /dev/null +++ b/ai/adr/0004-coinglass-numeric-boundary-extension.md @@ -0,0 +1,121 @@ +# ADR 0004 — CoinGlass 数值边界例外扩展 + +**状态**：Accepted +**日期**：2026-05-25 +**决策者**：项目维护者 +**适用范围**：`internal/usecase/coinglass_signal.go`（新）、`internal/usecase/coinglass_query.go`（新）、`internal/repo/webapi/coinglass/mapper.go`（新）、`internal/repo/webapi/coinglass/crypto.go`（新）、`Makefile` 的 `guard` 目标 + +--- + +## 背景 + +v3 phase-1 CoinGlass 接入（spec：`ai/specs/coinglass.md`）会在两个不同位置引入 transient `float64`： + +1. **`internal/usecase/coinglass_signal.go`** — 跨交易所 OI / funding / LSR 聚合，做加权平均、percentile、阈值比较。和 `derivatives_signal.go` 同形态。 +2. **`internal/repo/webapi/coinglass/mapper.go`** — 解密后的 plaintext JSON 通过 `json.Unmarshal` 默认进 `interface{}` 时数值就是 `float64`；mapper 把这些 `float64` 映射到 entity 字段（spec D6 锁定：出口 `strconv.FormatFloat(_, 'f', -1, 64)` 回 string）。 + +这两个位置和 ADR-0002 当时锁定的边界形态不完全一致： + +- ADR-0002 的白名单**只**覆盖 `internal/usecase/`（G12.1 grep 路径），且白名单文件名 pattern 是 `indicator|derivatives_signal`。 +- `coinglass_signal.go` 套用同款例外，**形态相同 → 直接扩白名单**就够。 +- `repo/webapi/coinglass/mapper.go` 是**新形态**：webapi 层的 transient float64。当前 G12.1 不扫 `repo/`，但下游 webapi mapper 第一次出现 `float64` 也意味着 G12 的"边界守住"语义需要明文表态。 + +问题：怎么把 CoinGlass 的两个 float64 位置纳入 G12，而不让"白名单越扩越松"失控。 + +--- + +## 决策 + +**沿用 ADR-0002 的"白名单 + 出口强转 string"模式，把 G12 的语义按场景扩两块：** + +### D1. usecase 层白名单扩 `coinglass` + +`Makefile` `guard` 的 G12.1 第三个白名单 token： + +```makefile +@echo "G12.1 (usecase float)..." && ! { \ + grep -rn --include="*.go" "float64\|float32" internal/usecase \ + | grep -vE "indicator|derivatives_signal|coinglass" \ + | grep -q . ; } +``` + +`coinglass` 这个 token 同时覆盖 `coinglass_signal.go` 与 `coinglass_query.go`。后者实际上**不应**用 float64（query usecase 只是从 repo 拉 entity 再返回），但 token 形态保持简单——靠 code review + 同步新增的 G12.7 强约束 query 文件不引 float64 也行（见 D3 取舍）。 + +### D2. 新增 G12.7 / G12.8（coinglass_signal 同款约束） + +镜像 G12.5 / G12.6 的形态： + +```makefile +@echo "G12.7 (coinglass_signal no repo)..." && ! grep -q "internal/repo" internal/usecase/coinglass_signal.go +@echo "G12.8 (coinglass_signal no DB)..." && ! grep -qE "binance|postgres|pgx" internal/usecase/coinglass_signal.go +``` + +`coinglass_query.go` **不**列入 no-repo 守卫——query usecase 本来就要打 repo（沿用 `market_query.go` 形态）。 + +### D3. repo/webapi/coinglass mapper 不进 G12 扫描，但必须满足出口规则 + +`repo/webapi/coinglass/mapper.go` 与 `crypto.go` 内部允许 transient `float64`，**不**在 G12.1 的 grep 路径里（G12.1 仍只扫 `internal/usecase`）。 + +约束改成两层兜底： + +- **静态**：G12.4 不变（`internal/entity` 任何位置不得 float）。CoinGlass entity 全 string（spec D6），mapper 出口若漏掉 FormatFloat，G12.4 会立刻 break。 +- **测试**：`mapper_test.go` 必须 table-driven 覆盖至少一个含小数的字段（funding rate、OI 数量），断言 entity.xxx 字段是 `string` 类型且形如 `"0.0123"`——这是 mapper 行为的回归底线。 + +**为什么不扩 G12.1 去扫 webapi**：`internal/repo/webapi/binance/` 下虽然没 float64（Binance 直接返回 string，mapper 无需转换），但将来若有别的数据源走相同的"JSON number → float64 → FormatFloat 回 string"路径，每个都要单独白名单会让 G12.1 越来越糙。CoinGlass mapper 的安全网是 G12.4（entity 全 string）+ unit test，**结构性**确保 float 不会泄漏到下游，比 grep 白名单更稳。 + +### D4. `coinglass_query.go` 严格禁 float64（隐式） + +`coinglass_query.go` 从 repo 拿到的 entity 已经是 string（D3 出口规则保证），usecase 内任何 float64 都属于 bug。G12.1 的 `coinglass` token 是"宽门"，但 code review 阶段对 query 文件出现 float64 应该当作必改项。 + +--- + +## 理由 + +### 为什么不把 mapper 单独列 G12.9 grep + +考虑过： + +```makefile +@echo "G12.9 (coinglass mapper exit)..." && grep -qE "FormatFloat\(.*'f'" internal/repo/webapi/coinglass/mapper.go +``` + +否决：grep 只能证明"有 FormatFloat 调用"，不能证明"每个 float64 字段都过了 FormatFloat"。真正能验证后者的是 table-driven 测试断言。把"伪安全"的 grep 加进 Makefile 反而稀释 G12 的可信度——所以 D3 选择"G12.4 + 单测"双兜底，明确拒绝在 Makefile 加 mapper 专用 grep。 + +### 为什么不为 mapper 引入 decimal + +CoinGlass JSON 数值上限远低于 ADR-0002 论证过的 1e6 USD 量级：funding rate ~ ±0.01、OI 数量 ~ 1e10 BTC、清算量 ~ 1e9 USD。IEEE-754 binary64 显著位数 ~15.95 decimal digits，绝对误差 << CoinGlass 自身的精度承诺（它的源数据来自交易所，再聚合一道）。引入 `shopspring/decimal` 在 mapper 这种 hot path（每个 endpoint 每周期数十到数百个数值字段）反而是性能税。 + +### 为什么单开 ADR 而不是改 ADR-0002 + +ADR-0002 已 Accepted，且 ADR 是不可变记录。扩白名单 = 新决策，按 he-maintainer 模式开 ADR-0004 链接回 ADR-0002 更清晰；将来若再有第三个数据源走类似路径（V3 phase-2 ETF Flow / CME），可继续开 ADR-0005，链路可追。 + +### 与 spec §6.3 的关系 + +spec `ai/specs/coinglass.md` §6.3 已经口头描述了 G12.1 扩白名单 + 新增 G12.7/G12.8。本 ADR 是这条 spec 子条款的**架构层背书**——把"为什么这样扩、不这样扩的边界在哪"固化下来，避免 PR-1 实施时 reviewer 来问"为什么 mapper 不也加 G12.9"再重新讨论。 + +--- + +## 代价 + +- G12 白名单从 2 个 token（`indicator|derivatives_signal`）变 3 个（`+coinglass`）。每多一个 token，"白名单越扩越松"的诱惑越大；本 ADR 明确：**只有 ADR 批准过的形态可以加 token**。未来增源前必须先开 ADR 再改 Makefile。 +- mapper 的 float64 边界靠 entity grep（G12.4）+ unit test 双兜底，没有 Makefile grep 直接守。若 mapper 单测被改/删而 entity 没破，理论上能漏过一段时间——缓解：mapper_test.go 在 PR-1 的 test plan 里列为 must-have，且未来 G12.4 一旦触发立即定位到 mapper 出口未 FormatFloat。 +- 比"全栈 decimal"方案差的精度：依 IEEE-754 累积误差。CoinGlass 数据用作下游 Hermes 量化信号的"参考量级"而非"成交价 / 落库价"，可接受；若某天 CoinGlass 数据要进入交易决策的资金计算路径，本 ADR 需重评（见下）。 + +--- + +## 何时重新评估 + +- 新增第三类 webapi mapper 走"JSON float64 → string entity"路径 → 评估是否升级到通用机制（codegen mapper 或 wrapper helper），而不是逐个 ADR。 +- CoinGlass 数据进入资金/订单决策（目前 spec 明确不进）→ 直接复用 ADR-0002 的升级路径：换 decimal 或上 Kahan 求和。 +- G12.1 白名单扩到第 5 个 token → 触发"白名单失控"复评，可能改成"显式列文件名清单"代替 grep pattern。 +- `coinglass_query.go` 出现 float64（PR review 抓到或 G12.1 漏过）→ 立即添加 G12.9 单独守 query 文件（grep `float64` 内嵌 `coinglass_query.go` 必须空）。 + +--- + +## 与其它决策的关系 + +- **ADR-0001 D5（全链路 string）**：CoinGlass entity 全 string（spec D6），严格沿用，无冲突。 +- **ADR-0002（指标数值边界）**：本 ADR 是其在 v3 数据源场景的**白名单扩展 + 边界形态新增**，不修改 ADR-0002 的核心决策。 +- **ADR-0003（per-interval 技术结构）**：无直接关系；CoinGlass 数据未来若并入 `TechnicalStructure.Intervals` 是另一份 ADR 的议题。 +- **守卫 G12**（risk-guardrails.md:196）：本 ADR 落地后，risk-guardrails.md 的 G12 章节需要同步：白名单文件清单加 `coinglass_signal.go`，子规则列表加 G12.7 / G12.8。该同步在 PR-1 落地 Makefile 改动时一起做。 +- **`ai/specs/coinglass.md` §6.3 / §16**：spec §16 已经把"暂列 ADR-0004"提前注册，本 ADR 是兑现该承诺。 diff --git a/ai/risk-guardrails.md b/ai/risk-guardrails.md index c6282f6..63cbf6f 100644 --- a/ai/risk-guardrails.md +++ b/ai/risk-guardrails.md @@ -179,9 +179,9 @@ grep -rn --include="*.go" "websocket\|fstream.binance" . ## G11. Controller 不编排 derivatives 多源查询 -**规则**：`internal/controller/**` 不得直接持有或调用 `FundingRepository` / `OpenInterestRepository` / `LongShortRatioRepository` / `TakerVolumeRepository`。这类衍生品历史数据的"查哪些表 + 多源拼装 + 错误降级 → warnings"必须在 `internal/usecase/` 完成，controller 只做参数解析、调用 usecase、返回 JSON。 +**规则**：`internal/controller/**` 不得直接持有或调用 `FundingRepository` / `OpenInterestRepository` / `LongShortRatioRepository` / `TakerVolumeRepository` / `CoinglassRepository`。这类衍生品历史数据的"查哪些表 + 多源拼装 + 错误降级 → warnings"必须在 `internal/usecase/` 完成，controller 只做参数解析、调用 usecase、返回 JSON。 -**为什么**：曾经 `/derivatives` handler 一次性编排了 6 处查询、4 个静默吞错的 `_`、硬编码的 limit 和响应结构——HTTP 层在替业务层做决定。要给衍生品加 taker volume、缓存、降级策略、数据质量 warnings 时，应该只动 usecase，不该让 HTTP handler 跟着变厚。 +**为什么**：曾经 `/derivatives` handler 一次性编排了 6 处查询、4 个静默吞错的 `_`、硬编码的 limit 和响应结构——HTTP 层在替业务层做决定。要给衍生品加 taker volume、缓存、降级策略、数据质量 warnings 时，应该只动 usecase，不该让 HTTP handler 跟着变厚。CoinGlass 接入沿用同款约束（v3-PR-1 起）。 **例外**：`KlineRepository` 暂时允许 controller 直查，因为 `/klines` 现在就是 thin query。一旦它要加缓存 / live fallback / interval 聚合，就同样要搬进 usecase。 @@ -189,22 +189,26 @@ grep -rn --include="*.go" "websocket\|fstream.binance" . ```bash grep -rn "FundingRepo\|OIRepo\|LSRepo\|TakerRepo" internal/controller # 期望：无输出 +grep -rn "CoinglassRepo\|coinglassRepo" internal/controller +# 期望：无输出（G11.2，由 make guard 自动跑） ``` --- ## G12. 指标计算的 float64 边界 -**规则**：`float64` 在 `internal/usecase/` 下**只允许出现在纯解析 usecase 文件**（当前为 `indicator.go` 与 `derivatives_signal.go`）内部作为 transient 计算变量；进入 `entity` / 返回上层之前必须 `strconv.FormatFloat(_, 'f', -1, 64)` 转回 string。这些文件同时**禁止** import `internal/repo/...` 的任何子包。 +**规则**：`float64` 在 `internal/usecase/` 下**只允许出现在纯解析 usecase 文件**（当前为 `indicator.go`、`derivatives_signal.go`、`coinglass_signal.go`）内部作为 transient 计算变量；进入 `entity` / 返回上层之前必须 `strconv.FormatFloat(_, 'f', -1, 64)` 转回 string。这些文件同时**禁止** import `internal/repo/...` 的任何子包。 -**为什么**：G2 已经预留"指标计算中间过程可用 float64"的例外，但没有可机械验证的边界。Milestone 6 之后会有更多指标和衍生品信号加入，必须把例外明文化、可 grep 化，避免某天 float64 悄悄出现在 entity 字段或被 repo 持久化。决策细节见 `ai/adr/0002-indicator-numeric-boundary.md`。 +`internal/repo/webapi/coinglass/mapper.go`（webapi 层）允许 transient `float64`（解 JSON number 默认 float64），不在 G12.1 grep 路径内；其安全边界靠 G12.4（entity 全 string，CoinGlass entity 严格沿用 ADR-0001 D5）+ `mapper_test.go` table-driven 双兜底。详见 `ai/adr/0004-coinglass-numeric-boundary-extension.md` D3。 -**新增同形态文件时**：必须同步扩 Makefile 的 G12.1 白名单与 G12.5/6 子规则（参考 `derivatives_signal.go` 接入的方式），并把文件名加入本规则的"当前为"括注。 +**为什么**：G2 已经预留"指标计算中间过程可用 float64"的例外，但没有可机械验证的边界。Milestone 6 之后会有更多指标和衍生品信号加入，必须把例外明文化、可 grep 化，避免某天 float64 悄悄出现在 entity 字段或被 repo 持久化。决策细节见 `ai/adr/0002-indicator-numeric-boundary.md` 与 `ai/adr/0004-coinglass-numeric-boundary-extension.md`。 + +**新增同形态文件时**：必须同步扩 Makefile 的 G12.1 白名单与 G12.5/6/7/8 子规则（参考 `derivatives_signal.go`、`coinglass_signal.go` 接入的方式），并把文件名加入本规则的"当前为"括注。**只有 ADR 批准过的形态可以加 token**（ADR-0004 决策）。 **怎么验证**（由 `make guard` 自动跑）： ```bash # G12.1：usecase 下除白名单文件外不得出现 float64 -grep -rn --include="*.go" "float64\|float32" internal/usecase | grep -vE "indicator|derivatives_signal" +grep -rn --include="*.go" "float64\|float32" internal/usecase | grep -vE "indicator|derivatives_signal|coinglass" # 期望：无输出 # G12.2/G12.3：indicator.go 不得 import repo 子包，也不得 import binance/postgres/pgx @@ -217,6 +221,11 @@ grep -n "internal/repo" internal/usecase/derivatives_signal.go grep -nE "binance|postgres|pgx" internal/usecase/derivatives_signal.go # 期望：无输出 +# G12.7/G12.8：coinglass_signal.go 同上（文件不存在时空跑） +grep -n "internal/repo" internal/usecase/coinglass_signal.go +grep -nE "binance|postgres|pgx" internal/usecase/coinglass_signal.go +# 期望：无输出 + # G12.4：entity 字段任何位置不得 float grep -rn --include="*.go" "float64\|float32" internal/entity # 期望：无输出 diff --git a/ai/specs/coinglass.md b/ai/specs/coinglass.md new file mode 100644 index 0000000..cdad9e0 --- /dev/null +++ b/ai/specs/coinglass.md @@ -0,0 +1,546 @@ +# spec — CoinGlass V2 数据源接入（v3 phase-1） + +**状态**：Accepted（D1–D7 全锁，§17 用户确认 2026-05-25；等 v3-PR-1 实施） +**日期**：2026-05-25 +**作者**：cryptoHermes 维护者 +**适用范围**：`internal/repo/webapi/coinglass/`（新）、`internal/entity/coinglass.go`（新）、`internal/usecase/coinglass_*.go`（新）、`internal/controller/restapi/v1/coinglass_routes.go`（新）、`internal/worker/coinglass_collector.go`（新）、`config/config.go`（扩 v3 段） + +参考： +- `docs/coinglass/cg.md`：V2 反混淆笔记（6 分支 dispatcher + 双阶段解密 + `data=` 签名） +- `docs/coinglass/coinglass_client.py`：Python 参考实现（2026-05 验证可跑） +- `docs/coinglass/endpoint.json`：209 endpoint 存活报告（154 ok） +- `ai/harness-health.md` line 84：新数据源 → 必填本 spec +- `README.md` 路线图 v3 段：CoinGlass + ETF Flow + CME gap + … 多源聚合 + +--- + +## 1. 目标 + +把 CoinGlass V2 私有 API 中**跨交易所聚合**的 4 个核心信号接入 cryptoHermes，作为 v3 的第一波交付。结束后下游 Hermes 量化引擎能从 `/v1/coinglass/*` 拿到 Binance 单交易所视角看不到的数据（清算热力图 + 多交易所 OI / 多空比 / funding 聚合）。 + +**为什么是 CoinGlass 而不是 ETF Flow / CME / 链上**： + +- 反混淆窗口期已开（cg.md + Python ref 都已在 2026-05 验证），错过这窗口下次 CoinGlass 版本号变（V3）又要重新逆向。 +- 4 个 endpoint 一次拿下，覆盖 README v3 段里的"多交易所聚合"子项；ETF Flow / CME gap 在 CoinGlass 自己的 endpoint 里也有，phase-2 直接复用本 phase 的 client 即可。 +- 链上 / 宏观日历是不同性质的数据源（不需要 TLS 指纹绕过、不需要私有解密），独立铺设更经济。 + +--- + +## 2. Phase 1 endpoint 范围（**用户锁定 2026-05-25**） + +| Path | 用途 | 参考 params | +|---|---|---| +| `/api/index/v2/liqHeatMap` | 清算热力图（价格 × 时间 × 清算量） | `merge=true symbol=Binance_BTCUSDT interval=5 limit=288` | +| `/api/openInterest/v3/chart` | 多交易所 OI 历史 | `symbol=BTC interval=h1 timeType=1 type=0` | +| `/api/futures/longShortRate` | 跨交易所多空比 | `symbol=BTC interval=h1 timeType=1` | +| `/api/fundingRate/v2/home` | 多交易所 funding 聚合 | `(空)` | + +**非目标（phase 2+）**：ETF Flow、CVD、CME gap、宏观日历、链上稳定币流动性、其它 200 个 CoinGlass endpoint 中的 150 个。 + +--- + +## 3. 决策锁定 + +| ID | 决策 | 锁定时间 | 备注 | +|---|---|---|---| +| **D1** | TLS 指纹库用 **github.com/enetx/surf v1.0.144** | 2026-05-25（用户，后经 surf spike 修订） | `surf` Firefox profile 已真打 CoinGlass liqHeatMap：能拿到 `v/user/data` 并解出 plaintext；pin v1.0.144，避免 latest 抬 Go 1.25 | +| **D2** | Phase 1 = 4 个 endpoint（见 §2） | 2026-05-25（用户） | 全部对应 Python ref `__main__` smoke test | +| **D3** | TOTP_SECRET 走 **config 注入**，env `COINGLASS_TOTP_SECRET`，`enabled=true` 时必填 | 2026-05-25（作者） | 公开 bundle 里能抠出 → 不算账户 secret；但 CoinGlass 可能旋转，env 化保旋转友好；Go client 不提供代码默认值 | +| **D4** | 新增 `/v1/coinglass/*` namespace，**不**折进 `/v1/market/context` | 2026-05-25（作者） | CoinGlass 故障模式（fake-success / 密钥旋转 / TLS 失效）独立，要求物理隔离；下游 Hermes 可独立降级 | +| **D5** | 每个 endpoint 独立 PG 表 + 独立 cron 周期 | 2026-05-25（作者） | 热力图频次需要更细（5min）、funding 聚合更粗（30min）；schema 形状差异大 | +| **D6** | 解密管线产生的 `float64` 数值字段，在 repo 出口处全部 `strconv.FormatFloat` 转 string 进 entity | 2026-05-25（作者） | 沿用 ADR-0001 D5 全链 string + ADR-0002 数值边界白名单语义。详见 §6.3 | +| **D7** | 新增独立 `CoinglassProvider` interface，**不**扩 `DerivativesProvider` | 2026-05-25（作者） | 失败语义不同（fake-success retry），返回类型不同（聚合后的 list 不是单 symbol） | + +D1 / D2 由用户锁定，D3-D7 由作者基于现有架构推断；任何一条用户不同意都可单点回旋。 + +--- + +## 4. 架构 + +### 4.1 分层（Clean Arch 边界沿用） + +``` +controller/restapi/v1/coinglass_routes.go + │ + ▼ +usecase/coinglass_query.go ← 只读路径：repo 拉历史 +usecase/coinglass_signal.go ← 纯函数：聚合多交易所 OI/funding → 语义信号 + │ + ▼ +repo/persistent/postgres/coinglass_*.go ← 5 个新表的 UpsertMany + FindRecent +repo/webapi/coinglass/ ← V2 client + 4 个 endpoint mapper + │ + ▼ +worker/coinglass_collector.go ← cron：周期性 fetch → upsert +``` + +**G6（边界）守卫不变**：usecase 不导入 `repo/webapi/coinglass` 或 `repo/persistent/postgres`，只对 ports。 +**G12（数值边界）守卫扩白名单**：`coinglass_signal.go` 类比 `derivatives_signal.go`，允许 `float64`；其余 usecase 文件仍禁。 + +### 4.2 ports（新） + +```go +// internal/usecase/ports.go 追加 +type CoinglassProvider interface { + GetLiqHeatMap(ctx context.Context, symbol string, intervalMin int, limit int) (*entity.CGLiqHeatMap, error) + GetOpenInterestChart(ctx context.Context, symbol, interval string) (*entity.CGOpenInterestChart, error) + GetLongShortRate(ctx context.Context, symbol, interval string) (*entity.CGLongShortRate, error) + GetFundingRateHome(ctx context.Context) (*entity.CGFundingHome, error) +} + +type CoinglassRepository interface { + UpsertLiqHeatMap(ctx context.Context, m *entity.CGLiqHeatMap) error + FindRecentLiqHeatMap(ctx context.Context, symbol string, limit int) ([]entity.CGLiqHeatMap, error) + UpsertOpenInterestChart(ctx context.Context, m *entity.CGOpenInterestChart) error + FindRecentOpenInterestChart(ctx context.Context, symbol, interval string, limit int) ([]entity.CGOpenInterestChart, error) + UpsertLongShortRate(ctx context.Context, m *entity.CGLongShortRate) error + FindRecentLongShortRate(ctx context.Context, symbol, interval string, limit int) ([]entity.CGLongShortRate, error) + UpsertFundingHome(ctx context.Context, m *entity.CGFundingHome) error + FindRecentFundingHome(ctx context.Context, limit int) ([]entity.CGFundingHome, error) +} +``` + +4 endpoint × (Upsert + FindRecent) = 8 repo 方法。每个独立表（D5）。 + +### 4.3 V2 dispatcher 6 分支 — 必须全实现 + +cg.md §"Known branches (2026-05)" 写得很清楚：服务器**per-response**挑 `v`，同一 endpoint 连续两次调用 v 可能不同。客户端必须能处理所有 6 个分支，**不能 pin**。 + +| `v` | seed 来源 | 已确认 | +|---|---|---| +| `"0"` | `base64("coinglass"+path+"coinglass")[:16]` | V1-compat，需要时再启 | +| `"1"` | `path`（待验证派生形状） | cg.md 标"shape — verify" | +| `"2"` | `response.headers.time` | V2 实战出现 | +| `"55"` | `"170b070da9654622"` | bundle 常量 | +| `"66"` | `"d6537d845a964081"` | bundle 常量 | +| `"77"` | `"863f08689c97435b"` | bundle 常量 | + +实现要点（沿用 Python ref `_resolve_seed`）： + +```go +func resolveSeed(v, path, reqCacheTsV2, respTime string) (string, error) { + switch v { + case "0": return reqCacheTsV2, nil + case "1": return path, nil + case "2": return respTime, nil + case "55": return "170b070da9654622", nil + case "66": return "d6537d845a964081", nil + case "77": return "863f08689c97435b", nil + default: return "", fmt.Errorf("unknown coinglass v=%q (bundle may have changed)", v) + } +} +``` + +未知 `v` → 返回错误（不重试不降级），让 collector log 一行 warn，**等人工 review bundle**。这是已知"以后会出现的故障模式"，预先把 hook 留好。 + +### 4.4 双阶段解密（universal pipeline） + +``` +Stage 1: user_key = base64(seed)[:16] （16 字节 ASCII） + token = strip_quotes(gunzip(unpad(aes_ecb_decrypt(b64dec(headers["user"]), user_key)))) +Stage 2: body_key = token[:16]（**原 ASCII**，**不再 base64**） + plain = strip_quotes(gunzip(unpad(aes_ecb_decrypt(b64dec(body["data"]), body_key)))) + result = json.Unmarshal(plain) +``` + +Go 实现位置：`internal/repo/webapi/coinglass/crypto.go`，纯函数，与 fetch 解耦，方便单测喂样本。 +依赖：stdlib `crypto/aes` + `crypto/cipher`（ECB 自己拼 block 循环，stdlib 没有 ECB 模式）+ `compress/gzip` + `encoding/base64`。零第三方。 + +PKCS#7 unpad 自己写 8 行；ECB decrypt 自己写 ~15 行（`cipher.Block.Decrypt(dst, src)` 一块一块走）。 + +### 4.5 请求侧 `data=` 签名（TOTP） + +``` +ts = unix_seconds_now() +totp_code = TOTP(secret=COINGLASS_TOTP_SECRET, period=30s, digits=6).At(ts) +plaintext = fmt.Sprintf("%d,%s", ts, totp_code) +cipher = AES-256-ECB-PKCS7(DATA_AES_KEY) +data_b64 = base64(cipher.Encrypt(pad(plaintext))) +url ?= ...&data= +``` + +依赖：`github.com/pquerna/otp/totp`（Go TOTP 事实标准）。 +`DATA_AES_KEY` = `"1f68efd73f8d4921acc0dead41dd39bc"`（32 字节 ASCII，AES-256），从 bundle 抠出，作为代码常量；与 TOTP_SECRET 不同 — DATA_AES_KEY 旋转概率显著低，且即便旋转也是同步换的。仍保留 env override `COINGLASS_DATA_AES_KEY` 作为 escape hatch。 + +### 4.6 TLS 指纹（D1 = github.com/enetx/surf v1.0.144） + +```go +client := surf.NewClient().Builder(). + Impersonate().FireFox(). // v1.0.144 Firefox/144 profile + Timeout(15 * time.Second). + Build() +``` + +请求头按 `coinglass_client.py` `_base_headers` 复写，但 `User-Agent` 由 surf Firefox profile 负责，避免 UA 与 TLS 指纹版本脱节。`Origin / Referer = https://www.coinglass.com`，`Encryption: true`，`Cache-Ts-V2: `。 + +2026-05-25 spike 结果：`surf@v1.0.144` 在 Go 1.24.1 下真打 `/api/index/v2/liqHeatMap?merge=true&symbol=Binance_BTCUSDT&interval=5&limit=288`，返回 `status=200`、`v=0`、`user` header、非空 `data`，并可用 §4.4 双阶段解密得到 plaintext。`surf@latest v1.0.200` 也可用，但会把模块 `go` 版本抬到 1.25.0，PR-1 不采用。 + +**Fake-success 检测**（cg.md §"fake success trap"）： + +判定（在 client 层，先于解密）： +``` +resp.status == 200 AND body.success == true AND body.data IS empty/null + → fake-success → 等 2s 重试 1 次 → 仍 fake → 抛 ErrFakeSuccess +``` + +不区分"业务 error（success=false + 明确 msg）"与"fake-success"，因为业务 error 调 retry 没用。两者必须明确分类。 + +### 4.7 配置（`config/config.go` 扩段） + +```yaml +coinglass: + enabled: false # 默认关，老 v1/v2 用户零侵入 + base_url: "https://capi.coinglass.com" + totp_secret: "${COINGLASS_TOTP_SECRET}" # env 注入，必填（enabled=true 时） + data_aes_key: "${COINGLASS_DATA_AES_KEY:-1f68efd73f8d4921acc0dead41dd39bc}" + tls_profile: "surf_firefox" # 当前实现固定 surf v1.0.144 Firefox/144 profile + request_timeout_sec: 15 + fake_success_retry: 1 + fake_success_retry_wait_ms: 2000 + collectors: + liq_heatmap_interval_sec: 300 # 5 min + oi_chart_interval_sec: 600 # 10 min + long_short_rate_interval_sec: 600 + funding_home_interval_sec: 1800 # 30 min + startup_jitter_max_sec: 30 # 启动时随机延迟 [0, max)，避免整点齐打 + backoff_initial_sec: 30 # 失败后退避起点 + backoff_max_sec: 600 # 退避上限 + single_flight: true # 任一 endpoint 不允许并发 fetch +``` + +**cron 行为硬约束（§9 Collector 实现必须满足）**： + +1. **不重叠（single-flight）**：同一 endpoint 上一次 fetch 未结束前，下一次 tick 跳过（log skip 一行），不堆积。 +2. **失败退避**：fetch 失败（任意错误类）后按 `backoff_initial_sec` 起步 ×2 增长到 `backoff_max_sec` 封顶；成功一次后重置回 interval。 +3. **启动 jitter**：每个 collector 启动时 sleep `rand.Intn(startup_jitter_max_sec)` 秒再开始第一次 tick，避免 4 endpoint 在整点齐打 CoinGlass。 + +PR-1 范围内：**只启 liqHeatMap 的 cron**（其它 3 个 endpoint cron 留到 v3-PR-2 接入时一起启）。但 §4.7 的 config schema 与 §9 的 backoff/jitter/single-flight 框架必须在 PR-1 落地，避免 PR-2 还要回头改 config 结构。 + +`enabled: false` 默认保护：未配置 TOTP_SECRET 时 cron 不启动、controller 路由 503。**G1（无私钥）守卫不变**：TOTP_SECRET 不是账户级签名，是 CoinGlass 站点级的反爬密钥，**不签账户行为**，不需要进 G1 灰名单。但 spec 内对此事实必须记下来： + +> CoinGlass TOTP_SECRET 是站点反爬签名密钥（用于 `data=` 参数），与"账户级 API 签名"性质不同，不在 G1 守卫范围内。其唯一暴露面是 CoinGlass 反爬规则，泄露不影响用户资产。`AGENTS.md §"绝不做"`不变。 + +--- + +## 5. 数据模型（`internal/entity/coinglass.go`） + +ADR-0001 D5：所有数值字段全链 `string`。下文 *type* 都是 string 除非显式标注。 + +```go +package entity + +// CGLiqHeatMap — /api/index/v2/liqHeatMap 响应建模。 +// 注意：CoinGlass 返回的 prices/volumes 是 2D 网格，要展平成行存或保 JSONB； +// 这里建议保 JSONB（一次拉一帧，按 Symbol+Timestamp 单行）。 +type CGLiqHeatMap struct { + Symbol string `json:"symbol"` // "Binance_BTCUSDT" + Interval string `json:"interval"` // "5"（分钟） + Timestamp int64 `json:"timestamp"` // 拉取时间（ms） + PriceMin string `json:"priceMin"` + PriceMax string `json:"priceMax"` + // RawGrid 是 CoinGlass 原 JSON 序列化字节；下游需要时再解析。 + // 不在 Go entity 层重建价格×时间二维网格，避免不必要 schema 锁定。 + RawGrid []byte `json:"rawGrid"` +} + +// CGOpenInterestChart — 多交易所 OI 时序。一帧多交易所。 +type CGOpenInterestChart struct { + Symbol string `json:"symbol"` // "BTC" + Interval string `json:"interval"` // "h1" + Timestamp int64 `json:"timestamp"` + Exchanges []CGExchangeOIPoint `json:"exchanges"` +} + +type CGExchangeOIPoint struct { + Exchange string `json:"exchange"` // "Binance" / "OKX" / "Bybit" ... + OI string `json:"oi"` // USD value +} + +// CGLongShortRate — 多交易所多空比。 +type CGLongShortRate struct { + Symbol string `json:"symbol"` + Interval string `json:"interval"` + Timestamp int64 `json:"timestamp"` + Exchanges []CGExchangeLongShortPoint `json:"exchanges"` +} + +type CGExchangeLongShortPoint struct { + Exchange string `json:"exchange"` + LongRate string `json:"longRate"` // 百分比，如 "55.32" + ShortRate string `json:"shortRate"` // 百分比 + Ratio string `json:"ratio"` // long/short +} + +// CGFundingHome — 多交易所 funding 聚合（home page format）。 +type CGFundingHome struct { + Timestamp int64 `json:"timestamp"` + Items []CGFundingHomeItem `json:"items"` +} + +type CGFundingHomeItem struct { + Symbol string `json:"symbol"` // "BTCUSDT" + Exchange string `json:"exchange"` // "Binance" + FundingRate string `json:"fundingRate"` // e.g. "0.0001" + NextFundingTs int64 `json:"nextFundingTs"` + Interval string `json:"interval"` // "8h" / "4h" +} +``` + +**确认前提**：上面字段名是基于 Python ref 4 个 smoke endpoint 的"预期形状"。**真 schema 必须等 Go 端跑通 client + dump 一次解密后的 JSON 才能定**。spec 阶段先占位；PR-1 spike 跑通后回填本节并把 spec 标 Accepted。 + +### 5.1 JSONB 字段使用纪律（强约束） + +`CGLiqHeatMap.RawGrid` 是本 spec 唯一允许的 JSONB 字段。约束如下： + +1. **JSONB 只承载供应商原始网格 / 元信息**：CoinGlass 返回的 2D 价格×时间×清算量网格因 schema 频繁微调且行数爆炸（一帧 288 × N 价位），关系化得不偿失，故保 JSONB。 +2. **usecase 层禁止直接解析 RawGrid 任意 JSON 结构**：如 `coinglass_signal.go` 需要"清算密集价位"等派生字段，必须先在 `mapper.go` 里展开成 entity 字段（如未来加 `LiqCluster []CGLiqLevel`），再由 usecase 消费。`json.Unmarshal(RawGrid, &map[string]any{})` 这种 ad-hoc 解析**禁止出现在 usecase**。 +3. **JSONB → 稳定 entity 字段的演进路径**：当某个派生指标稳定（≥2 个 PR 反复用）时，mapper 加一个显式字段；不要让 RawGrid 的形状沿伸到 controller / 下游 Hermes 合约。 +4. **其它 3 个 entity（OI / LSR / Funding）保持拍平结构**，不引入新 JSONB 字段。新增 JSONB 字段需开 ADR 锁定。 + +--- + +## 6. Repo 层（`internal/repo/webapi/coinglass/`） + +### 6.1 目录结构 + +``` +internal/repo/webapi/coinglass/ +├── client.go # surf wrapper + data= TOTP signing + dispatch GET + retry +├── crypto.go # AES-ECB + PKCS7 + gzip，零外部依赖 +├── crypto_test.go # 喂 fixed ciphertext + key → 解出已知 plaintext +├── dispatcher.go # resolveSeed 6 分支 +├── mapper.go # CoinGlass JSON → entity（per endpoint） +├── mapper_test.go # table-driven +├── liq_heatmap.go # GetLiqHeatMap +├── oi_chart.go # GetOpenInterestChart +├── long_short.go # GetLongShortRate +└── funding_home.go # GetFundingRateHome +``` + +### 6.2 错误分类（client 层导出） + +```go +var ( + ErrFakeSuccess = errors.New("coinglass fake-success (TLS fingerprint suspected)") + ErrUnknownV = errors.New("coinglass unknown v branch") + ErrDecryptFail = errors.New("coinglass decrypt pipeline failed") + ErrAPIBusiness = errors.New("coinglass API business error") // success=false + msg +) +``` + +Collector 见到 `ErrFakeSuccess` 连发 3 次 → 全局 disable 1 小时 + log alert（避免 IP 进黑名单）。 + +### 6.3 数值边界（G12 白名单扩张） + +`internal/repo/webapi/coinglass/mapper.go` 内部允许 transient `float64`（解 JSON 时 number 默认 float64），但 mapper 出口（写入 entity）前必须 `strconv.FormatFloat(f, 'f', -1, 64)` 回 string。 + +> **G12 守卫扩白名单**：Makefile `guard` 目标的 G12.1 规则增加 `coinglass` 排除： +> ```makefile +> @echo "G12.1 (usecase float)..." && ! { grep -rn --include="*.go" "float64\|float32" internal/usecase | grep -vE "indicator|derivatives_signal|coinglass" | grep -q . ; } +> ``` +> 同时新增 G12.7 / G12.8 覆盖：`coinglass_signal.go` 与 `derivatives_signal.go` 同款（no repo / no DB import）。 +> +> **G12.4（entity 不能含 float）**：CoinGlass entity 全 string，G12.4 不破。 + +### 6.4 测试策略 + +| 文件 | 测试范围 | 工具 | +|---|---|---| +| `crypto_test.go` | 喂 fixed `(key, ciphertext)` → 期望 plaintext；PKCS7 unpad 边界（0/16 块大小） | table-driven + testify | +| `dispatcher_test.go` | 6 个 `v` 分支各一例 + unknown v → ErrUnknownV | table-driven | +| `mapper_test.go` | 喂 fixed JSON（从 Python ref dump 出来）→ 期望 entity 字段；NaN/Inf/缺字段处理 | table-driven，fixture 放 `testdata/` | +| `client_test.go` | 用 `httptest.NewServer` 起 fake CoinGlass，验证 `data=` TOTP 签名、retry / fake-success 分类 | httptest（不需要 surf 真起 Firefox 指纹，测的是上层逻辑） | +| `integration_test.go`（build tag `integration_coinglass`，**默认关**） | 真打 CoinGlass，仅在手动 + nightly 跑；不进 push CI | CGO=0 即可 | + +PR-C 已经被跳，但 CoinGlass integration test 因为打外部服务，**必须**有 build tag 隔离。这是 D7 顺带的硬约束。 + +--- + +## 7. Usecase（`internal/usecase/coinglass_*.go`） + +### 7.1 `coinglass_query.go`（薄） + +只读路径：给 controller 调，从 repo 拉历史，组装 response，**不做计算**。 + +```go +type CoinglassQueryUsecase struct { + repo CoinglassRepository + log *slog.Logger +} + +func (u *CoinglassQueryUsecase) GetLiqHeatMap(ctx context.Context, symbol string, limit int) ([]entity.CGLiqHeatMap, error) { ... } +// ... 4 个 endpoint 各一个 Get +``` + +### 7.2 `coinglass_signal.go`（厚，纯函数） + +类比 `derivatives_signal.go`：把多交易所原始数据翻译成可读语义。 + +```go +// CoinglassSignal 把 4 个 endpoint 的原始数据聚合成上游 Hermes 直接消费的语义字段。 +type CoinglassSignal struct { + OIDominance string `json:"oiDominance"` // e.g. "binance:0.42,okx:0.28,bybit:0.18" + OIShiftBias string `json:"oiShiftBias"` // accumulating_binance / fleeing_to_okx / balanced + LSRDispersion string `json:"lsrDispersion"` // narrow / wide — 交易所间多空比方差 + FundingExtreme string `json:"fundingExtreme"` // 哪个 exchange funding 是 outlier + LiqClusterPrice string `json:"liqClusterPrice"` // 清算热力图里最密集的价位（中位数） +} + +type CoinglassSignalComputer interface { + Compute( + heat *entity.CGLiqHeatMap, + oi *entity.CGOpenInterestChart, + lsr *entity.CGLongShortRate, + fund *entity.CGFundingHome, + ) (*CoinglassSignal, []string) // signal + warnings +} +``` + +**这是 phase 1 真正的"算法增量"**。前面 4 个 endpoint 是底料；这里把"7 个交易所 OI 谁在涨"翻成"oi_shift_bias=accumulating_binance"才是给 Hermes 看的成品。 + +### 7.3 `MarketContextUsecase` 增强（可选 phase-1.5） + +ADR-0003 的 mirror 思路可以借用：`/v1/market/context` 增一个 `coinglass *CGSummary` 字段（可选，omitempty），把 `CoinglassSignal` 嵌入。**不在 phase-1 必做范围**；phase-1 先把 `/v1/coinglass/*` 独立起来，跑稳了再考虑要不要折回 context。 + +--- + +## 8. Controller（`internal/controller/restapi/v1/coinglass_routes.go`） + +``` +GET /v1/coinglass/liq-heatmap?symbol=BTCUSDT&limit=288 +GET /v1/coinglass/open-interest?symbol=BTC&interval=h1&limit=200 +GET /v1/coinglass/long-short?symbol=BTC&interval=h1&limit=200 +GET /v1/coinglass/funding-home +GET /v1/coinglass/signal?symbol=BTC ← 聚合 4 endpoint 的 latest，调 CoinglassSignalComputer +``` + +`coinglass.enabled=false` 时全部返 503 `service-unavailable: coinglass disabled in config`。 +G11（controller 不直依 derivatives repo）类比：controller 不直依 CoinglassRepository，只调 CoinglassQueryUsecase。新增 G11.2 守卫覆盖： + +```makefile +@echo "G11.2 (ctrl coinglass)..." && ! grep -rqn "CoinglassRepo\|coinglassRepo" internal/controller +``` + +--- + +## 9. Worker / Collector + +`internal/worker/coinglass_collector.go`：4 个独立 cron（间隔见 §4.7 config）。复用 `internal/worker/collector.go` 的 ticker 模式。 + +每个 endpoint 错误隔离 — 一个 endpoint 持续 fake-success 不影响其它 3 个的轮询。`ErrFakeSuccess` 连 3 次 → 该 endpoint disable 1h + log error。 + +--- + +## 10. 迁移与回滚 + +### 10.1 schema 迁移 + +新增 4 个表 + 0 个旧表改动： + +``` +migrations/00XX_create_coinglass_liq_heatmap.up.sql +migrations/00XX_create_coinglass_open_interest_chart.up.sql +migrations/00XX_create_coinglass_long_short_rate.up.sql +migrations/00XX_create_coinglass_funding_home.up.sql +``` + +每个表对应一个 down.sql。`migrations/` 已有 5 张 Binance 表；编号顺延。 + +### 10.2 回滚策略 + +- 配置层关：`coinglass.enabled=false` → cron 不跑、路由 503，但表保留。运维零代码回滚。 +- 代码层关：revert PR-1 起的 4 个 commit；表保留（下次启用时重新 collect 填）。 + +--- + +## 11. 失败模式 & 检测 + +| 症状 | 含义 | 处置 | +|---|---|---| +| 持续 HTTP 200 + `success:true` + 无 data | TLS 指纹漂移 / IP 进灰名单 | 该 endpoint disable 1h + log alert；运维换 IP / 升级 surf profile | +| `ErrUnknownV` | CoinGlass bundle 出了新 v 分支 | log + alert；人工 re-read bundle + 加分支；spec §4.3 表追加 | +| `ErrDecryptFail`（gzip / unpad / utf-8 任一失败） | 密钥派生错 / 算法变 | log + alert；先重试 1 次（可能是网络截断）；仍失败 → human inspect dump | +| 业务 `success:false code=40001` | params 错（如缺 symbol） | 不重试，直接 propagate 出错（编码 bug） | +| 业务 `success:false code=40000` | 之前 fake-success 后被风控 | 同 fake-success 处置 | +| 静默 schema 变（新字段 / 字段重命名） | CoinGlass 后端微调 | mapper 容忍（按 key 选字段，不依顺序）；新字段 phase-2 决定要不要接 | + +监控 metric（**phase-1.5 落地，不阻塞 phase-1**）： +- `coinglass_request_total{endpoint, outcome=ok|fake_success|business_err|unknown_v|decrypt_fail}` +- `coinglass_endpoint_last_success_age_seconds{endpoint}` + +--- + +## 12. 守卫扩张总结 + +| 守卫 | 改动 | +|---|---| +| G1（无账户签名） | 不变。TOTP_SECRET 不在范围（站点反爬密钥 ≠ 账户签名密钥），spec §4.7 已记 | +| G6（边界） | 不变。新 repo 仍只被 usecase 通过接口调 | +| G11（controller 不直依 repo） | 新增 G11.2 = controller 不见 `CoinglassRepo` | +| G12.1（usecase float 白名单） | 扩 `coinglass`：`grep -vE "indicator\|derivatives_signal\|coinglass"` | +| G12.7（新） | `coinglass_signal.go` 不 import `internal/repo` | +| G12.8（新） | `coinglass_signal.go` 不 import `binance/postgres/pgx` | +| G12.9（新） | `internal/repo/webapi/coinglass/mapper.go` 出口前必须 `formatPrice`/`FormatFloat`（grep 模式：mapper.go 内 `float64` 必须与 `FormatFloat` 共现） — 可选，phase-1 先靠 mapper_test 覆盖 | +| G13.x | 不变 | + +--- + +## 13. PR 切分（建议，可调） + +| PR | 范围 | 验收 | +|---|---|---| +| **v3-PR-1** | TLS spike + crypto.go + dispatcher.go + 1 endpoint（liqHeatMap）vertical slice：client / repo / usecase / controller / migration / 单 cron + §4.7 config 全骨架（含 jitter/backoff/single-flight） | **硬门槛**：(a) TLS fake-success 能被正确识别并抛 `ErrFakeSuccess`（fake-success unit test 必过）；(b) `curl /v1/coinglass/liq-heatmap?symbol=BTCUSDT` 返回非空真实响应；(c) 不触碰 `/v1/market/context` 任何字段（v2 合约不动） | +| **v3-PR-2** | 剩 3 endpoint（OI / longShort / fundingHome）+ 各自 mapper + 表 + cron | 4 个 endpoint 都能 `curl` 出数据；4 个 cron 都遵守 single-flight / backoff / jitter | +| **v3-PR-3** | `CoinglassSignalComputer` 实现 + `/v1/coinglass/signal` + signal 单测；如需 RawGrid 派生指标，新增显式 entity 字段（§5.1 纪律） | signal 字段语义稳，单测覆盖 ≥95%；usecase 层零 RawGrid 直接解析 | +| **v3-PR-4** | 守卫扩张落 Makefile + 文档对齐（AGENTS §2、harness-health 复评、README v3 ✅）+ spec 标 Accepted | `make guard` 全过；README v3 段第一项 ✅ | + +> **`/v1/market/context` 折回机制**：phase-1 内**绝不**讨论；CoinglassSignalComputer 稳定后（≥2 周连续运行 + signal 字段被下游 Hermes 实际消费），再开 ADR 评估是否扩 `MarketContext` schema。PR-1/2/3 路径上任何 PR 都不能改 `/v1/market/context` 响应体。 + +--- + +## 14. 非目标（明确写下来） + +- ETF Flow / CME gap / CVD / 链上 / 宏观日历 — phase 2+。 +- WebSocket 推送 — 仍在 README "绝不做（v1 范围外）" 范围，v3 也不开。 +- CoinGlass 账户级私有接口（如 portfolio）— 不接。CoinGlass 没有"账户级"概念，所有数据都是聚合公开；不存在 G1 violations 风险。 +- 历史回填 — Python ref 没做，CoinGlass V2 也没标 history backfill endpoint；phase-1 起从启动那刻开始 collect。 +- TLS profile 自定义（构造非常用浏览器指纹） — phase-1 沿用 `firefox_133`；profile 调优留 phase-2。 + +--- + +## 15. 复评触发器 + +- `firefox_133` 失效（CoinGlass 升级反爬）→ 评估升级 profile 或换 client。 +- 4 个 endpoint 任一持续 7 天 ≥10% fake-success 率 → 评估 IP 池 / 代理策略。 +- CoinGlass 出 V3 dispatcher（即出现 7+ 个 `v` 分支）→ 整段 §4.3 重做。 +- 下游 Hermes 要求把 CoinGlass 折回 `/v1/market/context` → phase-1.5 启动。 + +--- + +## 16. 关联 ADR / 文档 + +- ADR-0001（架构基础）：D5 全链 string，本 spec 严格沿用。 +- ADR-0002（数值边界）：本 spec §6.3 扩白名单到 coinglass，**需要新 ADR**（暂列 ADR-0004 — 等 PR-1 落地前写定）。 +- ADR-0003（per-interval 技术结构）：无直接冲突；CoinGlass 数据可未来作为新 interval/source 接入 `TechnicalStructure` 但本 phase 不做。 +- `ai/risk-guardrails.md`：G11/G12 扩张要同步更新（v3-PR-4）。 +- `docs/coinglass/cg.md`：本 spec 的协议事实底稿，保持单一来源；本 spec 引用而非复述。 +- `docs/coinglass/coinglass_client.py`：本 spec 的算法参考实现，Go 端是 1:1 移植。 + +--- + +## 17. 决策记录（2026-05-25 用户确认，全部 Accepted） + +| # | 决策 | 内容 | +|---|---|---| +| **Q1** | namespace 独立 | 锁 `/v1/coinglass/...` 单独 namespace。`/v1/market/context` 继续 Binance-only，PR-1/2/3 路径上**绝不**碰 v2 合约；后续融合需 CoinglassSignalComputer 稳定后通过显式版本/字段扩展开 ADR 讨论，**不在 phase-1 范围**。 | +| **Q2** | cron cadence 接受 | 5/10/10/30 min（liqHeatMap / OI / LSR / Funding）采用。强制三条实现约束：**single-flight 不重叠 + 失败指数退避 + 启动 jitter**。详见 §4.7 / §9。PR-1 范围内仅启 liqHeatMap 的 cron。 | +| **Q3** | entity 形状方向 OK | RawGrid JSONB 适用于 liqHeatMap（避免行爆炸）；其它 3 个 entity 拍平。string-numeric 全链路保持，零 float64 出 mapper 边界。新增强约束：**JSONB 仅供应商原始网格 / 元信息，usecase 不允许直接解析任意 JSON 结构，需要稳定字段时由 mapper 展开成 entity 字段**（§5.1）。 | +| **Q4** | 保持 4 PR | 不合并 PR-2 进 PR-1。PR-1 已含 TLS spike + 解密 + 1 endpoint + repo/usecase/controller/cron，风险足够大；额外 3 endpoint 塞入会模糊失败归因。按 §13 4 PR 切分推进。 | + +**v3-PR-1 硬门槛**（再强调，对应 §13 PR-1 验收）： + +1. TLS fake-success 识别通过（`ErrFakeSuccess` 单测覆盖 + 真打 CoinGlass 时能正确分类）。 +2. liqHeatMap vertical slice 打通（client → repo → usecase → controller → cron 全链，`curl /v1/coinglass/liq-heatmap` 出真实非空数据）。 +3. v2 合约零变更（`/v1/market/context` 响应体 diff = 0）。 + +任一未达标，PR-1 不合。 diff --git a/config/config.go b/config/config.go index a067b9f..d5f2212 100644 --- a/config/config.go +++ b/config/config.go @@ -10,6 +10,7 @@ type Config struct { App App `yaml:"app"` HTTP HTTP `yaml:"http"` Binance Binance `yaml:"binance"` + Coinglass Coinglass `yaml:"coinglass"` Postgres Postgres `yaml:"postgres"` Collector Collector `yaml:"collector"` } @@ -34,6 +35,22 @@ type Binance struct { Burst int `yaml:"burst" env-default:"40"` } +type Coinglass struct { + Enabled bool `yaml:"enabled" env:"COINGLASS_ENABLED" env-default:"false"` + BaseURL string `yaml:"base_url" env:"COINGLASS_BASE_URL" env-default:"https://capi.coinglass.com"` + TOTPSecret string `yaml:"totp_secret" env:"COINGLASS_TOTP_SECRET"` + DataAESKey string `yaml:"data_aes_key" env:"COINGLASS_DATA_AES_KEY" env-default:"1f68efd73f8d4921acc0dead41dd39bc"` + Timeout time.Duration `yaml:"timeout" env:"COINGLASS_TIMEOUT" env-default:"15s"` + FakeSuccessRetry int `yaml:"fake_success_retry" env:"COINGLASS_FAKE_SUCCESS_RETRY" env-default:"1"` + FakeSuccessRetryWait time.Duration `yaml:"fake_success_retry_wait" env:"COINGLASS_FAKE_SUCCESS_RETRY_WAIT" env-default:"2s"` + LiqHeatMapCron string `yaml:"liq_heatmap_cron" env:"COINGLASS_LIQ_HEATMAP_CRON" env-default:"*/5 * * * *"` + LiqHeatMapInterval string `yaml:"liq_heatmap_interval" env:"COINGLASS_LIQ_HEATMAP_INTERVAL" env-default:"5"` + LiqHeatMapLimit int `yaml:"liq_heatmap_limit" env:"COINGLASS_LIQ_HEATMAP_LIMIT" env-default:"288"` + StartupJitterMax time.Duration `yaml:"startup_jitter_max" env:"COINGLASS_STARTUP_JITTER_MAX" env-default:"30s"` + BackoffInitial time.Duration `yaml:"backoff_initial" env:"COINGLASS_BACKOFF_INITIAL" env-default:"30s"` + BackoffMax time.Duration `yaml:"backoff_max" env:"COINGLASS_BACKOFF_MAX" env-default:"10m"` +} + type Postgres struct { DSN string `yaml:"dsn" env:"POSTGRES_DSN" env-required:"true"` MaxConns int32 `yaml:"max_conns" env-default:"10"` diff --git a/config/config.yml b/config/config.yml index 6e368e0..64d6e86 100644 --- a/config/config.yml +++ b/config/config.yml @@ -15,6 +15,21 @@ binance: rps: 20 burst: 40 +coinglass: + enabled: false + base_url: https://capi.coinglass.com + totp_secret: "" + data_aes_key: 1f68efd73f8d4921acc0dead41dd39bc + timeout: 15s + fake_success_retry: 1 + fake_success_retry_wait: 2s + liq_heatmap_cron: "*/5 * * * *" + liq_heatmap_interval: "5" + liq_heatmap_limit: 288 + startup_jitter_max: 30s + backoff_initial: 30s + backoff_max: 10m + postgres: dsn: postgres://postgres:postgres@localhost:5432/hermes_market?sslmode=disable max_conns: 10 diff --git a/docker-compose.yml b/docker-compose.yml index 0a4e850..0f60160 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -68,6 +68,9 @@ services: APP_PORT: "8080" POSTGRES_DSN: postgres://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set, see .env.example}@postgres:5432/${POSTGRES_DB:-hermes_market}?sslmode=disable BINANCE_BASE_URL: ${BINANCE_BASE_URL:-https://fapi.binance.com} + COINGLASS_ENABLED: ${COINGLASS_ENABLED:-false} + COINGLASS_BASE_URL: ${COINGLASS_BASE_URL:-https://capi.coinglass.com} + COINGLASS_TOTP_SECRET: ${COINGLASS_TOTP_SECRET:-} # 透传代理：国内直连 fapi.binance.com 不稳，详见 AGENTS.md §7.5 HTTP_PROXY: ${HTTP_PROXY:-} HTTPS_PROXY: ${HTTPS_PROXY:-} diff --git a/go.mod b/go.mod index b08e995..bd941b2 100644 --- a/go.mod +++ b/go.mod @@ -1,40 +1,64 @@ module cryptoHermes -go 1.23.0 +go 1.24.1 require ( + github.com/enetx/g v1.0.195 + github.com/enetx/surf v1.0.144 github.com/gofiber/fiber/v2 v2.52.5 github.com/ilyakaznacheev/cleanenv v1.5.0 github.com/jackc/pgx/v5 v5.7.4 github.com/pashagolub/pgxmock/v4 v4.6.0 + github.com/pquerna/otp v1.5.0 github.com/robfig/cron/v3 v3.0.1 - github.com/stretchr/testify v1.9.0 - golang.org/x/sync v0.13.0 - golang.org/x/time v0.6.0 + github.com/stretchr/testify v1.11.1 + golang.org/x/sync v0.19.0 + golang.org/x/time v0.14.0 ) require ( github.com/BurntSushi/toml v1.2.1 // indirect - github.com/andybalholm/brotli v1.0.5 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/andybalholm/brotli v1.2.0 // indirect + github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/google/uuid v1.5.0 // indirect + github.com/enetx/http v1.0.19 // indirect + github.com/enetx/http2 v1.0.20 // indirect + github.com/enetx/iter v0.0.0-20250912135656-f1583323588f // indirect + github.com/enetx/uquic v0.0.0-20251211194956-3ebf03adbe9e // indirect + github.com/enetx/utls v0.0.0-20251024090823-efbd194d7328 // indirect + github.com/gaukas/clienthellod v0.4.2 // indirect + github.com/gaukas/godicttls v0.0.4 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/google/gopacket v1.1.19 // indirect + github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/joho/godotenv v1.5.1 // indirect - github.com/klauspost/compress v1.17.0 // indirect + github.com/klauspost/compress v1.18.2 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/onsi/ginkgo/v2 v2.27.3 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/quic-go/qpack v0.5.1 // indirect + github.com/refraction-networking/utls v1.8.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.51.0 // indirect github.com/valyala/tcplisten v1.0.0 // indirect - golang.org/x/crypto v0.37.0 // indirect - golang.org/x/sys v0.32.0 // indirect - golang.org/x/text v0.24.0 // indirect + github.com/wzshiming/socks5 v0.6.0 // indirect + go.uber.org/mock v0.6.0 // indirect + golang.org/x/crypto v0.46.0 // indirect + golang.org/x/exp v0.0.0-20251209150349-8475f28825e9 // indirect + golang.org/x/mod v0.31.0 // indirect + golang.org/x/net v0.48.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/text v0.32.0 // indirect + golang.org/x/tools v0.40.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect ) diff --git a/go.sum b/go.sum index 297fc47..45d7510 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,54 @@ github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= -github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= +github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= +github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/enetx/g v1.0.195 h1:QccmE1MC/jzFJDyJEuZV+Io+W13YCpNlhuxMyOtqFZc= +github.com/enetx/g v1.0.195/go.mod h1:s/URtzMtxdoF+PqyKIiWTm7ddGPXn8JwfXCFCNiogSw= +github.com/enetx/http v1.0.19 h1:4W97CyqKrPiR16wEm6UOesqNrt8l4RsVMjZHz6+I84E= +github.com/enetx/http v1.0.19/go.mod h1:1f4mytfF/SfjATEJnynpwGS6aa1ALjb8DtmYgFVblY0= +github.com/enetx/http2 v1.0.20 h1:181A9wyzQOxvj+LqIJ/oGGx6Vl0Ry1wTLOfX+zTnaS0= +github.com/enetx/http2 v1.0.20/go.mod h1:t54ex5HIS8V1+2j6cvEOv6umlrHsbUPFKQ54nYB58Nk= +github.com/enetx/iter v0.0.0-20250912135656-f1583323588f h1:GUW+4AWfECIEJ9oAxgEAVGCpaozMCjRiUYnuR6Q0bCQ= +github.com/enetx/iter v0.0.0-20250912135656-f1583323588f/go.mod h1:oMZN8hGLUpi7QBlMEUqailocNy0NFAO/7Lu+Nwh9HMM= +github.com/enetx/surf v1.0.144 h1:xkWWdmU7i9OvsHAR2KgPexKsClDCch0SKr6uqmy3w4M= +github.com/enetx/surf v1.0.144/go.mod h1:dF0MjqKkDu8qw5uKXcMyE+fnGOYNwy5y4ybiH7MS7FM= +github.com/enetx/uquic v0.0.0-20251211194956-3ebf03adbe9e h1:aFjWUldb+QkmrQ106bOz+fj0SdouLyrcQZ36QSpdpuc= +github.com/enetx/uquic v0.0.0-20251211194956-3ebf03adbe9e/go.mod h1:7/PRpM6H8OnsK5Z6lqwHSa8W7KQrnXnrwvCm4Jl4wFE= +github.com/enetx/utls v0.0.0-20251024090823-efbd194d7328 h1:C1RHsl1CKSDiXumSHpJzk2jSIhvPn2x2MFGeg2ZnQIQ= +github.com/enetx/utls v0.0.0-20251024090823-efbd194d7328/go.mod h1:oeYX2NTbsqs75GBc4Vv56KlItghZAveDkb4dGUwRtkM= +github.com/gaukas/clienthellod v0.4.2 h1:LPJ+LSeqt99pqeCV4C0cllk+pyWmERisP7w6qWr7eqE= +github.com/gaukas/clienthellod v0.4.2/go.mod h1:M57+dsu0ZScvmdnNxaxsDPM46WhSEdPYAOdNgfL7IKA= +github.com/gaukas/godicttls v0.0.4 h1:NlRaXb3J6hAnTmWdsEKb9bcSBD6BvcIjdGdeb0zfXbk= +github.com/gaukas/godicttls v0.0.4/go.mod h1:l6EenT4TLWgTdwslVb4sEMOCf7Bv0JAK67deKr9/NCI= +github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= +github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= +github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= +github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= +github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo= github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ= -github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= -github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= +github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/ilyakaznacheev/cleanenv v1.5.0 h1:0VNZXggJE2OYdXE87bfSSwGxeiGt9moSR2lOrsHHvr4= github.com/ilyakaznacheev/cleanenv v1.5.0/go.mod h1:a5aDzaJrLCQZsazHol1w8InnDcOX0OColm64SlIi6gk= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -21,12 +61,16 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= -github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= -github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= +github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -34,10 +78,22 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= +github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= +github.com/onsi/ginkgo/v2 v2.27.3 h1:ICsZJ8JoYafeXFFlFAG75a7CxMsJHwgKwtO+82SE9L8= +github.com/onsi/ginkgo/v2 v2.27.3/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= +github.com/onsi/gomega v1.38.3 h1:eTX+W6dobAYfFeGC2PV6RwXRu/MyT+cQguijutvkpSM= +github.com/onsi/gomega v1.38.3/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4= github.com/pashagolub/pgxmock/v4 v4.6.0 h1:ds0hIs+bJtkfo01vqjp0BOFirjt4Ea8XV082uorzM3w= github.com/pashagolub/pgxmock/v4 v4.6.0/go.mod h1:9VoVHXwS3XR/yPtKGzwQvwZX1kzGB9sM8SviDcHDa3A= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pquerna/otp v1.5.0 h1:NMMR+WrmaqXU4EzdGJEE1aUUI0AMRzsp96fFFWNPwxs= +github.com/pquerna/otp v1.5.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= +github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= +github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= +github.com/refraction-networking/utls v1.8.0 h1:L38krhiTAyj9EeiQQa2sg+hYb4qwLCqdMcpZrRfbONE= +github.com/refraction-networking/utls v1.8.0/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= @@ -47,26 +103,64 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA= github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= -golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= -golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= -golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= -golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +github.com/wzshiming/socks5 v0.6.0 h1:p5RFNs21Byv+Tnc7chYRFtz0SzK8TcqKV7xFkXSeZvw= +github.com/wzshiming/socks5 v0.6.0/go.mod h1:BvCAqlzocQN5xwLjBZDBbvWlrx8sCYSSbHEOf2wZgT0= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= +golang.org/x/exp v0.0.0-20251209150349-8475f28825e9 h1:MDfG8Cvcqlt9XXrmEiD4epKn7VJHZO84hejP9Jmp0MM= +golang.org/x/exp v0.0.0-20251209150349-8475f28825e9/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= +golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= -golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= -golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= -golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= -golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= +golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= +google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/internal/app/app.go b/internal/app/app.go index 731281c..bce9dbc 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -15,6 +15,7 @@ import ( "cryptoHermes/internal/controller/restapi" "cryptoHermes/internal/repo/persistent/postgres" "cryptoHermes/internal/repo/webapi/binance" + coinglassapi "cryptoHermes/internal/repo/webapi/coinglass" "cryptoHermes/internal/usecase" "cryptoHermes/internal/worker" pgpkg "cryptoHermes/pkg/postgres" @@ -47,6 +48,7 @@ func Run(cfg *config.Config, log *slog.Logger) error { oiRepo := postgres.NewOpenInterestRepo(pool) lsRepo := postgres.NewLongShortRatioRepo(pool) takerRepo := postgres.NewTakerVolumeRepo(pool) + cgLiqRepo := postgres.NewCoinglassLiqHeatMapRepo(pool) contextUC := usecase.NewMarketContextUsecase( binanceClient, @@ -67,23 +69,56 @@ func Run(cfg *config.Config, log *slog.Logger) error { lsRepo, ) - collector := worker.NewCollector(worker.Deps{ - MarketData: binanceClient, - Derivatives: binanceClient, - KlineRepo: klineRepo, - FundingRepo: fundingRepo, - OIRepo: oiRepo, - LSRepo: lsRepo, - TakerRepo: takerRepo, - Symbols: cfg.Collector.Symbols, - Intervals: cfg.Collector.Intervals, - Limit: cfg.Collector.DefaultLimit, - Logger: log, - }) + cgQueryUC := usecase.NewCoinglassQueryUsecase(cgLiqRepo) + + var collector *worker.Collector + if cfg.Collector.Enabled { + collector = worker.NewCollector(worker.Deps{ + MarketData: binanceClient, + Derivatives: binanceClient, + KlineRepo: klineRepo, + FundingRepo: fundingRepo, + OIRepo: oiRepo, + LSRepo: lsRepo, + TakerRepo: takerRepo, + Symbols: cfg.Collector.Symbols, + Intervals: cfg.Collector.Intervals, + Limit: cfg.Collector.DefaultLimit, + Logger: log, + }) + } + + var cgCollector *worker.CoinglassCollector + if cfg.Coinglass.Enabled { + cgClient, err := coinglassapi.NewClient(coinglassapi.ClientOptions{ + BaseURL: cfg.Coinglass.BaseURL, + TOTPSecret: cfg.Coinglass.TOTPSecret, + DataAESKey: cfg.Coinglass.DataAESKey, + Timeout: cfg.Coinglass.Timeout, + FakeSuccessRetry: cfg.Coinglass.FakeSuccessRetry, + FakeSuccessRetryWait: cfg.Coinglass.FakeSuccessRetryWait, + }) + if err != nil { + return fmt.Errorf("coinglass client: %w", err) + } + cgCollector = worker.NewCoinglassCollector(worker.CoinglassCollectorDeps{ + Provider: cgClient, + LiqHeatMapRepo: cgLiqRepo, + Symbols: cfg.Collector.Symbols, + LiqHeatMapInterval: cfg.Coinglass.LiqHeatMapInterval, + LiqHeatMapLimit: cfg.Coinglass.LiqHeatMapLimit, + BackoffInitial: cfg.Coinglass.BackoffInitial, + BackoffMax: cfg.Coinglass.BackoffMax, + Logger: log, + }) + } var sched *Scheduler - if cfg.Collector.Enabled { + if collector != nil || cgCollector != nil { sched = NewScheduler(collector, log) + if cgCollector != nil { + sched.WithCoinglass(cgCollector, cfg.Coinglass.LiqHeatMapCron, cfg.Coinglass.StartupJitterMax) + } sched.Start(rootCtx) defer sched.Stop() } @@ -97,11 +132,13 @@ func Run(cfg *config.Config, log *slog.Logger) error { }) restapi.Register(fiberApp, restapi.Deps{ - Logger: log, - MarketContext: contextUC, - MarketQuery: queryUC, - MarketData: binanceClient, - KlineRepo: klineRepo, + Logger: log, + MarketContext: contextUC, + MarketQuery: queryUC, + CoinglassQuery: cgQueryUC, + CoinglassEnabled: cfg.Coinglass.Enabled, + MarketData: binanceClient, + KlineRepo: klineRepo, }) listenErr := make(chan error, 1) diff --git a/internal/app/scheduler.go b/internal/app/scheduler.go index 55eee55..d375682 100644 --- a/internal/app/scheduler.go +++ b/internal/app/scheduler.go @@ -3,6 +3,8 @@ package app import ( "context" "log/slog" + "math/rand" + "time" "github.com/robfig/cron/v3" @@ -12,8 +14,12 @@ import ( type Scheduler struct { cron *cron.Cron collector *worker.Collector + coinglass *worker.CoinglassCollector log *slog.Logger rootCtx context.Context + + coinglassLiqHeatMapCron string + coinglassStartupJitter time.Duration } func NewScheduler(c *worker.Collector, log *slog.Logger) *Scheduler { @@ -24,19 +30,37 @@ func NewScheduler(c *worker.Collector, log *slog.Logger) *Scheduler { } } +func (s *Scheduler) WithCoinglass(c *worker.CoinglassCollector, liqHeatMapCron string, startupJitter time.Duration) *Scheduler { + s.coinglass = c + s.coinglassLiqHeatMapCron = liqHeatMapCron + s.coinglassStartupJitter = startupJitter + return s +} + func (s *Scheduler) Start(ctx context.Context) { s.rootCtx = ctx - s.add("*/15 * * * *", "klines_15m", func(c context.Context) { _ = s.collector.CollectKlines(c, "15m") }) - s.add("0 * * * *", "klines_1h", func(c context.Context) { _ = s.collector.CollectKlines(c, "1h") }) - s.add("0 */4 * * *", "klines_4h", func(c context.Context) { _ = s.collector.CollectKlines(c, "4h") }) - s.add("5 0 * * *", "klines_1d", func(c context.Context) { _ = s.collector.CollectKlines(c, "1d") }) - s.add("10 0 * * 1", "klines_1w", func(c context.Context) { _ = s.collector.CollectKlines(c, "1w") }) + if s.collector != nil { + s.add("*/15 * * * *", "klines_15m", func(c context.Context) { _ = s.collector.CollectKlines(c, "15m") }) + s.add("0 * * * *", "klines_1h", func(c context.Context) { _ = s.collector.CollectKlines(c, "1h") }) + s.add("0 */4 * * *", "klines_4h", func(c context.Context) { _ = s.collector.CollectKlines(c, "4h") }) + s.add("5 0 * * *", "klines_1d", func(c context.Context) { _ = s.collector.CollectKlines(c, "1d") }) + s.add("10 0 * * 1", "klines_1w", func(c context.Context) { _ = s.collector.CollectKlines(c, "1w") }) - s.add("*/15 * * * *", "funding", func(c context.Context) { _ = s.collector.CollectFunding(c) }) - s.add("*/15 * * * *", "oi", func(c context.Context) { _ = s.collector.CollectOpenInterest(c) }) - s.add("*/15 * * * *", "ls", func(c context.Context) { _ = s.collector.CollectLongShortRatio(c) }) - s.add("*/15 * * * *", "taker", func(c context.Context) { _ = s.collector.CollectTakerVolume(c) }) + s.add("*/15 * * * *", "funding", func(c context.Context) { _ = s.collector.CollectFunding(c) }) + s.add("*/15 * * * *", "oi", func(c context.Context) { _ = s.collector.CollectOpenInterest(c) }) + s.add("*/15 * * * *", "ls", func(c context.Context) { _ = s.collector.CollectLongShortRatio(c) }) + s.add("*/15 * * * *", "taker", func(c context.Context) { _ = s.collector.CollectTakerVolume(c) }) + } + + if s.coinglass != nil { + spec := s.coinglassLiqHeatMapCron + if spec == "" { + spec = "*/5 * * * *" + } + s.add(spec, "coinglass_liq_heatmap", func(c context.Context) { _ = s.coinglass.CollectLiqHeatMap(c) }) + s.startCoinglassWarmup(ctx) + } s.cron.Start() s.log.Info("scheduler_started") @@ -52,6 +76,21 @@ func (s *Scheduler) add(spec, name string, fn func(context.Context)) { } } +func (s *Scheduler) startCoinglassWarmup(ctx context.Context) { + go func() { + if s.coinglassStartupJitter > 0 { + jitter := time.Duration(rand.Int63n(int64(s.coinglassStartupJitter))) + select { + case <-ctx.Done(): + return + case <-time.After(jitter): + } + } + s.log.Info("cron_tick", "job", "coinglass_liq_heatmap_warmup") + _ = s.coinglass.CollectLiqHeatMap(ctx) + }() +} + func (s *Scheduler) Stop() { s.log.Info("scheduler_stopping") <-s.cron.Stop().Done() diff --git a/internal/controller/restapi/router.go b/internal/controller/restapi/router.go index 91f8a54..9e33ff4 100644 --- a/internal/controller/restapi/router.go +++ b/internal/controller/restapi/router.go @@ -11,11 +11,13 @@ import ( ) type Deps struct { - Logger *slog.Logger - MarketContext *usecase.MarketContextUsecase - MarketQuery *usecase.MarketQueryUsecase - MarketData usecase.MarketDataProvider - KlineRepo usecase.KlineRepository + Logger *slog.Logger + MarketContext *usecase.MarketContextUsecase + MarketQuery *usecase.MarketQueryUsecase + CoinglassQuery *usecase.CoinglassQueryUsecase + CoinglassEnabled bool + MarketData usecase.MarketDataProvider + KlineRepo usecase.KlineRepository } func Register(app *fiber.App, deps Deps) { @@ -30,4 +32,8 @@ func Register(app *fiber.App, deps Deps) { MarketData: deps.MarketData, KlineRepo: deps.KlineRepo, }) + v1.RegisterCoinglass(api, v1.CoinglassDeps{ + Enabled: deps.CoinglassEnabled, + Query: deps.CoinglassQuery, + }) } diff --git a/internal/controller/restapi/v1/coinglass_routes.go b/internal/controller/restapi/v1/coinglass_routes.go new file mode 100644 index 0000000..b6c7d87 --- /dev/null +++ b/internal/controller/restapi/v1/coinglass_routes.go @@ -0,0 +1,50 @@ +package v1 + +import ( + "strconv" + "strings" + + "github.com/gofiber/fiber/v2" + + "cryptoHermes/internal/usecase" +) + +type CoinglassDeps struct { + Enabled bool + Query *usecase.CoinglassQueryUsecase +} + +func RegisterCoinglass(r fiber.Router, d CoinglassDeps) { + g := r.Group("/coinglass") + + g.Get("/liq-heatmap", func(c *fiber.Ctx) error { + if !d.Enabled || d.Query == nil { + return fiber.NewError(fiber.StatusServiceUnavailable, "service-unavailable: coinglass disabled in config") + } + + symbol := strings.ToUpper(strings.TrimSpace(c.Query("symbol"))) + if symbol == "" { + return fiber.NewError(fiber.StatusBadRequest, "symbol is required") + } + + interval := strings.TrimSpace(c.Query("interval", "5")) + if interval == "" { + interval = "5" + } + + limit := 288 + if l := c.Query("limit"); l != "" { + v, err := strconv.Atoi(l) + if err != nil || v <= 0 || v > 1000 { + return fiber.NewError(fiber.StatusBadRequest, "limit must be between 1 and 1000") + } + limit = v + } + + rows, err := d.Query.GetLiqHeatMap(c.UserContext(), symbol, interval, limit) + if err != nil { + return fiber.NewError(fiber.StatusInternalServerError, err.Error()) + } + return c.JSON(rows) + }) +} diff --git a/internal/entity/coinglass.go b/internal/entity/coinglass.go new file mode 100644 index 0000000..18dee67 --- /dev/null +++ b/internal/entity/coinglass.go @@ -0,0 +1,54 @@ +package entity + +import "encoding/json" + +// CGLiqHeatMap — /api/index/v2/liqHeatMap 一帧。 +// RawGrid 承载 CoinGlass 原始 2D 网格（spec §5.1：JSONB 仅供应商原始网格 / 元信息， +// usecase 层禁止直接解析任意 JSON 结构，需要稳定字段时由 mapper 展开成 entity 字段）。 +type CGLiqHeatMap struct { + Symbol string `json:"symbol"` + Interval string `json:"interval"` + Timestamp int64 `json:"timestamp"` + PriceMin string `json:"priceMin"` + PriceMax string `json:"priceMax"` + RawGrid json.RawMessage `json:"rawGrid"` +} + +type CGOpenInterestChart struct { + Symbol string `json:"symbol"` + Interval string `json:"interval"` + Timestamp int64 `json:"timestamp"` + Exchanges []CGExchangeOIPoint `json:"exchanges"` +} + +type CGExchangeOIPoint struct { + Exchange string `json:"exchange"` + OI string `json:"oi"` +} + +type CGLongShortRate struct { + Symbol string `json:"symbol"` + Interval string `json:"interval"` + Timestamp int64 `json:"timestamp"` + Exchanges []CGExchangeLongShortPoint `json:"exchanges"` +} + +type CGExchangeLongShortPoint struct { + Exchange string `json:"exchange"` + LongRate string `json:"longRate"` + ShortRate string `json:"shortRate"` + Ratio string `json:"ratio"` +} + +type CGFundingHome struct { + Timestamp int64 `json:"timestamp"` + Items []CGFundingHomeItem `json:"items"` +} + +type CGFundingHomeItem struct { + Symbol string `json:"symbol"` + Exchange string `json:"exchange"` + FundingRate string `json:"fundingRate"` + NextFundingTs int64 `json:"nextFundingTs"` + Interval string `json:"interval"` +} diff --git a/internal/repo/persistent/postgres/coinglass_liq_heatmap_repo.go b/internal/repo/persistent/postgres/coinglass_liq_heatmap_repo.go new file mode 100644 index 0000000..a2c0f4c --- /dev/null +++ b/internal/repo/persistent/postgres/coinglass_liq_heatmap_repo.go @@ -0,0 +1,99 @@ +package postgres + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/jackc/pgx/v5/pgxpool" + + "cryptoHermes/internal/entity" +) + +const coinglassSource = "coinglass" + +type CoinglassLiqHeatMapRepo struct { + pool pgxPool +} + +func NewCoinglassLiqHeatMapRepo(pool *pgxpool.Pool) *CoinglassLiqHeatMapRepo { + return &CoinglassLiqHeatMapRepo{pool: pool} +} + +const cgLiqHeatMapUpsertSQL = ` +INSERT INTO coinglass_liq_heatmap ( + source, symbol, interval, timestamp, + price_min, price_max, raw_grid, + updated_at +) VALUES ( + $1, $2, $3, $4, + NULLIF($5, '')::NUMERIC, NULLIF($6, '')::NUMERIC, $7::JSONB, + now() +) +ON CONFLICT (source, symbol, interval, timestamp) DO UPDATE SET + price_min = EXCLUDED.price_min, + price_max = EXCLUDED.price_max, + raw_grid = EXCLUDED.raw_grid, + updated_at = now() +` + +func (r *CoinglassLiqHeatMapRepo) UpsertMany(ctx context.Context, items []entity.CGLiqHeatMap) error { + if len(items) == 0 { + return nil + } + tx, err := r.pool.Begin(ctx) + if err != nil { + return fmt.Errorf("begin: %w", err) + } + defer tx.Rollback(ctx) + + for _, item := range items { + if item.Symbol == "" || item.Interval == "" || item.Timestamp == 0 || !json.Valid(item.RawGrid) { + continue + } + if _, err := tx.Exec(ctx, cgLiqHeatMapUpsertSQL, + coinglassSource, item.Symbol, item.Interval, item.Timestamp, + item.PriceMin, item.PriceMax, string(item.RawGrid), + ); err != nil { + return fmt.Errorf("upsert coinglass liq heatmap %s/%s@%d: %w", item.Symbol, item.Interval, item.Timestamp, err) + } + } + return tx.Commit(ctx) +} + +const cgLiqHeatMapFindSQL = ` +SELECT symbol, interval, timestamp, + COALESCE(price_min::text, ''), COALESCE(price_max::text, ''), raw_grid::text +FROM coinglass_liq_heatmap +WHERE source = $1 AND symbol = $2 AND interval = $3 +ORDER BY timestamp DESC +LIMIT $4 +` + +func (r *CoinglassLiqHeatMapRepo) FindRecent(ctx context.Context, symbol, interval string, limit int) ([]entity.CGLiqHeatMap, error) { + if limit <= 0 { + limit = 288 + } + rows, err := r.pool.Query(ctx, cgLiqHeatMapFindSQL, coinglassSource, symbol, interval, limit) + if err != nil { + return nil, fmt.Errorf("query: %w", err) + } + defer rows.Close() + + out := make([]entity.CGLiqHeatMap, 0, limit) + for rows.Next() { + var ( + item entity.CGLiqHeatMap + raw string + ) + if err := rows.Scan(&item.Symbol, &item.Interval, &item.Timestamp, &item.PriceMin, &item.PriceMax, &raw); err != nil { + return nil, fmt.Errorf("scan: %w", err) + } + item.RawGrid = json.RawMessage(raw) + out = append(out, item) + } + for i, j := 0, len(out)-1; i < j; i, j = i+1, j-1 { + out[i], out[j] = out[j], out[i] + } + return out, rows.Err() +} diff --git a/internal/repo/persistent/postgres/coinglass_liq_heatmap_repo_test.go b/internal/repo/persistent/postgres/coinglass_liq_heatmap_repo_test.go new file mode 100644 index 0000000..918f4ea --- /dev/null +++ b/internal/repo/persistent/postgres/coinglass_liq_heatmap_repo_test.go @@ -0,0 +1,99 @@ +package postgres + +import ( + "context" + "errors" + "testing" + + "cryptoHermes/internal/entity" + + "github.com/pashagolub/pgxmock/v4" + "github.com/stretchr/testify/require" +) + +func TestCoinglassLiqHeatMapRepo_UpsertMany_HappyPath(t *testing.T) { + mock, err := pgxmock.NewPool() + require.NoError(t, err) + defer mock.Close() + + mock.ExpectBegin() + mock.ExpectExec("INSERT INTO coinglass_liq_heatmap"). + WithArgs("coinglass", "BTCUSDT", "5", int64(1700000000000), "1000.5", "2000.5", `{"liq":[[1,2,3]]}`). + WillReturnResult(pgxmock.NewResult("INSERT", 1)) + mock.ExpectCommit() + mock.ExpectRollback() + + r := &CoinglassLiqHeatMapRepo{pool: mock} + err = r.UpsertMany(context.Background(), []entity.CGLiqHeatMap{{ + Symbol: "BTCUSDT", + Interval: "5", + Timestamp: 1700000000000, + PriceMin: "1000.5", + PriceMax: "2000.5", + RawGrid: []byte(`{"liq":[[1,2,3]]}`), + }}) + require.NoError(t, err) + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestCoinglassLiqHeatMapRepo_UpsertMany_SkipsInvalidRows(t *testing.T) { + mock, err := pgxmock.NewPool() + require.NoError(t, err) + defer mock.Close() + + mock.ExpectBegin() + mock.ExpectCommit() + mock.ExpectRollback() + + r := &CoinglassLiqHeatMapRepo{pool: mock} + err = r.UpsertMany(context.Background(), []entity.CGLiqHeatMap{ + {Symbol: "", Interval: "5", Timestamp: 1, RawGrid: []byte(`{}`)}, + {Symbol: "BTCUSDT", Interval: "", Timestamp: 1, RawGrid: []byte(`{}`)}, + {Symbol: "BTCUSDT", Interval: "5", Timestamp: 0, RawGrid: []byte(`{}`)}, + {Symbol: "BTCUSDT", Interval: "5", Timestamp: 1, RawGrid: []byte(`not-json`)}, + }) + require.NoError(t, err) + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestCoinglassLiqHeatMapRepo_UpsertMany_ExecError(t *testing.T) { + mock, err := pgxmock.NewPool() + require.NoError(t, err) + defer mock.Close() + + mock.ExpectBegin() + mock.ExpectExec("INSERT INTO coinglass_liq_heatmap"). + WithArgs(pgxmock.AnyArg(), pgxmock.AnyArg(), pgxmock.AnyArg(), pgxmock.AnyArg(), pgxmock.AnyArg(), pgxmock.AnyArg(), pgxmock.AnyArg()). + WillReturnError(errors.New("db down")) + mock.ExpectRollback() + + r := &CoinglassLiqHeatMapRepo{pool: mock} + err = r.UpsertMany(context.Background(), []entity.CGLiqHeatMap{{ + Symbol: "BTCUSDT", Interval: "5", Timestamp: 1, RawGrid: []byte(`{}`), + }}) + require.Error(t, err) + require.Contains(t, err.Error(), "upsert coinglass liq heatmap") + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestCoinglassLiqHeatMapRepo_FindRecent_HappyPath(t *testing.T) { + mock, err := pgxmock.NewPool() + require.NoError(t, err) + defer mock.Close() + + rows := mock.NewRows([]string{"symbol", "interval", "timestamp", "price_min", "price_max", "raw_grid"}). + AddRow("BTCUSDT", "5", int64(2), "100", "200", `{"b":2}`). + AddRow("BTCUSDT", "5", int64(1), "90", "210", `{"a":1}`) + + mock.ExpectQuery("SELECT symbol, interval, timestamp"). + WithArgs("coinglass", "BTCUSDT", "5", 288). + WillReturnRows(rows) + + r := &CoinglassLiqHeatMapRepo{pool: mock} + got, err := r.FindRecent(context.Background(), "BTCUSDT", "5", 0) + require.NoError(t, err) + require.Len(t, got, 2) + require.Equal(t, int64(1), got[0].Timestamp) + require.JSONEq(t, `{"b":2}`, string(got[1].RawGrid)) + require.NoError(t, mock.ExpectationsWereMet()) +} diff --git a/internal/repo/webapi/coinglass/client.go b/internal/repo/webapi/coinglass/client.go new file mode 100644 index 0000000..738762f --- /dev/null +++ b/internal/repo/webapi/coinglass/client.go @@ -0,0 +1,321 @@ +package coinglass + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/enetx/g" + "github.com/enetx/surf" + "github.com/pquerna/otp/totp" +) + +const ( + defaultBaseURL = "https://capi.coinglass.com" + defaultOrigin = "https://www.coinglass.com" + defaultDataAESKey = "1f68efd73f8d4921acc0dead41dd39bc" + defaultRequestTimeout = 15 * time.Second + defaultFakeSuccessRetry = 1 + defaultFakeSuccessRetryWait = 2 * time.Second +) + +type Client struct { + baseURL string + totpSecret string + dataAESKey []byte + fakeSuccessRetry int + fakeSuccessRetryWait time.Duration + doer requestDoer + now func() time.Time +} + +type ClientOptions struct { + BaseURL string + TOTPSecret string + DataAESKey string + Timeout time.Duration + FakeSuccessRetry int + FakeSuccessRetryWait time.Duration +} + +type requestDoer interface { + Do(context.Context, httpRequest) (*httpResponse, error) +} + +type httpRequest struct { + URL string + Header http.Header +} + +type httpResponse struct { + StatusCode int + Header http.Header + Body []byte +} + +type transportEnvelope struct { + Success *bool `json:"success"` + Code json.RawMessage `json:"code"` + Msg string `json:"msg"` + Data *string `json:"data"` +} + +func NewClient(opts ClientOptions) (*Client, error) { + return newClientWithDoer(opts, newSurfDoer(opts.Timeout)) +} + +func newClientWithDoer(opts ClientOptions, doer requestDoer) (*Client, error) { + if doer == nil { + return nil, fmt.Errorf("coinglass client: nil doer") + } + if opts.BaseURL == "" { + opts.BaseURL = defaultBaseURL + } + if opts.TOTPSecret == "" { + return nil, fmt.Errorf("coinglass client: totp secret is required") + } + if opts.DataAESKey == "" { + opts.DataAESKey = defaultDataAESKey + } + if opts.FakeSuccessRetry == 0 { + opts.FakeSuccessRetry = defaultFakeSuccessRetry + } + if opts.FakeSuccessRetryWait == 0 { + opts.FakeSuccessRetryWait = defaultFakeSuccessRetryWait + } + + if _, err := aesECBEncrypt([]byte(opts.DataAESKey), pkcs7Pad([]byte("probe"), 16)); err != nil { + return nil, fmt.Errorf("coinglass client: bad data aes key: %w", err) + } + + return &Client{ + baseURL: strings.TrimRight(opts.BaseURL, "/"), + totpSecret: opts.TOTPSecret, + dataAESKey: []byte(opts.DataAESKey), + fakeSuccessRetry: opts.FakeSuccessRetry, + fakeSuccessRetryWait: opts.FakeSuccessRetryWait, + doer: doer, + now: time.Now, + }, nil +} + +// GetPlaintext dispatches one CoinGlass V2 GET request and returns decrypted plaintext bytes. +func (c *Client) GetPlaintext(ctx context.Context, path string, query url.Values) ([]byte, error) { + var lastFake error + for attempt := 0; attempt <= c.fakeSuccessRetry; attempt++ { + plain, err := c.getPlaintextOnce(ctx, path, query) + if err == nil { + return plain, nil + } + if !errors.Is(err, ErrFakeSuccess) { + return nil, err + } + lastFake = err + if attempt == c.fakeSuccessRetry { + break + } + if err := sleepContext(ctx, c.fakeSuccessRetryWait); err != nil { + return nil, err + } + } + return nil, lastFake +} + +func (c *Client) getPlaintextOnce(ctx context.Context, path string, query url.Values) ([]byte, error) { + now := c.now() + cacheTsV2 := fmt.Sprintf("%d", now.UnixMilli()) + + q := cloneValues(query) + dataParam, err := generateDataParam(c.totpSecret, c.dataAESKey, now) + if err != nil { + return nil, err + } + q.Set("data", dataParam) + + resp, err := c.doer.Do(ctx, httpRequest{ + URL: c.buildURL(path, q), + Header: baseHeaders(cacheTsV2), + }) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("coinglass %s status=%d: %s", path, resp.StatusCode, bodySnippet(resp.Body)) + } + + env, err := parseTransportEnvelope(resp.Body) + if err != nil { + return nil, fmt.Errorf("coinglass %s envelope: %w", path, err) + } + if env.Success != nil && !*env.Success { + return nil, fmt.Errorf("%w: code=%s msg=%q", ErrAPIBusiness, envelopeCode(env.Code), env.Msg) + } + if env.Data == nil || *env.Data == "" { + return nil, fmt.Errorf("%w: empty data code=%s msg=%q", ErrFakeSuccess, envelopeCode(env.Code), env.Msg) + } + + v := firstHeader(resp.Header, "v") + user := firstHeader(resp.Header, "user") + respTime := firstHeader(resp.Header, "time") + if v == "" || user == "" { + return nil, fmt.Errorf("%w: missing v/user response headers", ErrFakeSuccess) + } + + seed, err := resolveSeed(v, path, cacheTsV2, respTime) + if err != nil { + return nil, err + } + plain, err := DecryptUniversal(seed, user, *env.Data) + if err != nil { + return nil, err + } + return plain, nil +} + +func (c *Client) buildURL(path string, query url.Values) string { + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + if len(query) == 0 { + return c.baseURL + path + } + return c.baseURL + path + "?" + query.Encode() +} + +func generateDataParam(secret string, key []byte, now time.Time) (string, error) { + code, err := totp.GenerateCode(secret, now) + if err != nil { + return "", fmt.Errorf("coinglass data param totp: %w", err) + } + plain := []byte(fmt.Sprintf("%d,%s", now.Unix(), code)) + enc, err := aesECBEncrypt(key, pkcs7Pad(plain, 16)) + if err != nil { + return "", err + } + return base64.StdEncoding.EncodeToString(enc), nil +} + +func baseHeaders(cacheTsV2 string) http.Header { + h := http.Header{} + h.Set("Accept", "application/json") + h.Set("Accept-Language", "en-US,en;q=0.9") + h.Set("Language", "zh") + h.Set("Encryption", "true") + h.Set("Origin", defaultOrigin) + h.Set("Referer", defaultOrigin+"/") + h.Set("Cache-Ts-V2", cacheTsV2) + h.Set("Sec-Fetch-Dest", "empty") + h.Set("Sec-Fetch-Mode", "cors") + h.Set("Sec-Fetch-Site", "same-site") + return h +} + +func parseTransportEnvelope(body []byte) (*transportEnvelope, error) { + var env transportEnvelope + if err := json.Unmarshal(body, &env); err != nil { + return nil, err + } + return &env, nil +} + +func cloneValues(in url.Values) url.Values { + out := make(url.Values, len(in)) + for k, vs := range in { + out[k] = append([]string(nil), vs...) + } + return out +} + +func sleepContext(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func firstHeader(h http.Header, key string) string { + if v := h.Get(key); v != "" { + return v + } + return h.Get(http.CanonicalHeaderKey(key)) +} + +func envelopeCode(raw json.RawMessage) string { + raw = json.RawMessage(strings.TrimSpace(string(raw))) + if len(raw) == 0 || string(raw) == "null" { + return "" + } + var s string + if err := json.Unmarshal(raw, &s); err == nil { + return s + } + return string(raw) +} + +func bodySnippet(b []byte) string { + const max = 200 + if len(b) <= max { + return string(b) + } + return string(b[:max]) +} + +type surfDoer struct { + client *surf.Client +} + +func newSurfDoer(timeout time.Duration) *surfDoer { + if timeout == 0 { + timeout = defaultRequestTimeout + } + return &surfDoer{ + client: surf.NewClient().Builder(). + Impersonate().FireFox(). + Timeout(timeout). + Build(), + } +} + +func (d *surfDoer) Do(ctx context.Context, req httpRequest) (*httpResponse, error) { + result := d.client.Get(g.String(req.URL)). + WithContext(ctx). + SetHeaders(singleHeaderValues(req.Header)). + Do() + if result.IsErr() { + return nil, result.Err() + } + + resp := result.Ok() + header := http.Header{} + for _, key := range []string{"v", "user", "time"} { + if value := resp.Headers.Get(g.String(key)).Std(); value != "" { + header.Set(key, value) + } + } + + return &httpResponse{ + StatusCode: int(resp.StatusCode), + Header: header, + Body: []byte(resp.Body.Bytes()), + }, nil +} + +func singleHeaderValues(h http.Header) map[string]string { + out := make(map[string]string, len(h)) + for k, vs := range h { + if len(vs) > 0 { + out[k] = vs[0] + } + } + return out +} diff --git a/internal/repo/webapi/coinglass/client_test.go b/internal/repo/webapi/coinglass/client_test.go new file mode 100644 index 0000000..6f7ac18 --- /dev/null +++ b/internal/repo/webapi/coinglass/client_test.go @@ -0,0 +1,163 @@ +package coinglass + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +type netHTTPDoer struct { + client *http.Client +} + +func (d netHTTPDoer) Do(ctx context.Context, req httpRequest) (*httpResponse, error) { + httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, req.URL, nil) + if err != nil { + return nil, err + } + httpReq.Header = req.Header.Clone() + + resp, err := d.client.Do(httpReq) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return &httpResponse{ + StatusCode: resp.StatusCode, + Header: resp.Header.Clone(), + Body: body, + }, nil +} + +func TestClientGetPlaintext_HappyPath(t *testing.T) { + const ( + path = "/api/index/v2/liqHeatMap" + token = "dfa89d5d00cf4000" + ) + + plaintext := `{"instrument":{"exName":"Binance","instrumentId":"BTCUSDT"},"liq":[[1,2,3.4]]}` + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, path, r.URL.Path) + require.Equal(t, "true", r.URL.Query().Get("merge")) + require.Equal(t, "Binance_BTCUSDT", r.URL.Query().Get("symbol")) + require.NotEmpty(t, r.URL.Query().Get("data")) + require.Equal(t, "true", r.Header.Get("Encryption")) + require.Equal(t, defaultOrigin, r.Header.Get("Origin")) + require.NotEmpty(t, r.Header.Get("Cache-Ts-V2")) + + cacheTs := r.Header.Get("Cache-Ts-V2") + _, err := strconv.ParseInt(cacheTs, 10, 64) + require.NoError(t, err) + + w.Header().Set("v", "0") + w.Header().Set("user", encryptForTest(t, userKeyFromSeed(cacheTs), token)) + _ = json.NewEncoder(w).Encode(map[string]any{ + "code": "0", + "msg": "success", + "data": encryptForTest(t, bodyKeyFromToken(token), plaintext), + }) + })) + defer ts.Close() + + client := newTestClient(t, ts.URL) + got, err := client.GetPlaintext(context.Background(), path, url.Values{ + "merge": {"true"}, + "symbol": {"Binance_BTCUSDT"}, + "interval": {"5"}, + "limit": {"288"}, + }) + + require.NoError(t, err) + require.JSONEq(t, plaintext, string(got)) +} + +func TestClientGetPlaintext_RetriesFakeSuccessOnce(t *testing.T) { + var attempts int32 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&attempts, 1) + _ = json.NewEncoder(w).Encode(map[string]any{ + "success": true, + "code": "0", + "msg": "success", + "data": nil, + }) + })) + defer ts.Close() + + client := newTestClient(t, ts.URL) + _, err := client.GetPlaintext(context.Background(), "/api/index/v2/liqHeatMap", nil) + + require.ErrorIs(t, err, ErrFakeSuccess) + require.Equal(t, int32(2), atomic.LoadInt32(&attempts)) +} + +func TestClientGetPlaintext_BusinessErrorDoesNotRetry(t *testing.T) { + var attempts int32 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&attempts, 1) + _ = json.NewEncoder(w).Encode(map[string]any{ + "success": false, + "code": "40000", + "msg": "40000", + }) + })) + defer ts.Close() + + client := newTestClient(t, ts.URL) + _, err := client.GetPlaintext(context.Background(), "/api/index/v2/liqHeatMap", nil) + + require.Error(t, err) + require.True(t, errors.Is(err, ErrAPIBusiness)) + require.Equal(t, int32(1), atomic.LoadInt32(&attempts)) +} + +func TestClientGetPlaintext_MissingDecryptHeadersIsFakeSuccess(t *testing.T) { + const token = "dfa89d5d00cf4000" + + var attempts int32 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&attempts, 1) + _ = json.NewEncoder(w).Encode(map[string]any{ + "code": "0", + "msg": "success", + "data": encryptForTest(t, bodyKeyFromToken(token), `{"ok":true}`), + }) + })) + defer ts.Close() + + client := newTestClient(t, ts.URL) + _, err := client.GetPlaintext(context.Background(), "/api/index/v2/liqHeatMap", nil) + + require.ErrorIs(t, err, ErrFakeSuccess) + require.Equal(t, int32(2), atomic.LoadInt32(&attempts)) +} + +func newTestClient(t *testing.T, baseURL string) *Client { + t.Helper() + client, err := newClientWithDoer(ClientOptions{ + BaseURL: baseURL, + TOTPSecret: "JBSWY3DPEHPK3PXP", + FakeSuccessRetryWait: time.Nanosecond, + }, netHTTPDoer{client: http.DefaultClient}) + require.NoError(t, err) + client.now = func() time.Time { + return time.Unix(1735689600, 123*int64(time.Millisecond)) + } + return client +} diff --git a/internal/repo/webapi/coinglass/crypto.go b/internal/repo/webapi/coinglass/crypto.go new file mode 100644 index 0000000..84b33ec --- /dev/null +++ b/internal/repo/webapi/coinglass/crypto.go @@ -0,0 +1,155 @@ +package coinglass + +import ( + "bytes" + "compress/gzip" + "crypto/aes" + "encoding/base64" + "errors" + "fmt" + "io" +) + +var ( + // ErrFakeSuccess — HTTP 200 + success=true + data 空。spec §4.6 / §11。 + ErrFakeSuccess = errors.New("coinglass fake-success (TLS fingerprint suspected)") + + // ErrUnknownV — dispatcher 见到未知 v 分支（bundle 可能升级）。spec §4.3。 + ErrUnknownV = errors.New("coinglass unknown v branch") + + // ErrDecryptFail — 双阶段解密任一步失败（base64 / aes / pkcs7 / gzip / utf8）。 + ErrDecryptFail = errors.New("coinglass decrypt pipeline failed") + + // ErrAPIBusiness — success=false 且有明确 code/msg（params 错等编码 bug）。 + ErrAPIBusiness = errors.New("coinglass API business error") +) + +// aesECBDecrypt 解密 AES-ECB（stdlib 没有 ECB 模式，按 block 循环 Decrypt）。 +// 调用方保证 src 长度是 block size 的整数倍。 +func aesECBDecrypt(key, src []byte) ([]byte, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, fmt.Errorf("%w: aes new cipher: %v", ErrDecryptFail, err) + } + bs := block.BlockSize() + if len(src) == 0 || len(src)%bs != 0 { + return nil, fmt.Errorf("%w: ciphertext len %d not multiple of block %d", ErrDecryptFail, len(src), bs) + } + dst := make([]byte, len(src)) + for i := 0; i < len(src); i += bs { + block.Decrypt(dst[i:i+bs], src[i:i+bs]) + } + return dst, nil +} + +// aesECBEncrypt 加密 AES-ECB（仅 data= 签名用，spec §4.5）。 +func aesECBEncrypt(key, src []byte) ([]byte, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, fmt.Errorf("%w: aes new cipher: %v", ErrDecryptFail, err) + } + bs := block.BlockSize() + if len(src)%bs != 0 { + return nil, fmt.Errorf("%w: plaintext len %d not multiple of block %d", ErrDecryptFail, len(src), bs) + } + dst := make([]byte, len(src)) + for i := 0; i < len(src); i += bs { + block.Encrypt(dst[i:i+bs], src[i:i+bs]) + } + return dst, nil +} + +// pkcs7Pad / pkcs7Unpad — RFC 5652 §6.3。 +func pkcs7Pad(b []byte, blockSize int) []byte { + pad := blockSize - len(b)%blockSize + out := make([]byte, len(b)+pad) + copy(out, b) + for i := len(b); i < len(out); i++ { + out[i] = byte(pad) + } + return out +} + +func pkcs7Unpad(b []byte, blockSize int) ([]byte, error) { + if len(b) == 0 || len(b)%blockSize != 0 { + return nil, fmt.Errorf("%w: pkcs7 unpad bad len %d", ErrDecryptFail, len(b)) + } + pad := int(b[len(b)-1]) + if pad == 0 || pad > blockSize { + return nil, fmt.Errorf("%w: pkcs7 unpad bad value %d", ErrDecryptFail, pad) + } + for i := len(b) - pad; i < len(b); i++ { + if int(b[i]) != pad { + return nil, fmt.Errorf("%w: pkcs7 unpad inconsistent", ErrDecryptFail) + } + } + return b[:len(b)-pad], nil +} + +// decryptBlob — V2 通用管线：base64dec → AES-ECB → unpad → gunzip → strip outer quotes。 +func decryptBlob(ciphertextB64 string, key []byte) (string, error) { + raw, err := base64.StdEncoding.DecodeString(ciphertextB64) + if err != nil { + return "", fmt.Errorf("%w: b64dec: %v", ErrDecryptFail, err) + } + decrypted, err := aesECBDecrypt(key, raw) + if err != nil { + return "", err + } + unpadded, err := pkcs7Unpad(decrypted, aes.BlockSize) + if err != nil { + return "", err + } + gz, err := gzip.NewReader(bytes.NewReader(unpadded)) + if err != nil { + return "", fmt.Errorf("%w: gzip header: %v", ErrDecryptFail, err) + } + defer gz.Close() + plain, err := io.ReadAll(gz) + if err != nil { + return "", fmt.Errorf("%w: gunzip: %v", ErrDecryptFail, err) + } + text := string(plain) + if len(text) >= 2 && text[0] == '"' && text[len(text)-1] == '"' { + text = text[1 : len(text)-1] + } + return text, nil +} + +// userKeyFromSeed — Stage 1 key 派生：base64(seed)[:16]（cg.md L38）。 +func userKeyFromSeed(seed string) []byte { + enc := base64.StdEncoding.EncodeToString([]byte(seed)) + if len(enc) < 16 { + // 不应发生（任何非空 seed base64 后都 ≥ 4 chars， + // 实际 seed 至少 16 字节 → enc 至少 24 chars）；保险兜底。 + return []byte(enc) + } + return []byte(enc[:16]) +} + +// bodyKeyFromToken — Stage 2 key 派生：token[:16] 原 ASCII（cg.md §43-50：no second base64）。 +func bodyKeyFromToken(token string) []byte { + if len(token) < 16 { + return []byte(token) + } + return []byte(token[:16]) +} + +// DecryptUniversal 跑完整双阶段管线（spec §4.4）。 +// +// seed 由 dispatcher.resolveSeed(v, ...) 给出 +// userHeaderB64 = response.headers["user"] +// bodyDataB64 = body.data （envelope 外层，未解密的 ciphertext） +// +// 返回 plaintext JSON 字节流，可直接喂给 mapper。 +func DecryptUniversal(seed, userHeaderB64, bodyDataB64 string) ([]byte, error) { + token, err := decryptBlob(userHeaderB64, userKeyFromSeed(seed)) + if err != nil { + return nil, err + } + plain, err := decryptBlob(bodyDataB64, bodyKeyFromToken(token)) + if err != nil { + return nil, err + } + return []byte(plain), nil +} diff --git a/internal/repo/webapi/coinglass/crypto_test.go b/internal/repo/webapi/coinglass/crypto_test.go new file mode 100644 index 0000000..e92a95b --- /dev/null +++ b/internal/repo/webapi/coinglass/crypto_test.go @@ -0,0 +1,128 @@ +package coinglass + +import ( + "bytes" + "compress/gzip" + "crypto/aes" + "encoding/base64" + "errors" + "testing" + + "github.com/stretchr/testify/require" +) + +// helper：把 plaintext 用 V2 通用管线反向加密，产出可喂给 decryptBlob 的 base64 ciphertext。 +// 形态：plaintext → '"plaintext"' → gzip → pkcs7 pad → AES-ECB → base64 +func encryptForTest(t *testing.T, key []byte, plaintext string) string { + t.Helper() + + // CoinGlass 服务器端会在 plaintext 外加引号（decryptBlob 反向 strip）； + // 测试也照样加引号 → 验证 strip 逻辑。 + quoted := []byte(`"` + plaintext + `"`) + + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + _, err := gz.Write(quoted) + require.NoError(t, err) + require.NoError(t, gz.Close()) + + padded := pkcs7Pad(buf.Bytes(), aes.BlockSize) + enc, err := aesECBEncrypt(key, padded) + require.NoError(t, err) + return base64.StdEncoding.EncodeToString(enc) +} + +func TestDecryptBlob_RoundTrip(t *testing.T) { + cases := []struct { + name string + key []byte + text string + }{ + {"16-byte key (AES-128)", []byte("0123456789abcdef"), `{"hello":"world"}`}, + {"32-byte key (AES-256)", []byte("1f68efd73f8d4921acc0dead41dd39bc"), `{"foo":42,"bar":"baz"}`}, + {"empty json object", []byte("0123456789abcdef"), `{}`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ct := encryptForTest(t, tc.key, tc.text) + got, err := decryptBlob(ct, tc.key) + require.NoError(t, err) + require.Equal(t, tc.text, got) + }) + } +} + +func TestDecryptBlob_BadCiphertext(t *testing.T) { + key := []byte("0123456789abcdef") + + _, err := decryptBlob("!!!not_base64!!!", key) + require.Error(t, err) + require.ErrorIs(t, err, ErrDecryptFail) + + short := base64.StdEncoding.EncodeToString([]byte("toosmall")) + _, err = decryptBlob(short, key) + require.ErrorIs(t, err, ErrDecryptFail) +} + +func TestPKCS7_PadUnpadRoundTrip(t *testing.T) { + cases := [][]byte{ + []byte(""), + []byte("a"), + []byte("0123456789abcde"), // 15 → pad 1 + []byte("0123456789abcdef"), // 16 → pad 16 (full block) + []byte("0123456789abcdef0"), + } + for _, in := range cases { + padded := pkcs7Pad(in, 16) + require.Zero(t, len(padded)%16) + out, err := pkcs7Unpad(padded, 16) + require.NoError(t, err) + require.Equal(t, in, out) + } +} + +func TestPKCS7_UnpadRejectsBadInput(t *testing.T) { + bad := []byte("0123456789abcdef") // pad byte = 'f' (0x66 = 102 > 16) + _, err := pkcs7Unpad(bad, 16) + require.Error(t, err) + + _, err = pkcs7Unpad([]byte{}, 16) + require.Error(t, err) +} + +func TestUserKeyFromSeed_TruncatesTo16(t *testing.T) { + seed := "863f08689c97435b" // v=77 constant from bundle + got := userKeyFromSeed(seed) + require.Len(t, got, 16) + // base64("863f08689c97435b") = "ODYzZjA4Njg5Yzk3NDM1Yg==" → first 16 = "ODYzZjA4Njg5Yzk3" + require.Equal(t, "ODYzZjA4Njg5Yzk3", string(got)) +} + +func TestBodyKeyFromToken_RawASCIIFirst16(t *testing.T) { + token := "dfa89d5d00cf4000extra_chars" + got := bodyKeyFromToken(token) + require.Len(t, got, 16) + require.Equal(t, "dfa89d5d00cf4000", string(got)) +} + +func TestDecryptUniversal_TwoStage(t *testing.T) { + seed := "863f08689c97435b" // v=77 bundle constant + userKey := userKeyFromSeed(seed) + token := "dfa89d5d00cf4000" // exactly 16 chars (matches cg.md L79 example) + bodyKey := []byte(token) + + plaintext := `{"success":true,"code":"0","msg":"","data":{"priceMin":110000.0}}` + + userHeaderB64 := encryptForTest(t, userKey, token) + bodyDataB64 := encryptForTest(t, bodyKey, plaintext) + + got, err := DecryptUniversal(seed, userHeaderB64, bodyDataB64) + require.NoError(t, err) + require.JSONEq(t, plaintext, string(got)) +} + +func TestDecryptUniversal_PropagatesDecryptFail(t *testing.T) { + _, err := DecryptUniversal("seed", "!!!", "!!!") + require.Error(t, err) + require.True(t, errors.Is(err, ErrDecryptFail)) +} diff --git a/internal/repo/webapi/coinglass/dispatcher.go b/internal/repo/webapi/coinglass/dispatcher.go new file mode 100644 index 0000000..de3ad80 --- /dev/null +++ b/internal/repo/webapi/coinglass/dispatcher.go @@ -0,0 +1,33 @@ +package coinglass + +import "fmt" + +// V_CONSTANTS — 来自 bundle 的硬编码 seed（cg.md "Known branches"）。 +var vConstants = map[string]string{ + "55": "170b070da9654622", + "66": "d6537d845a964081", + "77": "863f08689c97435b", +} + +// resolveSeed 把 v 分支映射到 stage-1 解密的 seed 字符串（spec §4.3）。 +// +// v="0" → request 时挂的 cache-ts-v2（毫秒 unix ts，字符串） +// v="1" → URL path（待验证形状，cg.md 标 "shape — verify"） +// v="2" → response.headers.time +// v="55" / "66" / "77" → bundle 常量 +// +// 未知 v → ErrUnknownV，让 collector log + alert 等人工 review bundle。 +func resolveSeed(v, path, reqCacheTsV2, respTime string) (string, error) { + switch v { + case "0": + return reqCacheTsV2, nil + case "1": + return path, nil + case "2": + return respTime, nil + } + if c, ok := vConstants[v]; ok { + return c, nil + } + return "", fmt.Errorf("%w: v=%q (bundle may have added a new case)", ErrUnknownV, v) +} diff --git a/internal/repo/webapi/coinglass/dispatcher_test.go b/internal/repo/webapi/coinglass/dispatcher_test.go new file mode 100644 index 0000000..5339632 --- /dev/null +++ b/internal/repo/webapi/coinglass/dispatcher_test.go @@ -0,0 +1,44 @@ +package coinglass + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestResolveSeed_AllBranches(t *testing.T) { + cases := []struct { + name string + v string + path string + reqCacheTsV2 string + respTime string + wantSeed string + }{ + {"v=0 → cache-ts-v2", "0", "/api/x", "1735689600000", "1735689601000", "1735689600000"}, + {"v=1 → path", "1", "/api/index/v2/liqHeatMap", "ts", "rt", "/api/index/v2/liqHeatMap"}, + {"v=2 → resp time", "2", "/api/x", "ts", "1735689601000", "1735689601000"}, + {"v=55 → bundle const", "55", "", "", "", "170b070da9654622"}, + {"v=66 → bundle const", "66", "", "", "", "d6537d845a964081"}, + {"v=77 → bundle const", "77", "", "", "", "863f08689c97435b"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := resolveSeed(tc.v, tc.path, tc.reqCacheTsV2, tc.respTime) + require.NoError(t, err) + require.Equal(t, tc.wantSeed, got) + }) + } +} + +func TestResolveSeed_UnknownV(t *testing.T) { + cases := []string{"3", "88", "99", "v0", ""} + for _, v := range cases { + t.Run("unknown_v="+v, func(t *testing.T) { + _, err := resolveSeed(v, "/p", "ts", "rt") + require.Error(t, err) + require.True(t, errors.Is(err, ErrUnknownV), "want ErrUnknownV, got %v", err) + }) + } +} diff --git a/internal/repo/webapi/coinglass/liq_heatmap.go b/internal/repo/webapi/coinglass/liq_heatmap.go new file mode 100644 index 0000000..009943e --- /dev/null +++ b/internal/repo/webapi/coinglass/liq_heatmap.go @@ -0,0 +1,52 @@ +package coinglass + +import ( + "context" + "net/url" + "strconv" + "strings" + + "cryptoHermes/internal/entity" +) + +const ( + liqHeatMapPath = "/api/index/v2/liqHeatMap" + defaultLiqInterval = "5" + defaultLiqLimit = 288 + maxLiqHeatMapLimit = 1000 + cgBinanceSymbolPrefix = "Binance_" +) + +func (c *Client) GetLiqHeatMap(ctx context.Context, symbol, interval string, limit int) (*entity.CGLiqHeatMap, error) { + symbol = normalizeBinanceMarketSymbol(symbol) + if interval == "" { + interval = defaultLiqInterval + } + if limit <= 0 { + limit = defaultLiqLimit + } + if limit > maxLiqHeatMapLimit { + limit = maxLiqHeatMapLimit + } + + q := url.Values{} + q.Set("merge", "true") + q.Set("symbol", coinglassBinanceSymbol(symbol)) + q.Set("interval", interval) + q.Set("limit", strconv.Itoa(limit)) + + plaintext, err := c.GetPlaintext(ctx, liqHeatMapPath, q) + if err != nil { + return nil, err + } + return MapLiqHeatMap(symbol, interval, c.now().UnixMilli(), plaintext) +} + +func coinglassBinanceSymbol(symbol string) string { + return cgBinanceSymbolPrefix + symbol +} + +func normalizeBinanceMarketSymbol(symbol string) string { + symbol = strings.ToUpper(strings.TrimSpace(symbol)) + return strings.TrimPrefix(symbol, "BINANCE_") +} diff --git a/internal/repo/webapi/coinglass/mapper.go b/internal/repo/webapi/coinglass/mapper.go new file mode 100644 index 0000000..8f00aa2 --- /dev/null +++ b/internal/repo/webapi/coinglass/mapper.go @@ -0,0 +1,81 @@ +// Package coinglass — CoinGlass V2 私有 API 客户端实现。 +// spec：ai/specs/coinglass.md。 +package coinglass + +import ( + "encoding/json" + "fmt" + "math" + "strconv" + + "cryptoHermes/internal/entity" +) + +// CGEnvelope 是所有 CoinGlass V2 endpoint 的统一外壳。 +// 解密后的 plaintext JSON 形如：{"success":true,"code":"0","msg":"","data":{...}} +type CGEnvelope struct { + Success bool `json:"success"` + Code json.Number `json:"code"` + Msg string `json:"msg"` + Data json.RawMessage `json:"data"` +} + +// MapLiqHeatMap 把解密后的 plaintext envelope 映射成 entity.CGLiqHeatMap。 +// +// PR-1 阶段未拿到真实 schema dump，先做最薄映射：data 整体进 RawGrid； +// 若顶层有 minPrice/maxPrice/priceMin/priceMax 字段则抽出（容忍命名漂移）。 +// 真实 schema 经 spike dump 后回填到 spec §5（mapper 同步加显式字段）。 +// +// 数值边界（ADR-0004 D3）：解 JSON number 默认 float64 是 transient， +// 通过 formatFloat 转 string 进 entity；entity 字段全 string（G12.4 兜底）。 +func MapLiqHeatMap(symbol string, interval string, ts int64, plaintext []byte) (*entity.CGLiqHeatMap, error) { + var env CGEnvelope + if err := json.Unmarshal(plaintext, &env); err != nil { + return nil, fmt.Errorf("liq_heatmap envelope: %w", err) + } + if !env.Success { + return nil, fmt.Errorf("liq_heatmap business error code=%s msg=%s", env.Code, env.Msg) + } + out := &entity.CGLiqHeatMap{ + Symbol: symbol, + Interval: interval, + Timestamp: ts, + RawGrid: append(json.RawMessage(nil), env.Data...), + } + + var probe map[string]json.RawMessage + if err := json.Unmarshal(env.Data, &probe); err == nil { + out.PriceMin = pickFloatField(probe, "priceMin", "minPrice", "min") + out.PriceMax = pickFloatField(probe, "priceMax", "maxPrice", "max") + } + return out, nil +} + +// pickFloatField 在 probe map 里按候选 key 顺序找第一个能解成 float64 的字段， +// 用 formatFloat 转回 string。找不到返回 ""。 +func pickFloatField(probe map[string]json.RawMessage, keys ...string) string { + for _, k := range keys { + raw, ok := probe[k] + if !ok { + continue + } + var f float64 + if err := json.Unmarshal(raw, &f); err == nil { + return formatFloat(f) + } + var s string + if err := json.Unmarshal(raw, &s); err == nil { + return s + } + } + return "" +} + +// formatFloat 是 mapper 出口的唯一数值格式化点（ADR-0002 / ADR-0004 D3）。 +// NaN / Inf 统一映射成 ""，下游消费方自行判空，避免 entity 出现非法字面量。 +func formatFloat(f float64) string { + if math.IsNaN(f) || math.IsInf(f, 0) { + return "" + } + return strconv.FormatFloat(f, 'f', -1, 64) +} diff --git a/internal/repo/webapi/coinglass/mapper_test.go b/internal/repo/webapi/coinglass/mapper_test.go new file mode 100644 index 0000000..2c94d55 --- /dev/null +++ b/internal/repo/webapi/coinglass/mapper_test.go @@ -0,0 +1,85 @@ +package coinglass + +import ( + "encoding/json" + "math" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMapLiqHeatMap_FloatToString(t *testing.T) { + cases := []struct { + name string + envelope string + wantSymbol string + wantMin string + wantMax string + wantErr bool + probeRawKey string + }{ + { + name: "happy path with float number fields", + envelope: `{"success":true,"code":"0","msg":"","data":{"priceMin":110234.5,"priceMax":118888.0,"grid":[[1,2],[3,4]]}}`, + wantSymbol: "Binance_BTCUSDT", + wantMin: "110234.5", + wantMax: "118888", + probeRawKey: "grid", + }, + { + name: "string-typed numbers are passed through unchanged", + envelope: `{"success":true,"code":"0","msg":"","data":{"priceMin":"99999.99","priceMax":"100000.01"}}`, + wantSymbol: "Binance_BTCUSDT", + wantMin: "99999.99", + wantMax: "100000.01", + }, + { + name: "alternate field names (minPrice/maxPrice)", + envelope: `{"success":true,"code":"0","msg":"","data":{"minPrice":1000.5,"maxPrice":2000.5}}`, + wantSymbol: "Binance_BTCUSDT", + wantMin: "1000.5", + wantMax: "2000.5", + }, + { + name: "missing min/max fields → empty strings", + envelope: `{"success":true,"code":"0","msg":"","data":{"grid":[]}}`, + wantSymbol: "Binance_BTCUSDT", + wantMin: "", + wantMax: "", + }, + { + name: "business error propagates", + envelope: `{"success":false,"code":"40001","msg":"bad params","data":null}`, + wantErr: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := MapLiqHeatMap("Binance_BTCUSDT", "5", 1735689600000, []byte(tc.envelope)) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, tc.wantSymbol, got.Symbol) + + require.IsType(t, "", got.PriceMin, "PriceMin must be string (G12.4 boundary)") + require.IsType(t, "", got.PriceMax, "PriceMax must be string (G12.4 boundary)") + + require.Equal(t, tc.wantMin, got.PriceMin) + require.Equal(t, tc.wantMax, got.PriceMax) + + require.NotEmpty(t, got.RawGrid, "RawGrid must carry raw data verbatim") + require.True(t, json.Valid(got.RawGrid), "RawGrid must be valid JSON") + }) + } +} + +func TestFormatFloat_RejectsNaNAndInf(t *testing.T) { + require.Equal(t, "", formatFloat(math.NaN())) + require.Equal(t, "", formatFloat(math.Inf(1))) + require.Equal(t, "", formatFloat(math.Inf(-1))) + require.Equal(t, "0.0001", formatFloat(0.0001)) + require.Equal(t, "118234.567", formatFloat(118234.567)) +} diff --git a/internal/usecase/coinglass_query.go b/internal/usecase/coinglass_query.go new file mode 100644 index 0000000..3b3b3de --- /dev/null +++ b/internal/usecase/coinglass_query.go @@ -0,0 +1,27 @@ +package usecase + +import ( + "context" + "strings" + + "cryptoHermes/internal/entity" +) + +const defaultCGLiqHeatMapLimit = 288 + +type CoinglassQueryUsecase struct { + liqHeatMapRepo CoinglassLiqHeatMapRepository +} + +func NewCoinglassQueryUsecase(liqHeatMapRepo CoinglassLiqHeatMapRepository) *CoinglassQueryUsecase { + return &CoinglassQueryUsecase{liqHeatMapRepo: liqHeatMapRepo} +} + +func (u *CoinglassQueryUsecase) GetLiqHeatMap(ctx context.Context, symbol, interval string, limit int) ([]entity.CGLiqHeatMap, error) { + symbol = strings.ToUpper(strings.TrimSpace(symbol)) + symbol = strings.TrimPrefix(symbol, "BINANCE_") + if limit <= 0 { + limit = defaultCGLiqHeatMapLimit + } + return u.liqHeatMapRepo.FindRecent(ctx, symbol, interval, limit) +} diff --git a/internal/usecase/ports.go b/internal/usecase/ports.go index 511767d..03f1d3b 100644 --- a/internal/usecase/ports.go +++ b/internal/usecase/ports.go @@ -25,6 +25,10 @@ type DerivativesProvider interface { GetTakerBuySellVolume(ctx context.Context, symbol, period string, limit int) ([]entity.TakerBuySellVolume, error) } +type CoinglassProvider interface { + GetLiqHeatMap(ctx context.Context, symbol, interval string, limit int) (*entity.CGLiqHeatMap, error) +} + type KlineRepository interface { UpsertMany(ctx context.Context, items []entity.Kline) error FindRecent(ctx context.Context, symbol, interval string, limit int) ([]entity.Kline, error) @@ -50,6 +54,11 @@ type TakerVolumeRepository interface { FindRecent(ctx context.Context, symbol, period string, limit int) ([]entity.TakerBuySellVolume, error) } +type CoinglassLiqHeatMapRepository interface { + UpsertMany(ctx context.Context, items []entity.CGLiqHeatMap) error + FindRecent(ctx context.Context, symbol, interval string, limit int) ([]entity.CGLiqHeatMap, error) +} + // IndicatorComputer 纯函数式接口：接收已经被上层 fetch 的 slice， // 返回填充好的 TechnicalStructure。 // diff --git a/internal/worker/coinglass_collector.go b/internal/worker/coinglass_collector.go new file mode 100644 index 0000000..506c8a1 --- /dev/null +++ b/internal/worker/coinglass_collector.go @@ -0,0 +1,132 @@ +package worker + +import ( + "context" + "errors" + "log/slog" + "strings" + "sync/atomic" + "time" + + "cryptoHermes/internal/entity" + cgapi "cryptoHermes/internal/repo/webapi/coinglass" + "cryptoHermes/internal/usecase" +) + +type CoinglassCollectorDeps struct { + Provider usecase.CoinglassProvider + LiqHeatMapRepo usecase.CoinglassLiqHeatMapRepository + + Symbols []string + LiqHeatMapInterval string + LiqHeatMapLimit int + BackoffInitial time.Duration + BackoffMax time.Duration + Logger *slog.Logger +} + +type CoinglassCollector struct { + d CoinglassCollectorDeps + + running atomic.Bool + + fakeSuccessStreak int + disabledUntil time.Time + backoff time.Duration + nextAllowed time.Time +} + +func NewCoinglassCollector(d CoinglassCollectorDeps) *CoinglassCollector { + if d.LiqHeatMapInterval == "" { + d.LiqHeatMapInterval = "5" + } + if d.LiqHeatMapLimit <= 0 { + d.LiqHeatMapLimit = 288 + } + if d.BackoffInitial <= 0 { + d.BackoffInitial = 30 * time.Second + } + if d.BackoffMax <= 0 { + d.BackoffMax = 10 * time.Minute + } + if d.Logger == nil { + d.Logger = slog.Default() + } + return &CoinglassCollector{d: d} +} + +func (c *CoinglassCollector) CollectLiqHeatMap(ctx context.Context) error { + if !c.running.CompareAndSwap(false, true) { + c.d.Logger.Info("coinglass_liq_heatmap_skip_running") + return nil + } + defer c.running.Store(false) + + now := time.Now() + if now.Before(c.disabledUntil) { + c.d.Logger.Warn("coinglass_liq_heatmap_skip_disabled", "until", c.disabledUntil.Format(time.RFC3339)) + return nil + } + if now.Before(c.nextAllowed) { + c.d.Logger.Warn("coinglass_liq_heatmap_skip_backoff", "until", c.nextAllowed.Format(time.RFC3339)) + return nil + } + + hadError := false + for _, symbol := range c.d.Symbols { + symbol = strings.ToUpper(strings.TrimSpace(symbol)) + if symbol == "" { + continue + } + + item, err := c.d.Provider.GetLiqHeatMap(ctx, symbol, c.d.LiqHeatMapInterval, c.d.LiqHeatMapLimit) + if err != nil { + hadError = true + c.recordLiqHeatMapError(symbol, err) + continue + } + + if err := c.d.LiqHeatMapRepo.UpsertMany(ctx, []entity.CGLiqHeatMap{*item}); err != nil { + hadError = true + c.d.Logger.Error("coinglass_liq_heatmap_upsert_failed", "symbol", symbol, "err", err) + continue + } + + c.fakeSuccessStreak = 0 + c.d.Logger.Info("coinglass_liq_heatmap_collect_ok", "symbol", symbol, "interval", c.d.LiqHeatMapInterval) + } + + if hadError { + c.bumpBackoff() + } else { + c.backoff = 0 + c.nextAllowed = time.Time{} + } + return nil +} + +func (c *CoinglassCollector) recordLiqHeatMapError(symbol string, err error) { + if errors.Is(err, cgapi.ErrFakeSuccess) { + c.fakeSuccessStreak++ + c.d.Logger.Error("coinglass_liq_heatmap_fake_success", "symbol", symbol, "streak", c.fakeSuccessStreak, "err", err) + if c.fakeSuccessStreak >= 3 { + c.disabledUntil = time.Now().Add(time.Hour) + c.d.Logger.Error("coinglass_liq_heatmap_disabled", "until", c.disabledUntil.Format(time.RFC3339)) + } + return + } + + c.d.Logger.Error("coinglass_liq_heatmap_collect_failed", "symbol", symbol, "err", err) +} + +func (c *CoinglassCollector) bumpBackoff() { + if c.backoff == 0 { + c.backoff = c.d.BackoffInitial + } else { + c.backoff *= 2 + } + if c.backoff > c.d.BackoffMax { + c.backoff = c.d.BackoffMax + } + c.nextAllowed = time.Now().Add(c.backoff) +} diff --git a/migrations/000006_coinglass_liq_heatmap.down.sql b/migrations/000006_coinglass_liq_heatmap.down.sql new file mode 100644 index 0000000..c5d4a2d --- /dev/null +++ b/migrations/000006_coinglass_liq_heatmap.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS coinglass_liq_heatmap; diff --git a/migrations/000006_coinglass_liq_heatmap.up.sql b/migrations/000006_coinglass_liq_heatmap.up.sql new file mode 100644 index 0000000..df5b2bb --- /dev/null +++ b/migrations/000006_coinglass_liq_heatmap.up.sql @@ -0,0 +1,24 @@ +CREATE TABLE IF NOT EXISTS coinglass_liq_heatmap ( + source TEXT NOT NULL, + symbol TEXT NOT NULL, + interval TEXT NOT NULL, + + timestamp BIGINT NOT NULL, + + price_min NUMERIC(36, 18), + price_max NUMERIC(36, 18), + raw_grid JSONB NOT NULL, + + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + PRIMARY KEY (source, symbol, interval, timestamp) +); + +CREATE INDEX IF NOT EXISTS idx_coinglass_liq_heatmap_symbol_interval_time +ON coinglass_liq_heatmap(symbol, interval, timestamp DESC); + +SELECT create_hypertable('coinglass_liq_heatmap', 'timestamp', + chunk_time_interval => 604800000, + if_not_exists => TRUE, + migrate_data => TRUE);