Bug 报告 (用户排查后提出):
- main.go:61-68 在 normal 模式下 LoadRuntimeConfig 失败只打 log.Printf
- TTSConfigErr 被设, /v1/audio/speech 返 503
- 但服务进程继续跑, 监听端口
- 运维看到"服务在跑", 排查成本高
- /health 虽然 503 + body 有 config_error:true, 但普通用户不会看 body
- 没 metric 计数, 监控告警配不出来
修法 (P0+P1+P2 一起):
P0 - main.go: mode 区分
- setup mode + 失败 = WARN (没装正常)
- normal mode + 失败 = log.Fatalf (进程退出, K8s 拉起, 触发告警)
P1 - dto/health.go + controller/tts.go:
- ConfigStatusResponse 加 Error 字段 (omitempty)
- 错误时填 setting.TTSConfigErr.Error()
- /health body 运维一眼看出原因
P2 - metrics/metrics.go:
- 加 tts_config_load_failures_total counter, labeled by mode
- 不管 fail-fast 还是 warn 都计数, 但 mode 区分
- Prometheus 告警: rate(tts_config_load_failures_total{mode="normal"}[5m]) > 0
e2e 验证 (本机, 3 场景):
- 空 DB / setup 模式 → /health 200 + body error 字段 + 进程不退出 ✓
- 装好 OK / normal 模式 → /health 200, body 无 error 字段 ✓
- 装完 + default_resource_id='' (损坏) / normal 模式 →
log 出现 [FATAL] "service cannot start in normal mode..."
进程退出, K8s 拉起会循环触发, 直到 DB 修好 ✓
副作用:
- fail-fast 后, 自愈回退逻辑不变 (installer.Detect 仍把损坏 db
备份 + 转 setup mode), 但用户已正常装过的 db 不会被自愈删除
- 不影响启动日志, LogStartupSummary() 仍照常打印
- 不影响 setup 模式, 该 WARN 还是 WARN
未 push (待用户)
338 lines
12 KiB
Go
338 lines
12 KiB
Go
package controller
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"net/http"
|
|
"runtime"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/volcano-tts/tts-api/adapter/volcano"
|
|
"github.com/volcano-tts/tts-api/common"
|
|
"github.com/volcano-tts/tts-api/dto"
|
|
"github.com/volcano-tts/tts-api/installer"
|
|
"github.com/volcano-tts/tts-api/metrics"
|
|
"github.com/volcano-tts/tts-api/middleware"
|
|
"github.com/volcano-tts/tts-api/setting"
|
|
"github.com/volcano-tts/tts-api/store"
|
|
"github.com/volcano-tts/tts-api/telemetry"
|
|
"github.com/volcano-tts/tts-api/version"
|
|
)
|
|
|
|
var (
|
|
volcanoClient *volcano.HTTPClient
|
|
adapterRec volcano.MetricsRecorder = metrics.AdapterRecorder{}
|
|
)
|
|
|
|
func InitController() {
|
|
volcanoClient = volcano.NewHTTPClient()
|
|
}
|
|
|
|
func truncateForLog(b []byte, max int) string {
|
|
if len(b) > max {
|
|
return string(b[:max]) + fmt.Sprintf("...(truncated, total %d bytes)", len(b))
|
|
}
|
|
return string(b)
|
|
}
|
|
|
|
// resolveClientFormat 把 OpenAI 风格的 response_format 映射为最终输出格式;
|
|
// 不识别或未指定时回退到 setting.TTSOptions.Format。
|
|
func resolveClientFormat(reqFmt string) string {
|
|
switch strings.ToLower(reqFmt) {
|
|
case "mp3", "wav", "opus", "pcm", "aac", "flac":
|
|
if reqFmt == "opus" {
|
|
return "ogg_opus"
|
|
}
|
|
return strings.ToLower(reqFmt)
|
|
}
|
|
return setting.TTSOptions.Format
|
|
}
|
|
|
|
// OpenaiTTSHandler 是 /v1/audio/speech 的入口。
|
|
func OpenaiTTSHandler(w http.ResponseWriter, r *http.Request) {
|
|
start := time.Now()
|
|
|
|
if r.Method != http.MethodPost {
|
|
log.Printf("警告: 错误的方法 - 方法=%s 期望=POST 路径=%s 客户端=%s",
|
|
r.Method, r.URL.Path, middleware.GetClientIP(r))
|
|
metrics.RequestTotal.Inc(telemetry.Labels{"status": "method_not_allowed", "format": "", "speaker": "", "model": ""})
|
|
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
|
return
|
|
}
|
|
|
|
// 安装模式双保险:即使 InstallGuard 中间件没拦住,这里也 503 + 引导跳转
|
|
if installer.GetMode() == installer.ModeSetup {
|
|
log.Printf("[tts] 安装模式下拒绝 /v1/audio/speech - 客户端=%s", middleware.GetClientIP(r))
|
|
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
_, _ = w.Write([]byte(`{"error":"not installed","code":"install_required","redirect":"/setup"}`))
|
|
return
|
|
}
|
|
|
|
if !middleware.ValidateAPIKey(r) {
|
|
metrics.AuthFailed.Inc(telemetry.Labels{})
|
|
log.Printf("警告: API Key 鉴权失败 - 路径=%s 客户端=%s 远端=%s",
|
|
r.URL.Path, middleware.GetClientIP(r), r.RemoteAddr)
|
|
middleware.SendJSONError(w, http.StatusUnauthorized, "Invalid API key provided.", "invalid_request_error", "invalid_api_key")
|
|
return
|
|
}
|
|
|
|
if setting.TTSConfigErr != nil {
|
|
log.Printf("警告: TTS配置未就绪,拒绝请求 - 错误=%v 路径=%s 客户端=%s",
|
|
setting.TTSConfigErr, r.URL.Path, middleware.GetClientIP(r))
|
|
middleware.SendJSONError(w, http.StatusServiceUnavailable, "TTS service configuration error. Please check environment variables and restart the service.", "configuration_error", "service_unavailable")
|
|
return
|
|
}
|
|
|
|
r.Body = http.MaxBytesReader(w, r.Body, common.MaxRequestBodySize)
|
|
body, err := io.ReadAll(r.Body)
|
|
if err != nil {
|
|
if strings.Contains(err.Error(), "request body too large") {
|
|
log.Printf("警告: 请求体过大 - 路径=%s 客户端=%s 限制=%d字节",
|
|
r.URL.Path, middleware.GetClientIP(r), common.MaxRequestBodySize)
|
|
http.Error(w, "Request body too large", http.StatusRequestEntityTooLarge)
|
|
return
|
|
}
|
|
log.Printf("警告: 读取请求体失败 - 路径=%s 客户端=%s 错误=%v",
|
|
r.URL.Path, middleware.GetClientIP(r), err)
|
|
http.Error(w, "Failed to read request body", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
var req dto.OpenAITTSRequest
|
|
if err := json.Unmarshal(body, &req); err != nil {
|
|
log.Printf("警告: JSON 解析失败 - 路径=%s 客户端=%s 错误=%v body前200字节=%q",
|
|
r.URL.Path, middleware.GetClientIP(r), err, truncateForLog(body, 200))
|
|
http.Error(w, "Invalid JSON", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
if req.Model != "" {
|
|
if len(req.Model) > common.MaxModelNameLength {
|
|
log.Printf("警告: Model 名过长 - 路径=%s 客户端=%s 长度=%d 限制=%d",
|
|
r.URL.Path, middleware.GetClientIP(r), len(req.Model), common.MaxModelNameLength)
|
|
http.Error(w, fmt.Sprintf("Model name too long (max %d characters)", common.MaxModelNameLength), http.StatusBadRequest)
|
|
return
|
|
}
|
|
if strings.ContainsAny(req.Model, "\x00\n\r\t") {
|
|
log.Printf("警告: Model 名含非法字符 - 路径=%s 客户端=%s model前50字节=%q",
|
|
r.URL.Path, middleware.GetClientIP(r), truncateForLog([]byte(req.Model), 50))
|
|
http.Error(w, "Model name contains invalid characters", http.StatusBadRequest)
|
|
return
|
|
}
|
|
}
|
|
|
|
if req.Input == "" {
|
|
log.Printf("警告: input 字段为空 - 路径=%s 客户端=%s", r.URL.Path, middleware.GetClientIP(r))
|
|
http.Error(w, "Input text is required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
if len(req.Input) > common.MaxTextLength {
|
|
log.Printf("警告: input 文本过长 - 路径=%s 客户端=%s 长度=%d 限制=%d",
|
|
r.URL.Path, middleware.GetClientIP(r), len(req.Input), common.MaxTextLength)
|
|
http.Error(w, fmt.Sprintf("Input text too long (max %d characters)", common.MaxTextLength), http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
speed := req.Speed
|
|
if speed <= 0 {
|
|
speed = common.DefaultSpeed
|
|
}
|
|
if speed < common.MinSpeed {
|
|
speed = common.MinSpeed
|
|
}
|
|
if speed > common.MaxSpeed {
|
|
speed = common.MaxSpeed
|
|
}
|
|
|
|
clientFormat := resolveClientFormat(req.ResponseFormat)
|
|
|
|
opts := setting.TTSOptions
|
|
opts.Text = req.Input
|
|
|
|
// M3: voice 路由
|
|
// - voice 为空 → 走 LoadRuntimeConfig 解析过的 opts.Speaker (已是真 speaker ID,
|
|
// default_speaker 是 voice 名,LoadRuntimeConfig 查 voice 表后替换)
|
|
// - voice 非空 → 查 voices 表,替换 opts.Speaker / ResourceID / Model
|
|
// - 命中但 enabled=0 → 仍可用(用户显式传 voice 即覆盖 enabled 状态;若想禁用在 admin UI 关掉就行)
|
|
// - 未命中 → 400 "unknown voice: <name>"
|
|
if req.Voice != "" {
|
|
s := GetAdminStore()
|
|
if s == nil {
|
|
log.Printf("警告: voice=%s 路由但 store 未初始化 - 路径=%s", req.Voice, r.URL.Path)
|
|
middleware.SendJSONError(w, http.StatusServiceUnavailable,
|
|
"voice routing requires database; not initialized",
|
|
"configuration_error", "db_not_ready")
|
|
return
|
|
}
|
|
v, err := s.VoiceGetByName(req.Voice)
|
|
if err != nil {
|
|
if err == store.ErrNotFound {
|
|
log.Printf("警告: 未知 voice=%q - 路径=%s 客户端=%s", req.Voice, r.URL.Path, middleware.GetClientIP(r))
|
|
middleware.SendJSONError(w, http.StatusBadRequest,
|
|
fmt.Sprintf("unknown voice: '%s'", req.Voice),
|
|
"invalid_request_error", "unknown_voice")
|
|
return
|
|
}
|
|
log.Printf("警告: voice 查库失败 - 错误=%v voice=%s", err, req.Voice)
|
|
middleware.SendJSONError(w, http.StatusInternalServerError,
|
|
"voice lookup failed", "server_error", "db_read_failed")
|
|
return
|
|
}
|
|
// 覆盖 opts(API key / UID 保留自 setting.TTSOptions)
|
|
if !v.Enabled {
|
|
log.Printf("警告: voice=%q 已禁用 - 客户端=%s", req.Voice, middleware.GetClientIP(r))
|
|
middleware.SendJSONError(w, http.StatusForbidden,
|
|
fmt.Sprintf("voice '%s' is disabled", req.Voice),
|
|
"invalid_request_error", "voice_disabled")
|
|
return
|
|
}
|
|
opts.Speaker = v.Speaker
|
|
opts.ResourceID = v.ResourceID
|
|
if v.Model != "" {
|
|
opts.Model = v.Model
|
|
}
|
|
log.Printf("[tts] voice=%s 命中 (speaker=%s resource=%s model=%s) - 客户端=%s",
|
|
req.Voice, telemetry.MaskSpeaker(v.Speaker), v.ResourceID, v.Model, middleware.GetClientIP(r))
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(r.Context(), setting.TTSTimeout)
|
|
defer cancel()
|
|
|
|
result, err := volcano.Synthesis(ctx, volcanoClient, opts, req.Input, clientFormat, speed, adapterRec)
|
|
duration := time.Since(start)
|
|
|
|
finalLabels := telemetry.Labels{
|
|
"format": clientFormat,
|
|
// speaker 是火山复刻音色 ID(用户付费资产),不能直接出现在 /metrics label 里
|
|
//(无鉴权可枚举)。用 sha1[:8] 替代:同 speaker 同 label 保留 per-voice 观测,
|
|
//但反推不出原值。Admin UI 想要看原名通过 /api/voices 拿 name 字段。
|
|
"speaker": telemetry.SpeakerLabel(opts.Speaker),
|
|
"model": opts.Model,
|
|
}
|
|
if err != nil {
|
|
finalLabels["status"] = classifyStatus(err)
|
|
metrics.RequestTotal.Inc(finalLabels)
|
|
metrics.RequestDuration.Observe(duration.Seconds(), telemetry.Labels{"status": finalLabels["status"], "format": clientFormat})
|
|
log.Printf("警告: TTS 合成失败 - 路径=%s 客户端=%s 文本长度=%d 耗时=%v 错误=%v",
|
|
r.URL.Path, middleware.GetClientIP(r), len(req.Input), duration, err)
|
|
middleware.SendJSONError(w, http.StatusInternalServerError, "TTS synthesis failed.", "server_error", "synthesis_failed")
|
|
return
|
|
}
|
|
|
|
finalLabels["status"] = "ok"
|
|
metrics.RequestTotal.Inc(finalLabels)
|
|
metrics.RequestDuration.Observe(duration.Seconds(), telemetry.Labels{"status": "ok", "format": clientFormat})
|
|
|
|
w.Header().Set("Content-Type", contentTypeFor(result.Format))
|
|
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(result.AudioData)))
|
|
w.Header().Set("X-Request-Id", result.ReqID)
|
|
w.WriteHeader(http.StatusOK)
|
|
w.Write(result.AudioData)
|
|
}
|
|
|
|
func classifyStatus(err error) string {
|
|
if ue, ok := err.(*volcano.UpstreamError); ok {
|
|
switch ue.Stage {
|
|
case "request":
|
|
return "request_error"
|
|
case "http":
|
|
return fmt.Sprintf("http_%d", ue.Code)
|
|
case "stream":
|
|
return "upstream_error"
|
|
case "wrap":
|
|
return "wrap_error"
|
|
}
|
|
}
|
|
return "internal_error"
|
|
}
|
|
|
|
func contentTypeFor(format string) string {
|
|
switch strings.ToLower(format) {
|
|
case "wav":
|
|
return "audio/wav"
|
|
case "mp3":
|
|
return "audio/mpeg"
|
|
case "ogg_opus", "opus":
|
|
return "audio/ogg"
|
|
case "pcm":
|
|
return "audio/L16"
|
|
case "aac":
|
|
return "audio/aac"
|
|
case "flac":
|
|
return "audio/flac"
|
|
}
|
|
return "application/octet-stream"
|
|
}
|
|
|
|
// HealthHandler 暴露运行期状态;无鉴权。
|
|
func HealthHandler(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
|
|
// 安装模式下 /health 仍然 200,但通过 installed 字段让探针/运维识别
|
|
// (Kubernetes readiness probe 可以用 installed=false 决定是否放流量)
|
|
mode := installer.GetMode()
|
|
if mode == installer.ModeSetup {
|
|
w.WriteHeader(http.StatusOK) // 200,因为进程活着,只是还没初始化
|
|
} else if setting.TTSConfigErr != nil {
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
} else {
|
|
w.WriteHeader(http.StatusOK)
|
|
}
|
|
|
|
env := setting.CheckEnvironmentVariables()
|
|
allRequired := env["all_required_vars_set"].(bool)
|
|
|
|
status := "ok"
|
|
if mode == installer.ModeSetup {
|
|
status = "not_installed"
|
|
} else if !allRequired {
|
|
status = "configuration_error"
|
|
}
|
|
|
|
resp := dto.HealthResponse{
|
|
Status: status,
|
|
Service: "ByteDance TTS to OpenAI API Adapter",
|
|
Version: version.Version,
|
|
Commit: version.Commit,
|
|
Uptime: fmt.Sprintf("%.0f seconds", time.Since(startTime).Seconds()),
|
|
StartTime: startTime.Format(time.RFC3339),
|
|
Memory: collectMemorySnapshot(),
|
|
ConfigStatus: dto.ConfigStatusResponse{
|
|
AllRequiredVarsSet: allRequired,
|
|
ConfigError: setting.TTSConfigErr != nil,
|
|
Error: configErrorMessage(setting.TTSConfigErr),
|
|
},
|
|
Installed: mode == installer.ModeNormal,
|
|
Mode: mode.String(),
|
|
}
|
|
json.NewEncoder(w).Encode(resp)
|
|
}
|
|
|
|
// configErrorMessage 把 setting.TTSConfigErr 安全地转成可对外暴露的字符串。
|
|
// 仅在 normal 模式且有错时调用, error 为 nil 时返 "" (被 omitempty 跳过)。
|
|
func configErrorMessage(err error) string {
|
|
if err == nil {
|
|
return ""
|
|
}
|
|
return err.Error()
|
|
}
|
|
|
|
var startTime time.Time
|
|
|
|
func SetStartTime(t time.Time) { startTime = t }
|
|
|
|
func collectMemorySnapshot() map[string]interface{} {
|
|
var ms runtime.MemStats
|
|
runtime.ReadMemStats(&ms)
|
|
return map[string]interface{}{
|
|
"heap_alloc": ms.HeapAlloc,
|
|
"heap_inuse": ms.HeapInuse,
|
|
"goroutines": runtime.NumGoroutine(),
|
|
}
|
|
}
|