初始化仓库
This commit is contained in:
+533
@@ -0,0 +1,533 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// SpeechToText transcribes audio to text.
|
||||
type SpeechToText interface {
|
||||
Transcribe(ctx context.Context, audio []byte, format string, lang string) (string, error)
|
||||
}
|
||||
|
||||
// SpeechConfig holds STT configuration for the engine.
|
||||
type SpeechCfg struct {
|
||||
Enabled bool
|
||||
Provider string
|
||||
Language string
|
||||
STT SpeechToText
|
||||
}
|
||||
|
||||
// OpenAIWhisper implements SpeechToText using the OpenAI-compatible Whisper API.
|
||||
// Works with OpenAI, Groq, and any endpoint that implements the same multipart API.
|
||||
type OpenAIWhisper struct {
|
||||
APIKey string
|
||||
BaseURL string
|
||||
Model string
|
||||
Client *http.Client
|
||||
}
|
||||
|
||||
func NewOpenAIWhisper(apiKey, baseURL, model string) *OpenAIWhisper {
|
||||
if baseURL == "" {
|
||||
baseURL = "https://api.openai.com/v1"
|
||||
}
|
||||
if model == "" {
|
||||
model = "whisper-1"
|
||||
}
|
||||
return &OpenAIWhisper{
|
||||
APIKey: apiKey,
|
||||
BaseURL: strings.TrimRight(baseURL, "/"),
|
||||
Model: model,
|
||||
Client: &http.Client{Timeout: 5 * time.Minute},
|
||||
}
|
||||
}
|
||||
|
||||
func (w *OpenAIWhisper) Transcribe(ctx context.Context, audio []byte, format string, lang string) (string, error) {
|
||||
ext := formatToExt(format)
|
||||
|
||||
var buf bytes.Buffer
|
||||
writer := multipart.NewWriter(&buf)
|
||||
|
||||
part, err := writer.CreateFormFile("file", "audio."+ext)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create form file: %w", err)
|
||||
}
|
||||
if _, err := part.Write(audio); err != nil {
|
||||
return "", fmt.Errorf("write audio: %w", err)
|
||||
}
|
||||
_ = writer.WriteField("model", w.Model)
|
||||
_ = writer.WriteField("response_format", "text")
|
||||
if lang != "" {
|
||||
_ = writer.WriteField("language", lang)
|
||||
}
|
||||
writer.Close()
|
||||
|
||||
url := w.BaseURL + "/audio/transcriptions"
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, &buf)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+w.APIKey)
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
|
||||
resp, err := w.Client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("whisper request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read response: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("whisper API %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
// response_format=text returns plain text; try to handle JSON fallback
|
||||
text := strings.TrimSpace(string(body))
|
||||
if strings.HasPrefix(text, "{") {
|
||||
var jr struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
if json.Unmarshal(body, &jr) == nil {
|
||||
text = jr.Text
|
||||
}
|
||||
}
|
||||
return text, nil
|
||||
}
|
||||
|
||||
// QwenASR implements SpeechToText using the Qwen ASR model via DashScope's
|
||||
// OpenAI-compatible chat completions API. Unlike Whisper, audio is sent as a
|
||||
// base64 data URI inside the messages array.
|
||||
type QwenASR struct {
|
||||
APIKey string
|
||||
BaseURL string
|
||||
Model string
|
||||
Client *http.Client
|
||||
}
|
||||
|
||||
func NewQwenASR(apiKey, baseURL, model string) *QwenASR {
|
||||
if baseURL == "" {
|
||||
baseURL = "https://dashscope.aliyuncs.com/compatible-mode/v1"
|
||||
}
|
||||
if model == "" {
|
||||
model = "qwen3-asr-flash"
|
||||
}
|
||||
return &QwenASR{
|
||||
APIKey: apiKey,
|
||||
BaseURL: strings.TrimRight(baseURL, "/"),
|
||||
Model: model,
|
||||
Client: &http.Client{Timeout: 5 * time.Minute},
|
||||
}
|
||||
}
|
||||
|
||||
func (q *QwenASR) Transcribe(ctx context.Context, audio []byte, format string, lang string) (string, error) {
|
||||
b64 := base64.StdEncoding.EncodeToString(audio)
|
||||
dataURI := fmt.Sprintf("data:%s;base64,%s", formatToAudioMIME(format), b64)
|
||||
|
||||
reqBody := map[string]any{
|
||||
"model": q.Model,
|
||||
"messages": []map[string]any{
|
||||
{
|
||||
"role": "user",
|
||||
"content": []map[string]any{
|
||||
{
|
||||
"type": "input_audio",
|
||||
"input_audio": map[string]any{
|
||||
"data": dataURI,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("marshal request: %w", err)
|
||||
}
|
||||
|
||||
url := q.BaseURL + "/chat/completions"
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(jsonData))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+q.APIKey)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := q.Client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("qwen asr request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read response: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("qwen asr API %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
var result struct {
|
||||
Choices []struct {
|
||||
Message struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"message"`
|
||||
} `json:"choices"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &result); err != nil {
|
||||
return "", fmt.Errorf("parse response: %w", err)
|
||||
}
|
||||
if len(result.Choices) == 0 {
|
||||
return "", fmt.Errorf("qwen asr: empty choices in response")
|
||||
}
|
||||
|
||||
return strings.TrimSpace(result.Choices[0].Message.Content), nil
|
||||
}
|
||||
|
||||
// GeminiSTT implements SpeechToText using the Google Gemini API.
|
||||
// Audio is sent as inline_data (base64) in the contents array against the
|
||||
// generateContent endpoint; the API key is sent via the x-goog-api-key header.
|
||||
type GeminiSTT struct {
|
||||
APIKey string
|
||||
Model string
|
||||
BaseURL string // internal; defaults to Google API, overridable for testing
|
||||
Client *http.Client
|
||||
}
|
||||
|
||||
func NewGeminiSTT(apiKey, model string) *GeminiSTT {
|
||||
if model == "" {
|
||||
model = "gemini-flash-latest"
|
||||
}
|
||||
return &GeminiSTT{
|
||||
APIKey: apiKey,
|
||||
Model: model,
|
||||
BaseURL: "https://generativelanguage.googleapis.com/v1beta",
|
||||
Client: &http.Client{Timeout: 60 * time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
func (g *GeminiSTT) Transcribe(ctx context.Context, audio []byte, format string, lang string) (string, error) {
|
||||
b64 := base64.StdEncoding.EncodeToString(audio)
|
||||
mime := formatToAudioMIME(format)
|
||||
|
||||
prompt := "Transcribe this audio accurately. Output only the transcribed text, nothing else."
|
||||
if lang != "" {
|
||||
prompt = fmt.Sprintf("Transcribe this audio accurately in %s. Output only the transcribed text, nothing else.", lang)
|
||||
}
|
||||
|
||||
reqBody := map[string]any{
|
||||
"contents": []map[string]any{
|
||||
{
|
||||
"parts": []map[string]any{
|
||||
{
|
||||
"inline_data": map[string]any{
|
||||
"mime_type": mime,
|
||||
"data": b64,
|
||||
},
|
||||
},
|
||||
{
|
||||
"text": prompt,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("gemini stt: marshal request: %w", err)
|
||||
}
|
||||
|
||||
apiURL := fmt.Sprintf("%s/models/%s:generateContent", g.BaseURL, url.PathEscape(g.Model))
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, apiURL, bytes.NewReader(jsonData))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("gemini stt: create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("x-goog-api-key", g.APIKey)
|
||||
|
||||
resp, err := g.Client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("gemini stt: request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("gemini stt: read response: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("gemini stt API %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
var result struct {
|
||||
Candidates []struct {
|
||||
Content struct {
|
||||
Parts []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"parts"`
|
||||
} `json:"content"`
|
||||
} `json:"candidates"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &result); err != nil {
|
||||
return "", fmt.Errorf("gemini stt: parse response: %w", err)
|
||||
}
|
||||
if len(result.Candidates) == 0 || len(result.Candidates[0].Content.Parts) == 0 {
|
||||
return "", fmt.Errorf("gemini stt: empty response")
|
||||
}
|
||||
|
||||
return strings.TrimSpace(result.Candidates[0].Content.Parts[0].Text), nil
|
||||
}
|
||||
|
||||
// ConvertAudioToMP3 uses ffmpeg to convert audio from unsupported formats to mp3.
|
||||
// Returns the mp3 bytes. If ffmpeg is not installed, returns an error.
|
||||
// The ctx is honored: cancellation kills the ffmpeg subprocess, matching the
|
||||
// behavior of the other Convert* helpers in this file.
|
||||
func ConvertAudioToMP3(ctx context.Context, audio []byte, srcFormat string) ([]byte, error) {
|
||||
ffmpegPath, err := exec.LookPath("ffmpeg")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ffmpeg not found in PATH: install ffmpeg to enable voice message support")
|
||||
}
|
||||
|
||||
var cmd *exec.Cmd
|
||||
if srcFormat == "amr" || srcFormat == "silk" {
|
||||
cmd = exec.CommandContext(ctx, ffmpegPath,
|
||||
"-f", srcFormat,
|
||||
"-i", "pipe:0",
|
||||
"-f", "mp3",
|
||||
"-ac", "1",
|
||||
"-ar", "16000",
|
||||
"-y",
|
||||
"pipe:1",
|
||||
)
|
||||
} else {
|
||||
cmd = exec.CommandContext(ctx, ffmpegPath,
|
||||
"-i", "pipe:0",
|
||||
"-f", "mp3",
|
||||
"-ac", "1",
|
||||
"-ar", "16000",
|
||||
"-y",
|
||||
"pipe:1",
|
||||
)
|
||||
}
|
||||
|
||||
cmd.Stdin = bytes.NewReader(audio)
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
return nil, fmt.Errorf("ffmpeg conversion failed: %w (stderr: %s)", err, stderr.String())
|
||||
}
|
||||
return stdout.Bytes(), nil
|
||||
}
|
||||
|
||||
// ConvertAudioToOpus uses ffmpeg to convert audio to opus format (ogg container).
|
||||
// Returns the opus bytes. If ffmpeg is not installed, returns an error.
|
||||
func ConvertAudioToOpus(ctx context.Context, audio []byte, srcFormat string) ([]byte, error) {
|
||||
ffmpegPath, err := exec.LookPath("ffmpeg")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ffmpeg not found in PATH: install ffmpeg to enable audio conversion")
|
||||
}
|
||||
|
||||
args := []string{"-i", "pipe:0", "-c:a", "libopus", "-f", "opus", "-y", "pipe:1"}
|
||||
if srcFormat == "amr" || srcFormat == "silk" {
|
||||
args = append([]string{"-f", srcFormat}, args...)
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, ffmpegPath, args...)
|
||||
cmd.Stdin = bytes.NewReader(audio)
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
return nil, fmt.Errorf("ffmpeg opus conversion failed: %w (stderr: %s)", err, stderr.String())
|
||||
}
|
||||
return stdout.Bytes(), nil
|
||||
}
|
||||
|
||||
// ConvertAudioToAMR uses ffmpeg to convert audio to AMR-NB format.
|
||||
// AMR is a common voice codec for mobile messaging platforms.
|
||||
// Returns the AMR bytes. If ffmpeg is not installed, returns an error.
|
||||
func ConvertAudioToAMR(ctx context.Context, audio []byte, srcFormat string) ([]byte, error) {
|
||||
ffmpegPath, err := exec.LookPath("ffmpeg")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ffmpeg not found in PATH: install ffmpeg to enable audio conversion")
|
||||
}
|
||||
|
||||
args := []string{
|
||||
"-i", "pipe:0",
|
||||
"-c:a", "amr_nb",
|
||||
"-ar", "8000", // 8kHz sample rate (AMR-NB standard)
|
||||
"-ac", "1", // mono
|
||||
"-b:a", "12.2k", // 12.2 kbps bitrate (AMR-NB max)
|
||||
"-f", "amr",
|
||||
"-y",
|
||||
"pipe:1",
|
||||
}
|
||||
if srcFormat == "amr" || srcFormat == "silk" {
|
||||
args = append([]string{"-f", srcFormat}, args...)
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, ffmpegPath, args...)
|
||||
cmd.Stdin = bytes.NewReader(audio)
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
return nil, fmt.Errorf("ffmpeg AMR conversion failed: %w (stderr: %s)", err, stderr.String())
|
||||
}
|
||||
return stdout.Bytes(), nil
|
||||
}
|
||||
|
||||
// ConvertMP3ToOGG converts MP3 audio to OGG format using ffmpeg with stdin/stdout pipes.
|
||||
// Optimized for voice: Opus codec, 16kHz mono, 32kbps, voip application.
|
||||
func ConvertMP3ToOGG(ctx context.Context, mp3Data []byte) ([]byte, error) {
|
||||
ffmpegPath, err := exec.LookPath("ffmpeg")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ffmpeg not found in PATH: %w", err)
|
||||
}
|
||||
|
||||
args := []string{
|
||||
"-i", "pipe:0",
|
||||
"-c:a", "libopus",
|
||||
"-ar", "16000", // 16kHz sample rate for voice
|
||||
"-ac", "1", // mono
|
||||
"-b:a", "32k", // 32 kbps bitrate (voice quality)
|
||||
"-application", "voip", // optimize for voice
|
||||
"-f", "ogg",
|
||||
"-y",
|
||||
"pipe:1",
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, ffmpegPath, args...)
|
||||
cmd.Stdin = bytes.NewReader(mp3Data)
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
return nil, fmt.Errorf("ffmpeg MP3 to OGG conversion failed: %w (stderr: %s)", err, stderr.String())
|
||||
}
|
||||
return stdout.Bytes(), nil
|
||||
}
|
||||
|
||||
// ConvertMP3ToAMR converts MP3 audio to AMR format using ffmpeg with stdin/stdout pipes.
|
||||
// AMR format is smaller but lower quality than OGG (AMR-NB codec, 8kHz mono, 12.2kbps).
|
||||
func ConvertMP3ToAMR(ctx context.Context, mp3Data []byte) ([]byte, error) {
|
||||
ffmpegPath, err := exec.LookPath("ffmpeg")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ffmpeg not found in PATH: %w", err)
|
||||
}
|
||||
|
||||
args := []string{
|
||||
"-i", "pipe:0",
|
||||
"-c:a", "amr_nb",
|
||||
"-ar", "8000", // 8kHz sample rate (AMR-NB standard)
|
||||
"-ac", "1", // mono
|
||||
"-b:a", "12.2k", // 12.2 kbps bitrate (AMR-NB max)
|
||||
"-f", "amr",
|
||||
"-y",
|
||||
"pipe:1",
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, ffmpegPath, args...)
|
||||
cmd.Stdin = bytes.NewReader(mp3Data)
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
return nil, fmt.Errorf("ffmpeg MP3 to AMR conversion failed: %w (stderr: %s)", err, stderr.String())
|
||||
}
|
||||
return stdout.Bytes(), nil
|
||||
}
|
||||
|
||||
// NeedsConversion returns true if the audio format is not directly supported by Whisper API.
|
||||
func NeedsConversion(format string) bool {
|
||||
switch strings.ToLower(format) {
|
||||
case "mp3", "mp4", "mpeg", "mpga", "m4a", "wav", "webm":
|
||||
return false
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// HasFFmpeg checks if ffmpeg is available.
|
||||
func HasFFmpeg() bool {
|
||||
_, err := exec.LookPath("ffmpeg")
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func formatToExt(format string) string {
|
||||
switch strings.ToLower(format) {
|
||||
case "amr":
|
||||
return "amr"
|
||||
case "ogg", "oga", "opus":
|
||||
return "ogg"
|
||||
case "m4a", "mp4", "aac":
|
||||
return "m4a"
|
||||
case "mp3":
|
||||
return "mp3"
|
||||
case "wav":
|
||||
return "wav"
|
||||
case "webm":
|
||||
return "webm"
|
||||
case "silk":
|
||||
return "silk"
|
||||
default:
|
||||
return format
|
||||
}
|
||||
}
|
||||
|
||||
func formatToAudioMIME(format string) string {
|
||||
switch strings.ToLower(format) {
|
||||
case "mp3", "mpeg", "mpga":
|
||||
return "audio/mpeg"
|
||||
case "wav":
|
||||
return "audio/wav"
|
||||
case "ogg", "oga", "opus":
|
||||
return "audio/ogg"
|
||||
case "m4a", "mp4", "aac":
|
||||
return "audio/mp4"
|
||||
case "webm":
|
||||
return "audio/webm"
|
||||
default:
|
||||
return "audio/octet-stream"
|
||||
}
|
||||
}
|
||||
|
||||
// TranscribeAudio is a convenience function used by the Engine.
|
||||
// It handles format conversion (if needed) and calls the STT provider.
|
||||
func TranscribeAudio(ctx context.Context, stt SpeechToText, audio *AudioAttachment, lang string) (string, error) {
|
||||
data := audio.Data
|
||||
format := strings.ToLower(audio.Format)
|
||||
|
||||
if NeedsConversion(format) {
|
||||
slog.Debug("speech: converting audio", "from", format, "to", "mp3")
|
||||
converted, err := ConvertAudioToMP3(ctx, data, format)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
data = converted
|
||||
format = "mp3"
|
||||
}
|
||||
|
||||
slog.Debug("speech: transcribing", "format", format, "size", len(data))
|
||||
return stt.Transcribe(ctx, data, format, lang)
|
||||
}
|
||||
Reference in New Issue
Block a user