初始化仓库
This commit is contained in:
@@ -0,0 +1,175 @@
|
||||
//go:build blackbox
|
||||
|
||||
// Package p0 contains P0 blackbox tests — the hard release gate.
|
||||
//
|
||||
// These tests use real agents (Claude Code, Codex, etc.) and a MockPlatform.
|
||||
// They are skipped when agent binaries or API credentials are unavailable,
|
||||
// but FAIL on any logical assertion failure.
|
||||
//
|
||||
// Run all P0 tests:
|
||||
//
|
||||
// go test -tags blackbox ./tests/blackbox/p0/... -timeout 1800s -v
|
||||
//
|
||||
// Run against a specific agent:
|
||||
//
|
||||
// CC_BLACKBOX_CLAUDECODE_API_KEY=xxx go test -tags blackbox \
|
||||
// ./tests/blackbox/p0/... -run ClaudeCode -timeout 1800s -v
|
||||
package p0
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/chenhg5/cc-connect/tests/blackbox/helper"
|
||||
)
|
||||
|
||||
// ── P0-1: 基础消息收发 ────────────────────────────────────────────────────────
|
||||
|
||||
// TestP0_1_BasicMessageFlow_ClaudeCode verifies the core message loop:
|
||||
// user sends a message → cc-connect routes it to Claude Code → user receives
|
||||
// a non-empty reply.
|
||||
//
|
||||
// This is the most fundamental P0 test. If this fails, nothing else matters.
|
||||
func TestP0_1_BasicMessageFlow_ClaudeCode(t *testing.T) {
|
||||
t.Parallel()
|
||||
testBasicMessageFlow(t, "claudecode")
|
||||
}
|
||||
|
||||
func TestP0_1_BasicMessageFlow_Codex(t *testing.T) {
|
||||
t.Parallel()
|
||||
testBasicMessageFlow(t, "codex")
|
||||
}
|
||||
|
||||
// TestP0_1_BasicMessageFlow_Cursor tests the cursor agent (agent binary,
|
||||
// @anthropic-ai/cursor-agent). Set CC_BLACKBOX_CURSOR_API_KEY + optionally
|
||||
// CC_BLACKBOX_CURSOR_MODEL (e.g. "claude-sonnet-4-5") to run.
|
||||
// Cursor supports composer-2-fast via the model name; use a cheap model
|
||||
// for CI (e.g. "claude-haiku-3-5-20241022").
|
||||
func TestP0_1_BasicMessageFlow_Cursor(t *testing.T) {
|
||||
t.Parallel()
|
||||
testBasicMessageFlow(t, "cursor")
|
||||
}
|
||||
|
||||
// TestP0_1_BasicMessageFlow_OpenCode tests the opencode agent (opencode binary).
|
||||
// Set CC_BLACKBOX_OPENCODE_API_KEY and CC_BLACKBOX_OPENCODE_BASE_URL to use a
|
||||
// custom proxy (e.g. minimax, dragoncode) or set ANTHROPIC_API_KEY for direct
|
||||
// Anthropic API access.
|
||||
func TestP0_1_BasicMessageFlow_OpenCode(t *testing.T) {
|
||||
t.Parallel()
|
||||
testBasicMessageFlow(t, "opencode")
|
||||
}
|
||||
|
||||
func testBasicMessageFlow(t *testing.T, agentType string) {
|
||||
t.Helper()
|
||||
env := helper.NewEnv(t, agentType)
|
||||
|
||||
reply := env.Send("say hi briefly")
|
||||
|
||||
if reply == nil {
|
||||
t.Fatal("no reply received")
|
||||
}
|
||||
if strings.TrimSpace(reply.Text()) == "" {
|
||||
t.Fatalf("received empty reply; all messages:\n%s", env.Platform.AllText())
|
||||
}
|
||||
t.Logf("P0-1 OK: received reply (%d chars): %q", len(reply.Text()), truncate(reply.Text(), 200))
|
||||
}
|
||||
|
||||
// ── P0-2: /new 创建新会话 ─────────────────────────────────────────────────────
|
||||
|
||||
// TestP0_2_NewSession_ClaudeCode verifies /new creates a new session and
|
||||
// cc-connect sends a confirmation message.
|
||||
func TestP0_2_NewSession_ClaudeCode(t *testing.T) {
|
||||
t.Parallel()
|
||||
testNewSession(t, "claudecode")
|
||||
}
|
||||
|
||||
func testNewSession(t *testing.T, agentType string) {
|
||||
t.Helper()
|
||||
env := helper.NewEnv(t, agentType)
|
||||
|
||||
// First, establish a session by sending a real message.
|
||||
env.Send("say hello briefly")
|
||||
|
||||
// Now create a new session.
|
||||
before := env.Platform.MessageCount()
|
||||
env.SendNoWait("/new")
|
||||
|
||||
reply := env.Platform.WaitForReply(before, 30*time.Second)
|
||||
if reply == nil {
|
||||
t.Fatal("/new: no confirmation message received within 30s")
|
||||
}
|
||||
t.Logf("P0-2 OK: /new replied: %q", truncate(reply.Text(), 200))
|
||||
}
|
||||
|
||||
// ── P0-3: /list 显示会话列表 ──────────────────────────────────────────────────
|
||||
|
||||
// TestP0_3_ListSessions_ClaudeCode verifies /list returns a message containing
|
||||
// session information after at least one session has been created.
|
||||
func TestP0_3_ListSessions_ClaudeCode(t *testing.T) {
|
||||
t.Parallel()
|
||||
testListSessions(t, "claudecode")
|
||||
}
|
||||
|
||||
func testListSessions(t *testing.T, agentType string) {
|
||||
t.Helper()
|
||||
env := helper.NewEnv(t, agentType)
|
||||
|
||||
// Establish a session.
|
||||
env.Send("say hello briefly")
|
||||
|
||||
// /list should return session information.
|
||||
reply := env.Send("/list")
|
||||
|
||||
text := strings.ToLower(reply.Text())
|
||||
// The response should reference sessions in some form.
|
||||
hasSessionInfo := strings.Contains(text, "session") ||
|
||||
strings.Contains(text, "会话") ||
|
||||
strings.Contains(text, "#1") ||
|
||||
strings.Contains(text, "no sessions") ||
|
||||
strings.Contains(text, "没有会话")
|
||||
|
||||
if !hasSessionInfo {
|
||||
t.Errorf("/list response doesn't look like a session list\ngot: %q\nall messages:\n%s",
|
||||
reply.Text(), env.Platform.AllText())
|
||||
}
|
||||
t.Logf("P0-3 OK: /list replied: %q", truncate(reply.Text(), 300))
|
||||
}
|
||||
|
||||
// ── P0-5: /stop 停止当前任务 ──────────────────────────────────────────────────
|
||||
|
||||
// TestP0_5_StopCurrentTask_ClaudeCode verifies /stop causes cc-connect to
|
||||
// reply within a reasonable time even while a task is running.
|
||||
//
|
||||
// Strategy: we don't send a genuinely long-running task (too unreliable);
|
||||
// instead we verify /stop itself produces a reply when no task is active,
|
||||
// which still confirms the command is handled.
|
||||
func TestP0_5_StopCurrentTask_ClaudeCode(t *testing.T) {
|
||||
t.Parallel()
|
||||
testStop(t, "claudecode")
|
||||
}
|
||||
|
||||
func testStop(t *testing.T, agentType string) {
|
||||
t.Helper()
|
||||
env := helper.NewEnv(t, agentType)
|
||||
|
||||
// Establish a session first so /stop has context.
|
||||
env.Send("say hello briefly")
|
||||
|
||||
// Issue /stop and expect a timely response.
|
||||
reply := env.SendWithTimeout("/stop", 30*time.Second)
|
||||
|
||||
if strings.TrimSpace(reply.Text()) == "" {
|
||||
t.Fatalf("/stop produced empty reply; all messages:\n%s", env.Platform.AllText())
|
||||
}
|
||||
t.Logf("P0-5 OK: /stop replied: %q", truncate(reply.Text(), 200))
|
||||
}
|
||||
|
||||
// ── helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
func truncate(s string, max int) string {
|
||||
if len(s) <= max {
|
||||
return s
|
||||
}
|
||||
return s[:max] + "…"
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
//go:build blackbox
|
||||
|
||||
package p0
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/chenhg5/cc-connect/tests/blackbox/helper"
|
||||
)
|
||||
|
||||
// ── P0-11: 上下文保持（最核心场景）──────────────────────────────────────────
|
||||
//
|
||||
// This is arguably the most important test in the suite. It validates that:
|
||||
// 1. cc-connect maintains a persistent session across multiple messages
|
||||
// 2. the real agent (Claude Code) retains conversation context
|
||||
// 3. the session key routing is correct end-to-end
|
||||
//
|
||||
// If context retention breaks, users cannot have multi-turn conversations —
|
||||
// the entire product stops working for its core use case.
|
||||
|
||||
// TestP0_11_ContextRetention_ClaudeCode sends a name in turn 1 and verifies
|
||||
// the agent recalls it in turn 2. This is only possible with a real persistent
|
||||
// session — a fake agent would need the name hardcoded.
|
||||
func TestP0_11_ContextRetention_ClaudeCode(t *testing.T) {
|
||||
t.Parallel()
|
||||
testContextRetention(t, "claudecode")
|
||||
}
|
||||
|
||||
func TestP0_11_ContextRetention_Codex(t *testing.T) {
|
||||
t.Parallel()
|
||||
testContextRetention(t, "codex")
|
||||
}
|
||||
|
||||
func TestP0_11_ContextRetention_Cursor(t *testing.T) {
|
||||
t.Parallel()
|
||||
testContextRetention(t, "cursor")
|
||||
}
|
||||
|
||||
func TestP0_11_ContextRetention_OpenCode(t *testing.T) {
|
||||
t.Parallel()
|
||||
testContextRetention(t, "opencode")
|
||||
}
|
||||
|
||||
func testContextRetention(t *testing.T, agentType string) {
|
||||
t.Helper()
|
||||
env := helper.NewEnv(t, agentType)
|
||||
|
||||
// Turn 1: plant a unique fact. Use SendComplete so we wait for the full
|
||||
// agent turn before sending turn 2 — otherwise the second message gets
|
||||
// queued while the agent is still processing and loses context.
|
||||
const uniqueName = "ContextTestMarker42"
|
||||
turn1 := env.SendComplete("My name is " + uniqueName + ". Just acknowledge it briefly.")
|
||||
|
||||
turn1Text := helper.AnyText(turn1)
|
||||
if strings.TrimSpace(turn1Text) == "" {
|
||||
t.Fatalf("turn 1: empty reply; all messages:\n%s", env.Platform.AllText())
|
||||
}
|
||||
t.Logf("turn 1 (%d msgs): %q", len(turn1), truncate(turn1Text, 200))
|
||||
|
||||
// Turn 2: ask the agent to recall the fact.
|
||||
turn2 := env.SendComplete("What is my name? Reply with just the name.")
|
||||
|
||||
turn2Text := helper.AnyText(turn2)
|
||||
if strings.TrimSpace(turn2Text) == "" {
|
||||
t.Fatalf("turn 2: empty reply; all messages:\n%s", env.Platform.AllText())
|
||||
}
|
||||
t.Logf("turn 2 (%d msgs): %q", len(turn2), truncate(turn2Text, 200))
|
||||
|
||||
// The agent must recall the name planted in turn 1.
|
||||
if !strings.Contains(strings.ToLower(turn2Text), strings.ToLower(uniqueName)) {
|
||||
t.Errorf(
|
||||
"context retention FAILED: turn 2 does not contain %q\n"+
|
||||
"turn 1: %q\n"+
|
||||
"turn 2: %q\n"+
|
||||
"all messages:\n%s",
|
||||
uniqueName, turn1Text, turn2Text, env.Platform.AllText(),
|
||||
)
|
||||
} else {
|
||||
t.Logf("P0-11 OK: agent recalled %q in turn 2", uniqueName)
|
||||
}
|
||||
}
|
||||
|
||||
// TestP0_11_ContextIsolation_ClaudeCode verifies that two different users
|
||||
// (different session keys) have completely independent sessions.
|
||||
// User A's context must NOT appear in User B's session.
|
||||
func TestP0_11_ContextIsolation_ClaudeCode(t *testing.T) {
|
||||
t.Parallel()
|
||||
testContextIsolation(t, "claudecode")
|
||||
}
|
||||
|
||||
func testContextIsolation(t *testing.T, agentType string) {
|
||||
t.Helper()
|
||||
env := helper.NewEnv(t, agentType)
|
||||
|
||||
const nameA = "AliceIsolationTest"
|
||||
const nameB = "BobIsolationTest"
|
||||
|
||||
// User A plants their name (wait for full turn).
|
||||
msgsA1 := env.SendCompleteAs("userA", "chat1", "My name is "+nameA+". Just say ok.", 5*time.Second, helper.DefaultReplyTimeout)
|
||||
t.Logf("userA turn 1: %q", truncate(helper.AnyText(msgsA1), 150))
|
||||
|
||||
// User B (different session) plants their name (wait for full turn).
|
||||
msgsB1 := env.SendCompleteAs("userB", "chat1", "My name is "+nameB+". Just say ok.", 5*time.Second, helper.DefaultReplyTimeout)
|
||||
t.Logf("userB turn 1: %q", truncate(helper.AnyText(msgsB1), 150))
|
||||
|
||||
// User A asks — should see only their own name.
|
||||
msgsA2 := env.SendCompleteAs("userA", "chat1", "What is my name?", 5*time.Second, helper.DefaultReplyTimeout)
|
||||
textA2 := helper.AnyText(msgsA2)
|
||||
t.Logf("userA turn 2: %q", truncate(textA2, 150))
|
||||
|
||||
if !strings.Contains(strings.ToLower(textA2), strings.ToLower(nameA)) {
|
||||
t.Errorf("userA context lost: expected %q in reply, got: %q", nameA, textA2)
|
||||
}
|
||||
if strings.Contains(strings.ToLower(textA2), strings.ToLower(nameB)) {
|
||||
t.Errorf("session leak: userA reply contains userB's name %q: %q", nameB, textA2)
|
||||
}
|
||||
t.Logf("P0-11 isolation OK")
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
//go:build blackbox
|
||||
|
||||
package p0
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/chenhg5/cc-connect/tests/blackbox/helper"
|
||||
)
|
||||
|
||||
// ── P0-14: 工具调用 ───────────────────────────────────────────────────────────
|
||||
|
||||
// TestP0_14_ToolInvocation_ClaudeCode verifies that cc-connect routes a task
|
||||
// requiring tool use to the agent and the agent successfully executes it.
|
||||
// A real agent calling real tools is the only way to verify this end-to-end.
|
||||
func TestP0_14_ToolInvocation_ClaudeCode(t *testing.T) {
|
||||
t.Parallel()
|
||||
testToolInvocation(t, "claudecode")
|
||||
}
|
||||
|
||||
func testToolInvocation(t *testing.T, agentType string) {
|
||||
t.Helper()
|
||||
env := helper.NewEnv(t, agentType)
|
||||
|
||||
// Ask something that requires a Bash/filesystem tool call.
|
||||
msgs := env.SendComplete("List the files in the current directory using a shell command. Just show me the output.")
|
||||
|
||||
combined := helper.AnyText(msgs)
|
||||
if strings.TrimSpace(combined) == "" {
|
||||
t.Fatalf("empty reply; all messages:\n%s", env.Platform.AllText())
|
||||
}
|
||||
|
||||
// The agent should have invoked a tool — expect a tool-call indicator or
|
||||
// file listing output. We check for the Tool indicator message or any
|
||||
// file/directory names in the response.
|
||||
hasToolEvidence := strings.Contains(combined, "🔧") ||
|
||||
strings.Contains(combined, "Tool") ||
|
||||
strings.Contains(combined, "Bash") ||
|
||||
strings.Contains(combined, ".go") ||
|
||||
strings.Contains(combined, "go.mod") ||
|
||||
strings.Contains(combined, "main") ||
|
||||
strings.Contains(combined, "ls") ||
|
||||
strings.Contains(strings.ToLower(combined), "file") ||
|
||||
strings.Contains(strings.ToLower(combined), "directory") ||
|
||||
strings.Contains(strings.ToLower(combined), "目录") ||
|
||||
strings.Contains(strings.ToLower(combined), "文件")
|
||||
|
||||
if !hasToolEvidence {
|
||||
t.Errorf("P0-14: no evidence of tool invocation in response\nall messages:\n%s", env.Platform.AllText())
|
||||
} else {
|
||||
t.Logf("P0-14 OK: tool invocation detected (%d msgs, %d chars)", len(msgs), len(combined))
|
||||
}
|
||||
}
|
||||
|
||||
// ── P0-15: 长消息流式输出 ─────────────────────────────────────────────────────
|
||||
|
||||
// TestP0_15_LongResponse_ClaudeCode verifies that cc-connect correctly handles
|
||||
// a long agent response without truncation or corruption. The agent must
|
||||
// produce a substantive response (≥ 200 chars total).
|
||||
func TestP0_15_LongResponse_ClaudeCode(t *testing.T) {
|
||||
t.Parallel()
|
||||
testLongResponse(t, "claudecode")
|
||||
}
|
||||
|
||||
func testLongResponse(t *testing.T, agentType string) {
|
||||
t.Helper()
|
||||
env := helper.NewEnv(t, agentType)
|
||||
|
||||
msgs := env.SendComplete("Write a short paragraph (about 150 words) explaining what software testing is. Write in plain text.")
|
||||
|
||||
combined := helper.AnyText(msgs)
|
||||
if len(strings.TrimSpace(combined)) < 200 {
|
||||
t.Errorf("P0-15: response too short (%d chars); expected ≥ 200\nall messages:\n%s",
|
||||
len(combined), env.Platform.AllText())
|
||||
} else {
|
||||
t.Logf("P0-15 OK: long response received (%d msgs, %d chars)", len(msgs), len(combined))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user