354 lines
15 KiB
Python
354 lines
15 KiB
Python
"""E2E Basic Function Tests — CLI commands.
|
|
|
|
Verifies that all CLI commands execute correctly as a real user would invoke them.
|
|
Uses subprocess (OpenCLI pattern) to simulate actual CLI operations.
|
|
|
|
Test categories:
|
|
1. Utility commands: version, doctor, help
|
|
2. Init & config: agentkit init
|
|
3. Pair: API key generation
|
|
4. Skill management: list, load, info
|
|
5. Task management: submit, status, list, cancel
|
|
6. Server: serve startup
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
|
|
import pytest
|
|
|
|
from tests.e2e.conftest import CLIRunner, E2E_BASE_URL
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# 1. Utility Commands
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
@pytest.mark.e2e_basic
|
|
class TestCLIVersion:
|
|
"""agentkit version — basic sanity check."""
|
|
|
|
def test_version_returns_zero_exit_code(self, cli_runner: CLIRunner):
|
|
result = cli_runner.run(["version"])
|
|
assert result.returncode == 0, f"stdout: {result.stdout}\nstderr: {result.stderr}"
|
|
|
|
def test_version_outputs_version_string(self, cli_runner: CLIRunner):
|
|
result = cli_runner.run(["version"])
|
|
assert "0.1.0" in result.stdout or "fischer-agentkit" in result.stdout.lower()
|
|
|
|
def test_version_help(self, cli_runner: CLIRunner):
|
|
result = cli_runner.run(["version", "--help"])
|
|
assert result.returncode == 0
|
|
|
|
|
|
@pytest.mark.e2e_basic
|
|
class TestCLIDoctor:
|
|
"""agentkit doctor — server health check."""
|
|
|
|
def test_doctor_server_not_running(self, cli_runner: CLIRunner):
|
|
"""Doctor should report error when no server is running."""
|
|
result = cli_runner.run(["doctor"])
|
|
# Should indicate server not reachable
|
|
output = (result.stdout + result.stderr).lower()
|
|
assert (
|
|
result.returncode != 0
|
|
or "not running" in output
|
|
or "error" in output
|
|
or "connection" in output
|
|
)
|
|
|
|
def test_doctor_with_running_server(self, cli_runner_session: CLIRunner):
|
|
"""Doctor should report healthy when E2E server is running."""
|
|
result = cli_runner_session.run(["doctor", "--port", "18765"])
|
|
output = (result.stdout + result.stderr).lower()
|
|
# Should show some health info (ok, healthy, or at least not connection refused)
|
|
assert "connection refused" not in output or result.returncode == 0
|
|
|
|
|
|
@pytest.mark.e2e_basic
|
|
class TestCLIHelp:
|
|
"""agentkit --help — command discovery."""
|
|
|
|
def test_help_shows_all_subcommands(self, cli_runner: CLIRunner):
|
|
result = cli_runner.run(["--help"])
|
|
assert result.returncode == 0
|
|
for cmd in [
|
|
"serve",
|
|
"gui",
|
|
"chat",
|
|
"version",
|
|
"doctor",
|
|
"init",
|
|
"task",
|
|
"skill",
|
|
"usage",
|
|
"pair",
|
|
]:
|
|
assert cmd in result.stdout, f"Missing subcommand '{cmd}' in help output"
|
|
|
|
def test_task_help(self, cli_runner: CLIRunner):
|
|
result = cli_runner.run(["task", "--help"])
|
|
assert result.returncode == 0
|
|
for sub in ["submit", "status", "list", "cancel"]:
|
|
assert sub in result.stdout
|
|
|
|
def test_skill_help(self, cli_runner: CLIRunner):
|
|
result = cli_runner.run(["skill", "--help"])
|
|
assert result.returncode == 0
|
|
for sub in ["list", "load", "info"]:
|
|
assert sub in result.stdout
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# 2. Init & Config
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
@pytest.mark.e2e_basic
|
|
class TestCLIInit:
|
|
"""agentkit init — project initialization."""
|
|
|
|
def test_init_non_interactive(self, cli_runner: CLIRunner, tmp_path):
|
|
output_dir = str(tmp_path / "init_output")
|
|
os.makedirs(output_dir, exist_ok=True)
|
|
result = cli_runner.run(["init", "--non-interactive", "--output-dir", output_dir])
|
|
assert result.returncode == 0, f"stderr: {result.stderr}"
|
|
assert os.path.exists(os.path.join(output_dir, "agentkit.yaml"))
|
|
assert os.path.exists(os.path.join(output_dir, ".env.example"))
|
|
|
|
def test_init_generates_valid_yaml(self, cli_runner: CLIRunner, tmp_path):
|
|
import yaml
|
|
|
|
output_dir = str(tmp_path / "init_yaml")
|
|
os.makedirs(output_dir, exist_ok=True)
|
|
cli_runner.run(["init", "--non-interactive", "--output-dir", output_dir])
|
|
with open(os.path.join(output_dir, "agentkit.yaml")) as f:
|
|
config = yaml.safe_load(f)
|
|
assert "server" in config
|
|
assert "llm" in config
|
|
|
|
def test_init_no_overwrite_without_force(self, cli_runner: CLIRunner, tmp_path):
|
|
output_dir = str(tmp_path / "init_no_overwrite")
|
|
os.makedirs(output_dir, exist_ok=True)
|
|
# Create existing file
|
|
with open(os.path.join(output_dir, "agentkit.yaml"), "w") as f:
|
|
f.write("existing_content")
|
|
cli_runner.run(["init", "--non-interactive", "--output-dir", output_dir])
|
|
with open(os.path.join(output_dir, "agentkit.yaml")) as f:
|
|
content = f.read()
|
|
# Should not overwrite
|
|
assert content == "existing_content"
|
|
|
|
def test_init_force_overwrites(self, cli_runner: CLIRunner, tmp_path):
|
|
output_dir = str(tmp_path / "init_force")
|
|
os.makedirs(output_dir, exist_ok=True)
|
|
with open(os.path.join(output_dir, "agentkit.yaml"), "w") as f:
|
|
f.write("old")
|
|
result = cli_runner.run(
|
|
["init", "--non-interactive", "--force", "--output-dir", output_dir]
|
|
)
|
|
assert result.returncode == 0
|
|
with open(os.path.join(output_dir, "agentkit.yaml")) as f:
|
|
content = f.read()
|
|
assert "server" in content
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# 3. Pair (API Key Generation)
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
@pytest.mark.e2e_basic
|
|
class TestCLIPair:
|
|
"""agentkit pair — external system API key management."""
|
|
|
|
def test_pair_generates_api_key(self, cli_runner: CLIRunner, tmp_path):
|
|
config_dir = str(tmp_path / "pair_config")
|
|
os.makedirs(config_dir, exist_ok=True)
|
|
result = cli_runner.run(["pair", "--name", "e2e-test-client", "--config-dir", config_dir])
|
|
assert result.returncode == 0, f"stderr: {result.stderr}"
|
|
assert "ak_live_" in result.stdout
|
|
|
|
def test_pair_saves_client_config(self, cli_runner: CLIRunner, tmp_path):
|
|
import yaml
|
|
|
|
config_dir = str(tmp_path / "pair_save")
|
|
os.makedirs(config_dir, exist_ok=True)
|
|
cli_runner.run(["pair", "--name", "e2e-client", "--config-dir", config_dir])
|
|
clients_path = os.path.join(config_dir, "clients.yaml")
|
|
assert os.path.exists(clients_path)
|
|
with open(clients_path) as f:
|
|
clients = yaml.safe_load(f)
|
|
assert "e2e-client" in clients
|
|
assert clients["e2e-client"]["api_key"].startswith("ak_live_")
|
|
|
|
def test_pair_rejects_duplicate_name(self, cli_runner: CLIRunner, tmp_path):
|
|
config_dir = str(tmp_path / "pair_dup")
|
|
os.makedirs(config_dir, exist_ok=True)
|
|
cli_runner.run(["pair", "--name", "dup-client", "--config-dir", config_dir])
|
|
result = cli_runner.run(["pair", "--name", "dup-client", "--config-dir", config_dir])
|
|
output = (result.stdout + result.stderr).lower()
|
|
assert result.returncode != 0 or "already" in output or "exists" in output
|
|
|
|
def test_pair_list(self, cli_runner: CLIRunner, tmp_path):
|
|
config_dir = str(tmp_path / "pair_list")
|
|
os.makedirs(config_dir, exist_ok=True)
|
|
cli_runner.run(["pair", "--name", "client-a", "--config-dir", config_dir])
|
|
cli_runner.run(["pair", "--name", "client-b", "--config-dir", config_dir])
|
|
result = cli_runner.run(["pair", "--list", "--config-dir", config_dir])
|
|
assert result.returncode == 0
|
|
assert "client-a" in result.stdout
|
|
assert "client-b" in result.stdout
|
|
|
|
def test_pair_revoke(self, cli_runner: CLIRunner, tmp_path):
|
|
import yaml
|
|
|
|
config_dir = str(tmp_path / "pair_revoke")
|
|
os.makedirs(config_dir, exist_ok=True)
|
|
cli_runner.run(["pair", "--name", "revoke-me", "--config-dir", config_dir])
|
|
result = cli_runner.run(["pair", "--revoke", "revoke-me", "--config-dir", config_dir])
|
|
assert result.returncode == 0
|
|
with open(os.path.join(config_dir, "clients.yaml")) as f:
|
|
clients = yaml.safe_load(f)
|
|
assert "revoke-me" not in clients
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# 4. Skill Management (CLI → Server)
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
@pytest.mark.e2e_basic
|
|
class TestCLISkill:
|
|
"""agentkit skill — skill management via CLI against running server."""
|
|
|
|
def test_skill_list_via_server(self, cli_runner_session: CLIRunner):
|
|
result = cli_runner_session.run_server_command(["skill", "list"], E2E_BASE_URL)
|
|
assert result.returncode == 0, f"stderr: {result.stderr}"
|
|
|
|
def test_skill_load_yaml(self, cli_runner: CLIRunner, tmp_path):
|
|
import yaml
|
|
|
|
skill_file = tmp_path / "test_skill.yaml"
|
|
skill_file.write_text(
|
|
yaml.dump(
|
|
{
|
|
"name": "e2e_test_skill",
|
|
"description": "E2E test skill",
|
|
"agent_type": "assistant",
|
|
"task_mode": "llm_generate",
|
|
"prompt": {"system": "You are a test assistant"},
|
|
}
|
|
)
|
|
)
|
|
result = cli_runner.run(["skill", "load", str(skill_file)])
|
|
# Should load successfully or report loaded
|
|
output = (result.stdout + result.stderr).lower()
|
|
assert result.returncode == 0 or "loaded" in output or "e2e_test_skill" in output
|
|
|
|
def test_skill_info_via_server(self, cli_runner_session: CLIRunner, api_client):
|
|
# First register a skill via API
|
|
from tests.e2e.conftest import register_skill_via_api
|
|
|
|
register_skill_via_api(api_client, "cli_info_skill", keywords=["cli_info"])
|
|
# Then query via CLI
|
|
result = cli_runner_session.run_server_command(
|
|
["skill", "info", "cli_info_skill"], E2E_BASE_URL
|
|
)
|
|
assert result.returncode == 0
|
|
assert "cli_info_skill" in result.stdout
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# 5. Task Management (CLI → Server)
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
@pytest.mark.e2e_basic
|
|
class TestCLITask:
|
|
"""agentkit task — task management via CLI against running server."""
|
|
|
|
def test_task_submit_sync(self, cli_runner_session: CLIRunner, api_client):
|
|
from tests.e2e.conftest import register_skill_via_api
|
|
|
|
register_skill_via_api(api_client, "cli_task_skill", keywords=["cli_task"])
|
|
result = cli_runner_session.run_server_command(
|
|
[
|
|
"task",
|
|
"submit",
|
|
"--skill",
|
|
"cli_task_skill",
|
|
"--input",
|
|
json.dumps({"query": "test task submission"}),
|
|
],
|
|
E2E_BASE_URL,
|
|
)
|
|
assert result.returncode == 0, f"stdout: {result.stdout}\nstderr: {result.stderr}"
|
|
|
|
def test_task_submit_async(self, cli_runner_session: CLIRunner, api_client):
|
|
from tests.e2e.conftest import register_skill_via_api
|
|
|
|
register_skill_via_api(api_client, "cli_async_skill", keywords=["cli_async"])
|
|
result = cli_runner_session.run_server_command(
|
|
[
|
|
"task",
|
|
"submit",
|
|
"--skill",
|
|
"cli_async_skill",
|
|
"--mode",
|
|
"async",
|
|
"--input",
|
|
json.dumps({"query": "async task test"}),
|
|
],
|
|
E2E_BASE_URL,
|
|
)
|
|
assert result.returncode == 0
|
|
|
|
def test_task_list(self, cli_runner_session: CLIRunner):
|
|
result = cli_runner_session.run_server_command(["task", "list"], E2E_BASE_URL)
|
|
assert result.returncode == 0
|
|
|
|
def test_task_submit_input_file(self, cli_runner_session: CLIRunner, api_client, tmp_path):
|
|
from tests.e2e.conftest import register_skill_via_api
|
|
|
|
register_skill_via_api(api_client, "cli_file_skill", keywords=["cli_file"])
|
|
|
|
input_file = tmp_path / "task_input.json"
|
|
input_file.write_text(json.dumps({"query": "file input test"}))
|
|
|
|
result = cli_runner_session.run_server_command(
|
|
[
|
|
"task",
|
|
"submit",
|
|
"--skill",
|
|
"cli_file_skill",
|
|
"--input-file",
|
|
str(input_file),
|
|
],
|
|
E2E_BASE_URL,
|
|
)
|
|
assert result.returncode == 0
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# 6. Server Startup
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
@pytest.mark.e2e_basic
|
|
class TestCLIServe:
|
|
"""agentkit serve — server startup (basic check, not full lifecycle)."""
|
|
|
|
def test_serve_help(self, cli_runner: CLIRunner):
|
|
result = cli_runner.run(["serve", "--help"])
|
|
assert result.returncode == 0
|
|
assert "--host" in result.stdout
|
|
assert "--port" in result.stdout
|
|
|
|
def test_serve_invalid_port(self, cli_runner: CLIRunner):
|
|
"""Serve with an invalid port should fail gracefully."""
|
|
result = cli_runner.run(["serve", "--port", "not_a_port"], timeout=5)
|
|
# Should error out, not hang
|
|
assert result.returncode != 0 or "error" in (result.stdout + result.stderr).lower()
|