"""E2E Basic Function Tests — CLI commands. Verifies that all CLI commands execute correctly as a real user would invoke them. Uses subprocess (OpenCLI pattern) to simulate actual CLI operations. Test categories: 1. Utility commands: version, doctor, help 2. Init & config: agentkit init 3. Pair: API key generation 4. Skill management: list, load, info 5. Task management: submit, status, list, cancel 6. Server: serve startup """ import json import os import pytest from tests.e2e.conftest import CLIRunner, E2E_BASE_URL # ═══════════════════════════════════════════════════════════════════════════ # 1. Utility Commands # ═══════════════════════════════════════════════════════════════════════════ @pytest.mark.e2e_basic class TestCLIVersion: """agentkit version — basic sanity check.""" def test_version_returns_zero_exit_code(self, cli_runner: CLIRunner): result = cli_runner.run(["version"]) assert result.returncode == 0, f"stdout: {result.stdout}\nstderr: {result.stderr}" def test_version_outputs_version_string(self, cli_runner: CLIRunner): result = cli_runner.run(["version"]) assert "0.1.0" in result.stdout or "fischer-agentkit" in result.stdout.lower() def test_version_help(self, cli_runner: CLIRunner): result = cli_runner.run(["version", "--help"]) assert result.returncode == 0 @pytest.mark.e2e_basic class TestCLIDoctor: """agentkit doctor — server health check.""" def test_doctor_server_not_running(self, cli_runner: CLIRunner): """Doctor should report error when no server is running.""" result = cli_runner.run(["doctor"]) # Should indicate server not reachable output = (result.stdout + result.stderr).lower() assert ( result.returncode != 0 or "not running" in output or "error" in output or "connection" in output ) def test_doctor_with_running_server(self, cli_runner_session: CLIRunner): """Doctor should report healthy when E2E server is running.""" result = cli_runner_session.run(["doctor", "--port", "18765"]) output = (result.stdout + result.stderr).lower() # Should show some health info (ok, healthy, or at least not connection refused) assert "connection refused" not in output or result.returncode == 0 @pytest.mark.e2e_basic class TestCLIHelp: """agentkit --help — command discovery.""" def test_help_shows_all_subcommands(self, cli_runner: CLIRunner): result = cli_runner.run(["--help"]) assert result.returncode == 0 for cmd in [ "serve", "gui", "chat", "version", "doctor", "init", "task", "skill", "usage", "pair", ]: assert cmd in result.stdout, f"Missing subcommand '{cmd}' in help output" def test_task_help(self, cli_runner: CLIRunner): result = cli_runner.run(["task", "--help"]) assert result.returncode == 0 for sub in ["submit", "status", "list", "cancel"]: assert sub in result.stdout def test_skill_help(self, cli_runner: CLIRunner): result = cli_runner.run(["skill", "--help"]) assert result.returncode == 0 for sub in ["list", "load", "info"]: assert sub in result.stdout # ═══════════════════════════════════════════════════════════════════════════ # 2. Init & Config # ═══════════════════════════════════════════════════════════════════════════ @pytest.mark.e2e_basic class TestCLIInit: """agentkit init — project initialization.""" def test_init_non_interactive(self, cli_runner: CLIRunner, tmp_path): output_dir = str(tmp_path / "init_output") os.makedirs(output_dir, exist_ok=True) result = cli_runner.run(["init", "--non-interactive", "--output-dir", output_dir]) assert result.returncode == 0, f"stderr: {result.stderr}" assert os.path.exists(os.path.join(output_dir, "agentkit.yaml")) assert os.path.exists(os.path.join(output_dir, ".env.example")) def test_init_generates_valid_yaml(self, cli_runner: CLIRunner, tmp_path): import yaml output_dir = str(tmp_path / "init_yaml") os.makedirs(output_dir, exist_ok=True) cli_runner.run(["init", "--non-interactive", "--output-dir", output_dir]) with open(os.path.join(output_dir, "agentkit.yaml")) as f: config = yaml.safe_load(f) assert "server" in config assert "llm" in config def test_init_no_overwrite_without_force(self, cli_runner: CLIRunner, tmp_path): output_dir = str(tmp_path / "init_no_overwrite") os.makedirs(output_dir, exist_ok=True) # Create existing file with open(os.path.join(output_dir, "agentkit.yaml"), "w") as f: f.write("existing_content") cli_runner.run(["init", "--non-interactive", "--output-dir", output_dir]) with open(os.path.join(output_dir, "agentkit.yaml")) as f: content = f.read() # Should not overwrite assert content == "existing_content" def test_init_force_overwrites(self, cli_runner: CLIRunner, tmp_path): output_dir = str(tmp_path / "init_force") os.makedirs(output_dir, exist_ok=True) with open(os.path.join(output_dir, "agentkit.yaml"), "w") as f: f.write("old") result = cli_runner.run( ["init", "--non-interactive", "--force", "--output-dir", output_dir] ) assert result.returncode == 0 with open(os.path.join(output_dir, "agentkit.yaml")) as f: content = f.read() assert "server" in content # ═══════════════════════════════════════════════════════════════════════════ # 3. Pair (API Key Generation) # ═══════════════════════════════════════════════════════════════════════════ @pytest.mark.e2e_basic class TestCLIPair: """agentkit pair — external system API key management.""" def test_pair_generates_api_key(self, cli_runner: CLIRunner, tmp_path): config_dir = str(tmp_path / "pair_config") os.makedirs(config_dir, exist_ok=True) result = cli_runner.run(["pair", "--name", "e2e-test-client", "--config-dir", config_dir]) assert result.returncode == 0, f"stderr: {result.stderr}" assert "ak_live_" in result.stdout def test_pair_saves_client_config(self, cli_runner: CLIRunner, tmp_path): import yaml config_dir = str(tmp_path / "pair_save") os.makedirs(config_dir, exist_ok=True) cli_runner.run(["pair", "--name", "e2e-client", "--config-dir", config_dir]) clients_path = os.path.join(config_dir, "clients.yaml") assert os.path.exists(clients_path) with open(clients_path) as f: clients = yaml.safe_load(f) assert "e2e-client" in clients assert clients["e2e-client"]["api_key"].startswith("ak_live_") def test_pair_rejects_duplicate_name(self, cli_runner: CLIRunner, tmp_path): config_dir = str(tmp_path / "pair_dup") os.makedirs(config_dir, exist_ok=True) cli_runner.run(["pair", "--name", "dup-client", "--config-dir", config_dir]) result = cli_runner.run(["pair", "--name", "dup-client", "--config-dir", config_dir]) output = (result.stdout + result.stderr).lower() assert result.returncode != 0 or "already" in output or "exists" in output def test_pair_list(self, cli_runner: CLIRunner, tmp_path): config_dir = str(tmp_path / "pair_list") os.makedirs(config_dir, exist_ok=True) cli_runner.run(["pair", "--name", "client-a", "--config-dir", config_dir]) cli_runner.run(["pair", "--name", "client-b", "--config-dir", config_dir]) result = cli_runner.run(["pair", "--list", "--config-dir", config_dir]) assert result.returncode == 0 assert "client-a" in result.stdout assert "client-b" in result.stdout def test_pair_revoke(self, cli_runner: CLIRunner, tmp_path): import yaml config_dir = str(tmp_path / "pair_revoke") os.makedirs(config_dir, exist_ok=True) cli_runner.run(["pair", "--name", "revoke-me", "--config-dir", config_dir]) result = cli_runner.run(["pair", "--revoke", "revoke-me", "--config-dir", config_dir]) assert result.returncode == 0 with open(os.path.join(config_dir, "clients.yaml")) as f: clients = yaml.safe_load(f) assert "revoke-me" not in clients # ═══════════════════════════════════════════════════════════════════════════ # 4. Skill Management (CLI → Server) # ═══════════════════════════════════════════════════════════════════════════ @pytest.mark.e2e_basic class TestCLISkill: """agentkit skill — skill management via CLI against running server.""" def test_skill_list_via_server(self, cli_runner_session: CLIRunner): result = cli_runner_session.run_server_command(["skill", "list"], E2E_BASE_URL) assert result.returncode == 0, f"stderr: {result.stderr}" def test_skill_load_yaml(self, cli_runner: CLIRunner, tmp_path): import yaml skill_file = tmp_path / "test_skill.yaml" skill_file.write_text( yaml.dump( { "name": "e2e_test_skill", "description": "E2E test skill", "agent_type": "assistant", "task_mode": "llm_generate", "prompt": {"system": "You are a test assistant"}, } ) ) result = cli_runner.run(["skill", "load", str(skill_file)]) # Should load successfully or report loaded output = (result.stdout + result.stderr).lower() assert result.returncode == 0 or "loaded" in output or "e2e_test_skill" in output def test_skill_info_via_server(self, cli_runner_session: CLIRunner, api_client): # First register a skill via API from tests.e2e.conftest import register_skill_via_api register_skill_via_api(api_client, "cli_info_skill", keywords=["cli_info"]) # Then query via CLI result = cli_runner_session.run_server_command( ["skill", "info", "cli_info_skill"], E2E_BASE_URL ) assert result.returncode == 0 assert "cli_info_skill" in result.stdout # ═══════════════════════════════════════════════════════════════════════════ # 5. Task Management (CLI → Server) # ═══════════════════════════════════════════════════════════════════════════ @pytest.mark.e2e_basic class TestCLITask: """agentkit task — task management via CLI against running server.""" def test_task_submit_sync(self, cli_runner_session: CLIRunner, api_client): from tests.e2e.conftest import register_skill_via_api register_skill_via_api(api_client, "cli_task_skill", keywords=["cli_task"]) result = cli_runner_session.run_server_command( [ "task", "submit", "--skill", "cli_task_skill", "--input", json.dumps({"query": "test task submission"}), ], E2E_BASE_URL, ) assert result.returncode == 0, f"stdout: {result.stdout}\nstderr: {result.stderr}" def test_task_submit_async(self, cli_runner_session: CLIRunner, api_client): from tests.e2e.conftest import register_skill_via_api register_skill_via_api(api_client, "cli_async_skill", keywords=["cli_async"]) result = cli_runner_session.run_server_command( [ "task", "submit", "--skill", "cli_async_skill", "--mode", "async", "--input", json.dumps({"query": "async task test"}), ], E2E_BASE_URL, ) assert result.returncode == 0 def test_task_list(self, cli_runner_session: CLIRunner): result = cli_runner_session.run_server_command(["task", "list"], E2E_BASE_URL) assert result.returncode == 0 def test_task_submit_input_file(self, cli_runner_session: CLIRunner, api_client, tmp_path): from tests.e2e.conftest import register_skill_via_api register_skill_via_api(api_client, "cli_file_skill", keywords=["cli_file"]) input_file = tmp_path / "task_input.json" input_file.write_text(json.dumps({"query": "file input test"})) result = cli_runner_session.run_server_command( [ "task", "submit", "--skill", "cli_file_skill", "--input-file", str(input_file), ], E2E_BASE_URL, ) assert result.returncode == 0 # ═══════════════════════════════════════════════════════════════════════════ # 6. Server Startup # ═══════════════════════════════════════════════════════════════════════════ @pytest.mark.e2e_basic class TestCLIServe: """agentkit serve — server startup (basic check, not full lifecycle).""" def test_serve_help(self, cli_runner: CLIRunner): result = cli_runner.run(["serve", "--help"]) assert result.returncode == 0 assert "--host" in result.stdout assert "--port" in result.stdout def test_serve_invalid_port(self, cli_runner: CLIRunner): """Serve with an invalid port should fail gracefully.""" result = cli_runner.run(["serve", "--port", "not_a_port"], timeout=5) # Should error out, not hang assert result.returncode != 0 or "error" in (result.stdout + result.stderr).lower()