Initial Commit

tests/test_tool_toggle.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""The tool switch: configs/tools.toml and how main.py applies it.
+
+    python3 tests/test_tool_toggle.py
+
+Pure Python, no container and no network. Two different things are being
+checked and only one of them is about the loader.
+
+The first is loader semantics - what a missing file, a typo, or an
+unmentioned tool does. Those are all "fail open" by design, because this
+is config that REMOVES capability and losing it should give you the
+system it replaced rather than a server with no tools.
+
+The second is that the file and the code agree. A tool added to a router
+without a line here would be on by default, which is correct, but it
+would also be invisible in the one file that is supposed to answer "what
+does this deployment offer" - so the agreement is asserted rather than
+hoped for.
+
+The route-level behaviour - that a disabled tool leaves the OpenAPI spec
+AND stops answering - needs fastapi and redis, so it lives in
+tests/test_tool_toggle.sh, which runs it inside the api image.
+"""
+
+import os
+import re
+import sys
+import tempfile
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "app"))
+
+from owuitools import toolconfig  # noqa: E402
+
+ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
+CONFIG = os.path.join(ROOT, "configs", "tools.toml")
+ROUTERS = os.path.join(ROOT, "app", "owuitools", "routers")
+
+FAILS: list[str] = []
+
+
+def check(label: str, condition: bool, detail: str = "") -> None:
+    if condition:
+        print(f"ok    {label}")
+    else:
+        FAILS.append(f"{label}: {detail}")
+        print(f"FAIL  {label}\n      {detail}")
+
+
+def implemented() -> set[str]:
+    """Every operation_id the routers define, read from the source.
+
+    Deliberately a regex over the files rather than an import: the app
+    needs fastapi and redis, and this test is meant to run anywhere.
+    """
+    found = set()
+    for name in sorted(os.listdir(ROUTERS)):
+        if not name.endswith(".py"):
+            continue
+        with open(os.path.join(ROUTERS, name), encoding="utf-8") as handle:
+            found |= set(re.findall(r'operation_id="([a-z0-9_]+)"', handle.read()))
+    return found
+
+
+# --- the file itself --------------------------------------------------
+config = toolconfig.load(CONFIG)
+tools = implemented()
+
+check("configs/tools.toml parses and is not empty", bool(config), repr(config)[:120])
+check("every tool the routers define has a line in the config",
+      tools <= set(config), f"missing: {sorted(tools - set(config))}")
+check("the config names no tool that does not exist",
+      set(config) <= tools, f"unknown: {sorted(set(config) - tools)}")
+check("every tool ships enabled by default",
+      all(config.values()), f"off: {sorted(k for k, v in config.items() if not v)}")
+
+# Every tool that needs setup beyond `docker compose up` must say where
+# to read about it. A switch you cannot act on is worse than no switch.
+with open(CONFIG, encoding="utf-8") as handle:
+    body = handle.read()
+for needs_setup in ("web_search", "wikipedia_lookup", "search_stackexchange",
+                    "lookup_cve", "lookup_package", "matlab_run", "vm_agent",
+                    "find_datasheet", "search_papers"):
+    block = body.rsplit(f"\n{needs_setup} = ", 1)[0]
+    comment = block.rsplit("\n\n", 1)[-1]
+    check(f"{needs_setup} points at a setup doc",
+          "docs/setup/" in comment or "Setup:" in comment,
+          f"no docs/setup pointer in the comment above {needs_setup}")
+
+# --- loader semantics -------------------------------------------------
+check("a missing config file leaves every tool on",
+      toolconfig.load("/nonexistent/tools.toml") == {}, "should be empty, i.e. no opinion")
+check("a missing config disables nothing",
+      toolconfig.disabled(tools, "/nonexistent/tools.toml") == set(), "should disable nothing")
+
+with tempfile.NamedTemporaryFile("w", suffix=".toml", delete=False) as handle:
+    handle.write("[tools]\nweb_search = false\nnot_a_tool = false\nfetch_page = true\n")
+    partial = handle.name
+off = toolconfig.disabled(tools, partial)
+check("a tool set to false is reported disabled", off == {"web_search"}, f"got {off}")
+check("a name the server does not implement is ignored, not disabled",
+      "not_a_tool" not in off, f"got {off}")
+check("a tool the config does not mention stays on",
+      "subnet_calc" not in off, f"got {off}")
+
+with tempfile.NamedTemporaryFile("w", suffix=".toml", delete=False) as handle:
+    handle.write("[tools]\nthis is not valid toml = = =\n")
+    broken = handle.name
+check("an unparseable config leaves every tool on",
+      toolconfig.disabled(tools, broken) == set(), "a broken file must not remove tools")
+
+os.unlink(partial)
+os.unlink(broken)
+
+# --- the seam -------------------------------------------------------
+# FastAPI 0.141 stopped flattening included routers: include_router()
+# appends one opaque _IncludedRouter to app.routes, so filtering
+# app.routes removes nothing while looking exactly like it worked.
+# Measured - the first version of this feature shipped that bug.
+with open(os.path.join(ROOT, "app", "owuitools", "main.py"), encoding="utf-8") as handle:
+    main_src = handle.read()
+check("the filter runs on the routers, before include_router",
+      "for router in _routers for route in router.routes" in main_src,
+      "main.py must filter router.routes, not app.routes")
+check("main.py does not filter app.routes",
+      "app.routes[:]" not in main_src,
+      "app.routes holds no APIRoute on fastapi >= 0.141; that filter is a no-op")
+
+print()
+if FAILS:
+    print(f"{len(FAILS)} FAILED")
+    for line in FAILS:
+        print(f"  - {line}")
+    raise SystemExit(1)
+print("all tool-toggle checks passed")