Grant/owui-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial Commit

Browse Source
This commit is contained in:
Grant
2026-09-15 15:24:03 -04:00
commit 4dc7086e7e
605 changed files with 101380 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/deployment/ollama.compose.yml
+242
View File
@@ -0,0 +1,242 @@
# =====================================================================
# Ollama, plus a watchdog that restarts it when the GPU goes away.
#
# This is the model host. Both the assistant and the VM sub-agent talk to
# one Ollama with one model loaded, which is the whole reason the
# sub-agent's session is a single long block rather than many short ones:
# a handover evicts the other model's prefix cache and forces a cold
# re-prefill of its entire context.
#
# WHAT TO CHANGE
# * the port binding, if 172.17.0.1 is not your docker bridge
# * mem_limit, to suit the machine
# * the ntfy block at the bottom, or delete it - alerting is optional
#
# WHAT NOT TO CHANGE
# OLLAMA_NUM_PARALLEL and OLLAMA_MAX_LOADED_MODELS are both 1 on
# purpose. The assistant runs at a 160k context, and the KV cache costs
# roughly 85 KB of VRAM per token on a 27B Q4_K_M model - about 13.4 GB
# on top of 16.5 GB of weights. A second slot of that size does not fit
# in 32 GB, and letting Ollama try means it unloads and reloads the
# model mid-conversation.
#
# WHY IT BINDS TO THE DOCKER BRIDGE AND NOT 0.0.0.0
# Ollama has NO AUTHENTICATION and serves DELETE /api/delete,
# POST /api/pull and POST /api/create. Anything that can reach it can
# delete your model. Nothing in this project talks to it directly: the
# sub-agent reaches it through owuitools-vmgw, an nginx that exposes
# exactly /api/chat and /api/ps and 404s everything else.
# =====================================================================
# Pinned so the project, its volume and its network have the same names
# wherever this file is run from - see the note in open-webui.compose.yml.
name: ollama
services:
ollama:
image: ollama/ollama:latest
container_name: ollama
restart: unless-stopped
ports:
# The docker bridge address, not 0.0.0.0. See the note above.
- "172.17.0.1:11434:11434"
volumes:
- ollama:/root/.ollama
environment:
OLLAMA_MAX_LOADED_MODELS: "1"
OLLAMA_NUM_PARALLEL: "1"
NVIDIA_VISIBLE_DEVICES: "all"
NVIDIA_DRIVER_CAPABILITIES: "compute,utility"
mem_limit: 24g
memswap_limit: 24g
gpus: all
deploy:
resources:
limits:
memory: 24g
reservations:
devices:
- driver: nvidia
count: 1
capabilities: [gpu]
# -------------------------------------------------------------------
# The watchdog. Optional, and worth having.
#
# It exists because of a failure mode that does not look like a
# failure: the container keeps running and answering, and silently
# falls back to CPU, so every reply still arrives and takes twenty
# times as long. Three probes catch it - NVML through nvidia-smi, an
# HTTP GET that catches a hung server, and a grep of the logs for the
# CPU-fallback strings.
#
# Restarts are rate-limited, and alerts are throttled to one a day, so
# a persistent fault does not become a persistent notification.
# -------------------------------------------------------------------
ollama-gpu-watchdog:
container_name: ollama-gpu-watchdog
depends_on:
- ollama
image: docker:cli
restart: unless-stopped
init: true
mem_limit: 128m
volumes:
- /var/run/docker.sock:/var/run/docker.sock
healthcheck:
test: ["CMD-SHELL", "test $$(( $$(date +%s) - $$(stat -c %Y /tmp/heartbeat) )) -lt 300"]
interval: 60s
timeout: 5s
retries: 3
start_period: 30s
entrypoint: ["/bin/sh", "-c"]
command:
- |
set -u
: "$${TARGET:=ollama}"
: "$${INSTANCE:=$$TARGET}"
: "$${CHECK_INTERVAL:=30}"
: "$${RESTART_COOLDOWN:=600}"
: "$${STARTUP_GRACE:=60}"
: "$${GPU_FAIL_LIMIT:=2}"
: "$${API_FAIL_LIMIT:=3}"
: "$${LOG_WINDOW:=60s}"
: "$${PROBE_TIMEOUT:=20}"
: "$${GPU_CHECK_INTERVAL:=600}"
: "$${GPU_RETRY_INTERVAL:=60}"
: "$${GPU_PROBE_OFFSET:=0}"
: "$${API_URL:=http://$$TARGET:11434/api/tags}"
: "$${NOTIFY_COOLDOWN:=86400}"
: "$${NTFY_URL:=}"
: "$${NTFY_TOPIC:=}"
: "$${NTFY_TOKEN:=}"
log() { echo "$$(date -u +%Y-%m-%dT%H:%M:%SZ) [$$INSTANCE] $$*"; }
LAST_RESTART=0
LAST_NOTIFY=0
GPU_FAILS=0
API_FAILS=0
notify() {
[ -n "$$NTFY_URL" ] && [ -n "$$NTFY_TOKEN" ] || return 0
now="$$(date +%s)"
if [ "$$((now - LAST_NOTIFY))" -lt "$$NOTIFY_COOLDOWN" ]; then
log "ntfy suppressed (24h throttle): $$*"
return 0
fi
if wget -q -T 10 -O /dev/null \
--header "Authorization: Bearer $$NTFY_TOKEN" \
--post-data "$$(date -u +%Y-%m-%dT%H:%M:%SZ) $$*" \
"$$NTFY_URL/$$NTFY_TOPIC?title=Ollama+watchdog:+$$INSTANCE&priority=high&tags=rotating_light"; then
LAST_NOTIFY="$$(date +%s)"
log "ntfy alert sent: $$*"
else
log "ERROR: ntfy publish failed (will retry on next event)"
fi
}
restart_target() {
reason="$$1"
notify "$$reason - watchdog intervening on $$INSTANCE"
now="$$(date +%s)"
if [ "$$((now - LAST_RESTART))" -lt "$$RESTART_COOLDOWN" ]; then
log "restart wanted ($$reason) but in cooldown"
return 0
fi
log "RESTARTING $$TARGET: $$reason"
if timeout 180 docker restart "$$TARGET" >/dev/null 2>&1; then
LAST_RESTART="$$(date +%s)"
GPU_FAILS=0
API_FAILS=0
log "restart issued OK; grace sleep $${STARTUP_GRACE}s"
sleep "$$STARTUP_GRACE"
else
log "ERROR: docker restart $$TARGET failed/timed out"
notify "docker restart of $$INSTANCE FAILED after: $$reason"
fi
}
log "watchdog up"
NEXT_GPU_PROBE="$$(( $$(date +%s) + GPU_PROBE_OFFSET ))"
while true; do
touch /tmp/heartbeat
state="$$(timeout 10 docker inspect -f '{{.State.Status}}' "$$TARGET" 2>/dev/null || echo unknown)"
if [ "$$state" != "running" ]; then
log "target state=$$state - waiting (docker restart policy owns crash recovery)"
sleep "$$CHECK_INTERVAL"
continue
fi
# 1. GPU / NVML, only every GPU_CHECK_INTERVAL: without a
# persistence client every nvidia-smi is a full driver
# init/teardown. After a failure, re-probe sooner.
now="$$(date +%s)"
if [ "$$now" -ge "$$NEXT_GPU_PROBE" ]; then
if timeout "$$PROBE_TIMEOUT" docker exec "$$TARGET" nvidia-smi >/dev/null 2>&1; then
[ "$$GPU_FAILS" -gt 0 ] && log "GPU probe recovered"
GPU_FAILS=0
NEXT_GPU_PROBE="$$((now + GPU_CHECK_INTERVAL))"
else
GPU_FAILS="$$((GPU_FAILS + 1))"
log "GPU probe FAILED ($$GPU_FAILS/$$GPU_FAIL_LIMIT)"
NEXT_GPU_PROBE="$$((now + GPU_RETRY_INTERVAL))"
if [ "$$GPU_FAILS" -ge "$$GPU_FAIL_LIMIT" ]; then
restart_target "container lost GPU (NVML dead)"
NEXT_GPU_PROBE="$$(( $$(date +%s) + GPU_RETRY_INTERVAL ))"
fi
fi
fi
# 2. the API itself - catches a hung server with no docker exec
if timeout "$$PROBE_TIMEOUT" wget -q -T "$$PROBE_TIMEOUT" -O /dev/null "$$API_URL"; then
[ "$$API_FAILS" -gt 0 ] && log "API probe recovered"
API_FAILS=0
else
API_FAILS="$$((API_FAILS + 1))"
log "API probe FAILED ($$API_FAILS/$$API_FAIL_LIMIT)"
if [ "$$API_FAILS" -ge "$$API_FAIL_LIMIT" ]; then
restart_target "API unresponsive"
fi
fi
# 3. the silent one: still answering, but on the CPU
if timeout 30 docker logs --since "$$LOG_WINDOW" "$$TARGET" 2>&1 | grep -qE \
'msg="inference compute".*id=cpu|library=cpu|total_vram="0 B"|no compatible GPUs|failed to discover.*GPU|CUDA.*error|CUDA.*failure|unable to load CUDA'; then
restart_target "GPU fallback / CUDA error in logs"
fi
sleep "$$CHECK_INTERVAL"
done
environment:
TARGET: "ollama"
INSTANCE: "ollama"
CHECK_INTERVAL: "30"
RESTART_COOLDOWN: "600"
STARTUP_GRACE: "60"
GPU_FAIL_LIMIT: "2"
API_FAIL_LIMIT: "3"
LOG_WINDOW: "60s"
PROBE_TIMEOUT: "20"
GPU_CHECK_INTERVAL: "600"
GPU_RETRY_INTERVAL: "60"
GPU_PROBE_OFFSET: "0"
NOTIFY_COOLDOWN: "86400"
# Alerting is optional: leave these empty and the watchdog still
# restarts, it just does not tell anyone. Fill them in to get a
# push when it intervenes. Use a write-only token.
NTFY_URL: ""
NTFY_TOPIC: ""
NTFY_TOKEN: ""
volumes:
ollama: