Make Pipelines Docker-only with OpenCode and Pi

Install current verified harness releases inside Docker, preserve persistent accounts and settings, and provide separate interactive and pipeline model defaults with terminal onboarding and fuzzy pickers.

Add Docker-aware authoring transport, Pi stage execution, bounded workflow controls, reliable final-result readback and usage settlement, and finite short-lived CLI pipe draining. Keep dated installation evidence and packaged authoring guidance consistent.

Verified with a real Docker installation, both harness stages and interactive authors, the built-in plan/implement/review starter, independent host race/vet checks, installer tests, and packaged skill tests. Generated release receipts stay ignored in clean checkouts.
This commit is contained in:
2026-10-11 23:00:25 +01:00
parent c42a19f7d3
commit 05ae530442
131 changed files with 9297 additions and 355 deletions
+328 -17
View File
@@ -23,13 +23,44 @@ def module(name, file):
configure = module("configure_compose", "configure-compose.py")
tokens = module("save_ntfy_token", "save-ntfy-token.py")
lan = module("lan_address", "lan-address.py")
build_metadata = module("build_metadata", "build_metadata.py")
release_lookup = module("resolve_releases", "resolve-releases.py")
# Exact executable version formats reported by the built image. Anything with a
# banner, suffix or extra token is rejected instead of being partially parsed.
# OpenCode's `--version` prints its executable name before the version token
# ("opencode v2.0.26", verified probe output), so accept exactly that optional
# lowercase prefix in addition to the bare/v-prefixed token; unrelated banners
# such as "OpenCode v2.0.26" or trailing text are still rejected. The patch
# component accepts arbitrary digits (valid/future versions could report e.g.
# 260 or 100; these are illustrative examples, not verified actual releases);
# verify_harness_versions still rejects a parsed 260 when the expected value is
# 26 by exact string equality.
HARNESS_VERSION_FORMATS = {
"opencode": re.compile(r"(?:opencode )?v?(\d+\.\d+\.\d+)"),
"pi": re.compile(r"(\d+\.\d+\.\d+)"),
"node": re.compile(r"v?(\d+\.\d+\.\d+)"),
}
def parse_harness_version(name, value):
"""Accept only the exact known version format for a harness executable."""
pattern = HARNESS_VERSION_FORMATS.get(name)
if pattern is None:
raise ValueError(f"Unknown harness executable {name!r}; refusing to parse an unchecked version")
match = pattern.fullmatch((value or "").strip())
if not match:
raise ValueError(f"Unexpected {name} version output {value!r}; refusing to continue on an unverified image")
return match.group(1)
class Setup:
def __init__(self, directory, runner=subprocess.run, prompt=input):
def __init__(self, directory, runner=subprocess.run, prompt=input, resolver=None):
self.directory = Path(directory)
self.runner = runner
self.prompt = prompt
self.resolver = resolver or release_lookup.resolve
self.accounts = []
def compose(self, *args, capture=False, check=True, **kwargs):
@@ -132,7 +163,30 @@ done
'''
result = self.compose("run", "--rm", "--no-deps", "-T", "--entrypoint", "sh", "pipelines", "-c", discovery, capture=True)
return [line.strip() for line in result.stdout.splitlines()
if re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_./:-]+", line.strip())]
if re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_./:-]+", line.strip())]
def choose_model(self, models, title, purpose="", step="", current=""):
# Keep terminal stdin free for the fuzzy picker; catalog/result use private
# files. The picker always runs inside the configured Docker installation so
# it never executes a host harness, even when host Go is available.
with tempfile.TemporaryDirectory(prefix=".pipelines-picker-", dir=self.directory) as temporary:
root = Path(temporary)
catalog, result = root/"models.json", root/"selected.txt"
catalog.write_text(json.dumps(models))
catalog.chmod(0o600)
options = ["--title", title]
if purpose:
options += ["--purpose", purpose]
if step:
options += ["--step", step]
if current:
options += ["--current", current]
self.compose("run", "--rm", "--no-deps", "--entrypoint", "pipelines", "pipelines",
"choose-model", "--options-file", str(catalog), "--output", str(result), *options)
selected = result.read_text().strip()
if selected not in models:
raise ValueError("Model selection did not produce an available model; defaults unchanged")
return selected
def validate_provider(self, args):
models = self.discover_models()
@@ -140,18 +194,31 @@ done
available = [model for model in models if model.split("/",1)[0] in ids]
saved = dict(line.split("=",1) for line in (self.directory/".env").read_text().splitlines() if "=" in line)
model = args.model or saved.get("PIPELINES_DEFAULT_MODEL", "")
harness_model = args.harness_model or saved.get("PIPELINES_HARNESS_MODEL", "")
# Fresh setup selects the interactive model first and the new-pipeline
# model second, matching `pipelines harness setup` (and reconfiguration).
if (not model or not harness_model) and not available:
connected = ", ".join(sorted(str(id) for id in ids)) or "none"
visible = ", ".join(sorted({model.split('/', 1)[0] for model in models})) or "none"
raise ValueError(f"OpenCode catalog discovery found no models for connected providers [{connected}] (visible model providers: [{visible}]). Your saved key was not replaced; this is NOT proof it is invalid. Rerun ./install; if it persists, share this message and the OpenCode version, not your key. No model was validated.")
if not harness_model:
harness_model = self.choose_model(
available,
"Choose the interactive OpenCode harness model",
purpose="Runs your interactive OpenCode authoring and debugging sessions. It does not change new-pipeline-stage models.",
step="Step 1 of 2 · interactive model",
current=saved.get("PIPELINES_HARNESS_MODEL", ""))
if not model:
if not available:
connected = ", ".join(sorted(str(id) for id in ids)) or "none"
visible = ", ".join(sorted({model.split('/', 1)[0] for model in models})) or "none"
raise ValueError(f"OpenCode catalog discovery found no models for connected providers [{connected}] (visible model providers: [{visible}]). Your saved key was not replaced; this is NOT proof it is invalid. Rerun ./install.sh; if it persists, share this message and the OpenCode version, not your key. No model was validated.")
print("Choose the model to validate (one small tool-free test; provider charges may apply):")
for i, item in enumerate(available[:25],1):
print(f" {i}. {item}")
choice = self.prompt("Model number or full provider/model: ").strip()
model = available[int(choice)-1] if choice.isdigit() and 1 <= int(choice) <= len(available) else choice
model = self.choose_model(
available,
"Choose the default model for new pipelines",
purpose="Runs model stages in new pipelines and starters. Stored separately from the interactive harness model.",
step="Step 2 of 2 · new-pipeline-stage model",
current=saved.get("PIPELINES_DEFAULT_MODEL", ""))
if model not in models:
raise ValueError("Selected model is not available in this OpenCode configuration. Choose a listed model with --model PROVIDER/MODEL.")
if harness_model not in models:
raise ValueError("Selected interactive harness model is unavailable; defaults unchanged")
print(f"Validating {model} with one tool-free generation; the session is retained for debugging.")
validation = '''set -eu
opencode api get /api/integration >/dev/null
@@ -174,13 +241,25 @@ exec pipelines provider-check --model "$1" --yes --json --timeout 90s
if not any(line.startswith(key) for line in lines):
updated = updated.rstrip("\n")+"\n"+key+model+"\n"
path.write_text(updated)
lines = path.read_text().splitlines(keepends=True)
key = "PIPELINES_HARNESS_MODEL="
updated = "".join(key+harness_model+"\n" if line.startswith(key) else line for line in lines)
if not any(line.startswith(key) for line in lines):
updated = updated.rstrip("\n")+"\n"+key+harness_model+"\n"
path.write_text(updated)
setup_options = ["--model", harness_model, "--pipeline-model", model]
versions = getattr(self, "harness_version_label", "")
if versions:
setup_options += ["--versions", versions]
self.compose("run", "--rm", "--no-deps", "-T", "--entrypoint", "pipelines", "pipelines", "harness", "setup", "opencode", *setup_options)
self.selected_harness_model, self.selected_pipeline_model = harness_model, model
print(f"Provider/model validated: {model} · session {report.get('session_id','unknown')}")
def update_host_client(self):
if not shutil.which("go"):
print("Host Go is unavailable: Docker binaries are updated, but an existing host CLI/TUI is NOT updated. Install/update that client before expecting new UI behavior.")
return
for command in (["go","install","-trimpath","./cmd/..."], ["go","run","./cmd/pipelines","skill","install"]):
for command in (["go","install","-trimpath","./cmd/pipelines","./cmd/pipelines-tui"], ["go","run","./cmd/pipelines","skill","install"]):
result = self.runner(command, cwd=self.directory)
if result.returncode:
raise ValueError("Host client update failed; Docker setup is incomplete")
@@ -225,6 +304,208 @@ exec pipelines provider-check --model "$1" --yes --json --timeout 90s
"pipelines-workspace-check", directory, capture=True)
print("Verified: Docker server can enter the host installation repository at " + directory)
def resolve_releases(self):
fixture = os.environ.get("PIPELINES_RELEASE_FIXTURE")
try:
if fixture:
return release_lookup.load_fixture(fixture)
return self.resolver()
except release_lookup.ReleaseError as error:
raise ValueError(
f"Could not resolve current harness releases ({error}); keeping the last verified Dockerfile pins. "
"Rerun when the registry/npm is reachable; refusing to silently install stale copies.") from None
def reconfigure_harness(self, name, options):
# Reuse the setup resolution/verification path without rebuilding or
# restarting services: a stale image needs an explicit ./install upgrade.
releases = self.resolve_releases()
build_identity = self.build_identity()
if self.needs_image_build(releases, build_identity):
raise ValueError(
f"The Docker image does not match the current resolved releases "
f"(OpenCode {releases['opencode']['version']}, Pi {releases['pi']['version']}); "
"run ./install to rebuild it, then rerun this reconfiguration")
found = self.verify_harness_versions(releases) or {}
label = self.harness_versions_label({
"opencode": found.get("opencode", releases["opencode"]["version"]),
"pi": found.get("pi", releases["pi"]["version"]),
"node": found.get("node", ""),
})
self.compose("exec", "pipelines", "pipelines", "harness", "setup", name, "--login", "--versions", label, *options)
def link_host(self, name=None, config=None):
# Shared by fresh setup and --opencode/--pi reconfiguration. The host
# launcher is linked from the container profile; it is never executed
# from a host harness runtime.
if config is None:
config = json.loads(self.compose(
"exec", "-T", "pipelines", "pipelines", "harness", "list", "--json", capture=True).stdout)
name = name or config["default"]
profile = config["profiles"][name]
if shutil.which("go"):
executable = ["go", "run", "./cmd/pipelines"]
elif shutil.which("pipelines"):
executable = [shutil.which("pipelines")]
else:
print("Host harness not linked: rerun ./install with host Go available to build the Docker client.")
return
args = [*executable, "harness", "link", name, "--docker-directory", str(self.directory.resolve()),
"--model", profile["model"], "--pipeline-model", profile["pipeline_model"]]
if config.get("default") == name:
args.append("--default")
result = self.runner(args, cwd=self.directory)
if result.returncode:
raise ValueError("Host harness link failed; container configuration retained")
def release_receipt_path(self):
return self.directory / ".pipelines-releases.json"
def read_release_receipt(self):
path = self.release_receipt_path()
if path.is_symlink():
raise ValueError("Refusing to read a symlinked release receipt")
if not path.exists():
return {}
try:
data = json.loads(path.read_text())
except ValueError:
return {}
return data if isinstance(data, dict) else {}
def record_release_receipt(self, releases, build_identity=None):
path = self.release_receipt_path()
if path.is_symlink():
raise ValueError("Refusing to replace a symlinked release receipt")
data = dict(releases)
if build_identity:
data["build"] = dict(build_identity)
encoded = json.dumps(data, indent=2, sort_keys=True) + "\n"
handle, temporary = tempfile.mkstemp(dir=str(self.directory), prefix=".pipelines-releases-")
try:
os.chmod(temporary, 0o600)
with os.fdopen(handle, "w") as file:
file.write(encoded)
os.replace(temporary, path)
finally:
if os.path.exists(temporary):
os.unlink(temporary)
def release_build_args(self, releases):
return build_metadata.release_args(releases)
def build_identity(self):
args = build_metadata.docker_args(self.directory, self.runner)
return dict(arg.split("=", 1) for arg in args if arg != "--build-arg")
def image_identity(self, image_id):
# Read the actual built image's labels rather than trusting the receipt
# file, so an unrelated/stale image cannot satisfy the build decision.
result = self.runner(["docker", "image", "inspect", image_id, "--format", "{{json .Config.Labels}}"],
cwd=self.directory, capture_output=True, text=True)
if result.returncode or not result.stdout.strip():
return {}
try:
labels = json.loads(result.stdout)
except ValueError:
return {}
return labels if isinstance(labels, dict) else {}
def expected_labels(self, releases, build_identity=None):
labels = {
"net.a-wan.pipelines.opencode-version": releases["opencode"]["version"],
"net.a-wan.pipelines.opencode-digest": releases["opencode"]["digest"],
"net.a-wan.pipelines.pi-version": releases["pi"]["version"],
}
if build_identity:
labels["net.a-wan.pipelines.source-version"] = build_identity.get("PIPELINES_VERSION", "")
labels["net.a-wan.pipelines.source-commit"] = build_identity.get("PIPELINES_COMMIT", "")
labels["net.a-wan.pipelines.source-dirty"] = build_identity.get("PIPELINES_DIRTY", "")
return labels
def receipt_matches(self, receipt, releases, build_identity=None):
if not isinstance(receipt, dict):
return False
opencode, pi = receipt.get("opencode"), receipt.get("pi")
if not isinstance(opencode, dict) or not isinstance(pi, dict):
# A corrupted receipt must fail closed and rebuild, not raise.
return False
if (opencode.get("version") != releases["opencode"]["version"]
or opencode.get("digest") != releases["opencode"]["digest"]
or pi.get("version") != releases["pi"]["version"]):
return False
return not build_identity or receipt.get("build") == build_identity
def needs_image_build(self, releases, build_identity=None, force_dirty=False):
# A dirty tree has no content identity: the same commit/version/dirty
# strings can describe different bytes. During install, always invoke the
# Docker build for a dirty source and let Docker layer caching decide. The
# reconfiguration path (force_dirty=False) deliberately checks only the
# verified release image so a model/login-only change is not a rebuild:
# source identity is compared only for a full install.
if force_dirty and build_identity and build_identity.get("PIPELINES_DIRTY") == "true":
return True
identity = build_identity if force_dirty else None
images = self.compose("images", "-q", "pipelines", capture=True, check=False).stdout.strip().splitlines()
if not images:
return True
if not self.receipt_matches(self.read_release_receipt(), releases, identity):
return True
labels = self.image_identity(images[0].strip())
return not all(labels.get(key) == value
for key, value in self.expected_labels(releases, identity).items())
def verify_harness_versions(self, releases):
# One bounded in-image check per harness; fail closed on stale/missing.
probe = '''set -eu
printf 'opencode=%s\\n' "$(opencode --version)"
printf 'pi=%s\\n' "$(pi --version)"
printf 'node=%s\\n' "$(node --version)"
'''
result = self.compose("run", "--rm", "--no-deps", "-T", "--entrypoint", "sh", "pipelines", "-c", probe, capture=True)
found = {}
for line in result.stdout.splitlines():
name, separator, value = line.strip().partition("=")
if separator and name in HARNESS_VERSION_FORMATS:
found[name] = parse_harness_version(name, value)
expected_opencode = releases["opencode"]["version"]
expected_opencode = expected_opencode[1:] if expected_opencode.startswith("v") else expected_opencode
expected_pi = releases["pi"]["version"]
expected_pi = expected_pi[1:] if expected_pi.startswith("v") else expected_pi
if found.get("opencode") != expected_opencode or found.get("pi") != expected_pi:
raise ValueError(
f"Built image does not report the resolved harness versions (expected OpenCode {expected_opencode}, "
f"Pi {expected_pi}); refusing to continue on a stale or mismatched image")
engine = releases["pi"].get("engine", "")
if engine:
if "node" not in found:
raise ValueError(f"Could not verify Node compatibility for Pi ({engine}); refusing to continue")
if not release_lookup.node_satisfies(engine, found["node"]):
raise ValueError(f"Built image Node {found['node']} does not satisfy Pi's required engine {engine}")
print(f"Verified image harnesses: OpenCode {expected_opencode}, Pi {expected_pi}"
+ (f", Node {found['node']} ({engine})" if engine else ""))
return found
def harness_versions_label(self, found):
# Rendered by the Go onboarding wizard; no extra probe or subprocess.
parts = [f"OpenCode {found.get('opencode', '?')}", f"Pi {found.get('pi', '?')}"]
if found.get("node"):
parts.append(f"Node {found['node']}")
return " · ".join(parts)
def verify_build(self, expected):
result = self.compose("exec", "-T", "pipelines", "pipelines", "--url",
"ws://127.0.0.1:8080/connect", "ping", "--json", capture=True)
try:
pong = json.loads(result.stdout)
build = pong["build"]
if (pong["type"] != "pong" or build["commit"] != expected["PIPELINES_COMMIT"]
or build["version"] != expected["PIPELINES_VERSION"]
or build["dirty"] is not (expected["PIPELINES_DIRTY"] == "true")):
raise ValueError("build mismatch")
except (ValueError, KeyError, TypeError):
raise ValueError("Running server does not match the built source; setup is NOT complete. Check docker compose logs pipelines") from None
print("Verified running server build: " + build["version"] + " (" + build["commit"] + ")")
def verify(self):
# Read the token inside the container; never put it in argv or output.
probe = '''set -eu
@@ -251,7 +532,7 @@ printf 'header = "Authorization: Bearer %s"\\n' "$(cat "$PIPELINES_NTFY_TOKEN_FI
self.verify_host_endpoint(endpoint)
self.configure_client(endpoint)
print(f"Host CLI/TUI connection: pipelines --url {endpoint} tui")
print("Install host client binaries with ./install.sh --native if needed.")
print("Run ./install with host Go available to install/update the Docker CLI/TUI clients.")
def verify_host_endpoint(self, endpoint):
if shutil.which("go"):
@@ -301,6 +582,12 @@ printf 'header = "Authorization: Bearer %s"\\n' "$(cat "$PIPELINES_NTFY_TOKEN_FI
def run(self, args):
self.compose("version", capture=True)
# Refuse an unidentifiable source archive before changing settings/services.
build_args = build_metadata.docker_args(self.directory, self.runner)
expected_build = dict(arg.split("=", 1) for arg in build_args if arg != "--build-arg")
# Resolve current releases before touching settings/services; a failure
# keeps the verified pins and stops rather than installing stale copies.
releases = self.resolve_releases()
self.settings(args)
self.publish_address()
self.configure_workspaces()
@@ -309,16 +596,38 @@ printf 'header = "Authorization: Bearer %s"\\n' "$(cat "$PIPELINES_NTFY_TOKEN_FI
self.compose("config", "--quiet", capture=True)
self.compose("up", "-d", "--wait", "--wait-timeout", "60", "ntfy")
self.bootstrap_ntfy()
self.compose("build", "pipelines")
if self.needs_image_build(releases, expected_build, force_dirty=True):
self.compose("build", *build_args, *self.release_build_args(releases), "pipelines")
found = self.verify_harness_versions(releases) or {}
self.record_release_receipt(releases, expected_build)
else:
print("Keeping the existing image: resolved harness versions, digest and source identity are unchanged.")
found = {}
self.harness_version_label = self.harness_versions_label({
"opencode": found.get("opencode", releases["opencode"]["version"]),
"pi": found.get("pi", releases["pi"]["version"]),
"node": found.get("node", ""),
})
self.compose("stop", "--timeout", "60", "pipelines")
self.prepare_volume_ownership()
self.compose("run", "--rm", "--no-deps", "-T", "--entrypoint", "pipelines", "pipelines", "skill", "install")
self.authenticate(args.relogin)
self.validate_provider(args)
if args.harness == "opencode":
self.authenticate(args.relogin)
self.validate_provider(args)
else:
options = ["--default", "--login", "--versions", self.harness_version_label]
if args.model:
options += ["--pipeline-model", args.model]
if args.harness_model:
options += ["--model", args.harness_model]
self.compose("run", "--rm", "--no-deps", "--entrypoint", "pipelines", "pipelines", "harness", "setup", "pi", *options)
# Recreate to reload file secrets and environment, without deleting volumes.
self.compose("up", "-d", "--force-recreate", "--wait", "--wait-timeout", "60", "pipelines")
self.verify_build(expected_build)
self.verify_workspace()
self.verify()
if shutil.which("go") or shutil.which("pipelines"):
self.link_host()
print("Docker setup complete. Use docker compose ps / logs to manage it; no OS user service was registered.")
@@ -330,6 +639,8 @@ def arguments(argv=None):
parser.add_argument("--port", type=int)
parser.add_argument("--relogin", action="store_true", help="open OpenCode provider login again without deleting accounts")
parser.add_argument("--model", help="Provider/model for actual generation validation (saved after success)")
parser.add_argument("--harness-model", help="Separate interactive OpenCode model default (catalog validation only)")
parser.add_argument("--harness", choices=["opencode", "pi"], default="opencode", help="Harness for fresh Docker setup; Pi skips OpenCode login/inference validation")
return parser.parse_args(argv)
@@ -339,5 +650,5 @@ if __name__ == "__main__":
except KeyboardInterrupt:
print("\nSetup canceled; existing configuration, credentials and Docker volumes were retained.", file=sys.stderr)
sys.exit(130)
except (ValueError, OSError, EOFError, json.JSONDecodeError, configure.ZoneInfoNotFoundError) as error:
except (ValueError, OSError, EOFError, json.JSONDecodeError, subprocess.CalledProcessError, configure.ZoneInfoNotFoundError) as error:
sys.exit(f"Docker setup incomplete: {error}")