From 2250c19c34948243a235543c43df798120f2c84a Mon Sep 17 00:00:00 2001 From: Peng-YM Date: Thu, 6 Aug 2026 22:09:13 +0800 Subject: [PATCH 01/12] feat(sdk): add strict Dockerfile sandbox launch Add a backend-neutral direct-launch path for a deliberately strict Dockerfile subset. The FROM image supplies only the sandbox root filesystem; explicitly declared RUN, COPY, ADD, ENV, WORKDIR, USER, CMD, and ENTRYPOINT behavior is applied through the public sandbox facades without BuildKit, a Docker daemon, or a registry push. Validate Dockerfiles and build contexts before remote side effects. Unsupported syntax fails closed, remote ADD URLs are rejected, and a filtered manifest applies .dockerignore consistently to local and custom contexts. Local paths use no-follow directory-relative opens, all COPY and ADD inputs are materialized before sandbox operations, tar members are restricted to safe regular files and directories, and ownership changes are limited to outputs of the current instruction. Expose the background CMD or ENTRYPOINT CommandHandle while defining constructor success as sandbox readiness and successful dispatch rather than application health. Integrate cleanup with the backend-neutral BackendSession lifecycle and document the rootfs-only and no-snapshot contract. Signed-off-by: Peng-YM --- AGENTS.md | 7 + sdk/python/README.md | 87 ++ sdk/python/akernel_sdk/__init__.py | 22 + sdk/python/akernel_sdk/_dockercontext.py | 538 ++++++++ sdk/python/akernel_sdk/_dockerfile.py | 838 ++++++++++++ sdk/python/akernel_sdk/_dockerfile_runner.py | 785 +++++++++++ sdk/python/akernel_sdk/sandbox.py | 62 +- sdk/python/examples/dockerfile_launch.py | 186 +++ sdk/python/pyproject.toml | 2 + sdk/python/tests/unit/test_dockercontext.py | 321 +++++ sdk/python/tests/unit/test_dockerfile.py | 1283 ++++++++++++++++++ sdk/python/tests/unit/test_sandbox.py | 125 ++ sdk/python/tests/unit/test_types.py | 11 + 13 files changed, 4264 insertions(+), 3 deletions(-) create mode 100644 sdk/python/akernel_sdk/_dockercontext.py create mode 100644 sdk/python/akernel_sdk/_dockerfile.py create mode 100644 sdk/python/akernel_sdk/_dockerfile_runner.py create mode 100644 sdk/python/examples/dockerfile_launch.py create mode 100644 sdk/python/tests/unit/test_dockercontext.py create mode 100644 sdk/python/tests/unit/test_dockerfile.py diff --git a/AGENTS.md b/AGENTS.md index 8d240ea..1d80518 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -347,6 +347,13 @@ Keep public `Sandbox`, `Commands`, `Filesystem`, and value types independent of both native packages; all native conversions belong under `akernel_sdk._backends`. +The SDK also supports a strict Dockerfile direct-launch subset through +`DockerContext` and `Sandbox(context=...)`. `FROM` supplies only the root +filesystem; inherited OCI image configuration is not applied. Context transfer +must remain backend-neutral, honor `.dockerignore`, and fail closed for syntax +outside the documented subset. Keep the contract, security boundaries, public +types, unit tests, SDK README, and `examples/dockerfile_launch.py` in sync. + When changing a public SDK method, update its type annotations and docstring, add or update unit coverage, and keep the SDK README and maintained examples in sync. Benchmark programs under `sdk/python/benchmarks/` are manual tools and diff --git a/sdk/python/README.md b/sdk/python/README.md index 56b981d..bb5c821 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -101,9 +101,95 @@ Sandbox( xpu: str | None = None, storage_mb: int | None = None, network_policy: NetworkPolicy | None = None, + context: DockerContext | None = None, + auto_start_cmd: bool = True, + build_run_timeout: int = 600, ) ``` +### Launch a sandbox from a Dockerfile + +`image`, `rootfs`, and `context` are mutually exclusive. `Sandbox(context=...)` +implements a deliberately narrow, direct-launch subset of Dockerfile syntax +through the public `Sandbox`, `Commands`, and `Filesystem` facades. It needs no +BuildKit, Docker daemon, or registry push. + +**`FROM` is rootfs-only.** The selected image supplies the sandbox root +filesystem only. Its OCI `ENV`, `USER`, `WORKDIR`, `CMD`, and `ENTRYPOINT` +configuration is not inherited. Declare every required runtime setting in the +Dockerfile passed to `DockerContext`. + +Pre-check the Dockerfile before creating a sandbox. A check returning `False` +means direct launch will fail closed; pre-build externally instead. This example +creates one context instance, reports diagnostic warnings, launches it, and +inspects the optional startup handle: + +```python +from akernel_sdk import LocalDockerContext, Sandbox, check_direct_launch + +context = LocalDockerContext("Dockerfile", context_dir=".") +check = check_direct_launch(context) +for warning in check.warnings: + print(f"Dockerfile warning: {warning}") + +if check.direct_launchable: + with Sandbox(context=context, build_run_timeout=300) as sandbox: + startup = sandbox.startup_command + if startup is not None: + result = startup.wait(timeout=60) + print(result.exit_code, result.stderr) + # Perform the application's own health check here when it is long-lived. +else: + image = build_image_externally("Dockerfile") + with Sandbox(image=image) as sandbox: + # An image launch does not auto-start its configured CMD or ENTRYPOINT. + sandbox.commands.run("/srv/app --serve", background=True) +``` + +`check_direct_launch()` is diagnostic and returns reason codes such as +`multi_stage`, `remote_add`, `no_from`, `unsupported_instruction`, and +`unsupported_syntax`. `Sandbox(context=...)` parses strictly, while +`parse_dockerfile(strict=False)` is diagnostic only. `apply_dockerfile()` also +rejects a parsed result containing unsupported syntax. + +| Supported direct-launch subset | Rejected and requires an external build | +| --- | --- | +| Exactly one literal `FROM`, optionally `AS alias`; shell-form `RUN` | Multiple stages, `COPY --from`, `FROM` flags or variables, and exec-form `RUN` | +| Shell-form local `COPY` and `ADD` paths: files, directories, `.`, and wildcards; literal `--chown`; literal local tar extraction for `ADD` | JSON-form `COPY`/`ADD`, remote `ADD` URLs, `--chmod`, `--link`, unknown flags, or build-time variable expansion | +| Literal `ENV`, absolute `WORKDIR`, literal `USER`, and `EXPOSE` metadata | Any `ARG`, relative `WORKDIR`, and `VOLUME`, `LABEL`, `HEALTHCHECK`, `SHELL`, `STOPSIGNAL`, `ONBUILD`, `MAINTAINER`, or unknown instructions | +| Exec- or shell-form `CMD` and `ENTRYPOINT`, normalized and combined | — | + +A `DockerContext` exposes Dockerfile text plus context files. `LocalDockerContext` +uses a local directory; custom contexts may implement the same abstraction. +Before any sandbox operation, direct launch walks and validates the context +manifest, applies root `.dockerignore` rules, excludes `Dockerfile` and +`.dockerignore` from selection, plans every `COPY`/`ADD`, and materializes all +selected files. Local context symbolic links are rejected, and paths and target +collisions fail closed. `--chown` affects only files and directories created by +the current instruction. Local tar `ADD` accepts only regular files and +directories with safe paths. Remote `ADD` URLs are rejected; the SDK never +fetches them. Secure local context opening requires platform support for +directory-relative file descriptors and no-follow flags; unsupported platforms +fail closed. + +Each launch executes `RUN`, `COPY`, and `ADD` again in a new sandbox. There is +no snapshot or build cache. After instructions finish, the SDK polls **sandbox +readiness** and dispatches the resolved `CMD`/`ENTRYPOINT` in the background. +`Sandbox.startup_command` is a `CommandHandle | None`; use `wait()` for finite +commands or `kill()` when needed. Construction confirms dispatch, not that an +application remains running or passes a health check. Callers own +application-specific readiness checks. + +For Dockerfiles outside this subset, or for build-once reuse, pre-build the +image with your chosen build system and use `Sandbox(image=...)`. Then launch +the desired command explicitly with `sandbox.commands.run(..., background=True)`; +image configuration does not auto-start a process in this SDK path. + +Parsing uses [`dockerfile-parse`](https://github.com/containerbuildsystem/dockerfile-parse) +(BSD-3-Clause), and context matching uses [`pathspec`](https://github.com/cpburnz/python-path-specification) +(MIT). The value post-processing approach was informed by the +[E2B Python SDK](https://github.com/e2b-dev/E2B) (MIT). + ### Experimental GPU and writable storage Request a whole NVIDIA GPU by type, exact product model, and count: @@ -461,6 +547,7 @@ Maintained examples are under [`examples/`](./examples): - `basic_usage.py` - `command_stdin.py` - `custom_image.py` +- `dockerfile_launch.py` - `gpu_sandbox.py` - `named_sandbox.py` - `network_policy.py` diff --git a/sdk/python/akernel_sdk/__init__.py b/sdk/python/akernel_sdk/__init__.py index 0134a4c..eacdffd 100644 --- a/sdk/python/akernel_sdk/__init__.py +++ b/sdk/python/akernel_sdk/__init__.py @@ -56,6 +56,17 @@ "BackendNotInstalledError", "UnsupportedBackendFeatureError", "BackendOperationError", + "DockerContext", + "LocalDockerContext", + "parse_dockerfile", + "check_direct_launch", + "apply_dockerfile", + "ParsedDockerfile", + "DockerfileApplyResult", + "DockerfileCheckResult", + "DockerfileBuildError", + "DockerfileParseError", + "BuildInstruction", ] _LAZY_IMPORTS = { @@ -65,6 +76,17 @@ "PtySession": (".pty", "PtySession"), "PtyError": (".pty", "PtyError"), "resources": ("._resources", "resources"), + "DockerContext": ("._dockercontext", "DockerContext"), + "LocalDockerContext": ("._dockercontext", "LocalDockerContext"), + "parse_dockerfile": ("._dockerfile", "parse_dockerfile"), + "check_direct_launch": ("._dockerfile", "check_direct_launch"), + "ParsedDockerfile": ("._dockerfile", "ParsedDockerfile"), + "DockerfileCheckResult": ("._dockerfile", "DockerfileCheckResult"), + "DockerfileParseError": ("._dockerfile", "DockerfileParseError"), + "BuildInstruction": ("._dockerfile", "BuildInstruction"), + "apply_dockerfile": ("._dockerfile_runner", "apply_dockerfile"), + "DockerfileApplyResult": ("._dockerfile_runner", "DockerfileApplyResult"), + "DockerfileBuildError": ("._dockerfile", "DockerfileBuildError"), } diff --git a/sdk/python/akernel_sdk/_dockercontext.py b/sdk/python/akernel_sdk/_dockercontext.py new file mode 100644 index 0000000..626be14 --- /dev/null +++ b/sdk/python/akernel_sdk/_dockercontext.py @@ -0,0 +1,538 @@ +# Copyright (c) 2026 Ant Group Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Dockerfile + build context source abstraction. + +The Dockerfile sandbox-launch path parses a Dockerfile and executes its +build-time instructions inside an AKernel sandbox. A DockerContext abstracts +where the Dockerfile text and the context files come from. Local directories +are the built-in implementation (LocalDockerContext); subclasses can load from +OSS, S3, memory, etc. +""" + +from __future__ import annotations + +import fnmatch +import os +import posixpath +import stat +from abc import ABC, abstractmethod +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import BinaryIO, Literal + +from pathspec import GitIgnoreSpec + +_SECURE_OPEN_SUPPORTED = ( + os.open in getattr(os, "supports_dir_fd", ()) + and hasattr(os, "O_DIRECTORY") + and hasattr(os, "O_NOFOLLOW") +) + + +class DockerContextError(ValueError): + """Raised when a Docker build context cannot be safely selected.""" + + +@dataclass(frozen=True) +class _SelectedContextFile: + """A source context file and its relative destination target.""" + + source_path: str + relative_target: str + + +@dataclass(frozen=True) +class _ContextSelection: + """A deterministic file selection for one COPY or ADD source.""" + + source: str + kind: Literal["literal_file", "literal_directory", "dot", "wildcard"] + files: tuple[_SelectedContextFile, ...] + + +@dataclass(frozen=True) +class _ContextManifest: + """Validated, ignored-filtered context entries used to plan selections.""" + + _raw_files: tuple[str, ...] + _visible_files: tuple[str, ...] + _raw_directories: frozenset[str] + + @classmethod + def from_context(cls, context: DockerContext) -> _ContextManifest: + """Build a manifest without opening selectable context files.""" + + try: + raw_files = tuple(context.walk()) + except Exception as error: + raise DockerContextError("failed to walk Docker context") from error + + _validate_walk_paths(raw_files) + raw_files = tuple(sorted(raw_files)) + raw_directories = _directories_from_files(raw_files) + ignore_spec = _read_ignore_spec(context, raw_files) + + reserved = {path for path in raw_files if _is_reserved_root_path(path)} + visible_files = tuple( + path + for path in raw_files + if path not in reserved and not ignore_spec.match_file(path) + ) + return cls(raw_files, visible_files, frozenset(raw_directories)) + + def select(self, source: str) -> _ContextSelection: + """Plan one normalized Docker COPY or ADD source without opening files.""" + + normalized = _normalize_source(source) + if normalized == ".": + if not self._visible_files: + reason = "ignored" if self._raw_files else "no match" + raise DockerContextError(f"context source is {reason}: {normalized!r}") + return _ContextSelection( + source=normalized, + kind="dot", + files=tuple( + _SelectedContextFile(path, path) for path in self._visible_files + ), + ) + if _has_wildcard(normalized): + return self._select_wildcard(normalized) + return self._select_literal(normalized) + + def _select_literal(self, source: str) -> _ContextSelection: + if source in self._raw_files: + if source not in self._visible_files: + raise DockerContextError(f"context source is ignored: {source!r}") + return _ContextSelection( + source=source, + kind="literal_file", + files=(_SelectedContextFile(source, posixpath.basename(source)),), + ) + + if source in self._raw_directories: + files = tuple( + _SelectedContextFile(path, path.removeprefix(f"{source}/")) + for path in self._visible_files + if path.startswith(f"{source}/") + ) + if not files: + raise DockerContextError(f"context source is ignored: {source!r}") + return _ContextSelection( + source=source, + kind="literal_directory", + files=_validate_selection_files(files), + ) + + raise DockerContextError(f"context source has no match: {source!r}") + + def _select_wildcard(self, source: str) -> _ContextSelection: + matched_files = [ + path for path in self._raw_files if _glob_matches(source, path) + ] + matched_directories = [ + path for path in self._raw_directories if _glob_matches(source, path) + ] + if not matched_files and not matched_directories: + raise DockerContextError(f"context source has no match: {source!r}") + + top_directories: list[str] = [] + for directory in sorted( + matched_directories, key=lambda item: (item.count("/"), item) + ): + if not any( + directory.startswith(f"{parent}/") for parent in top_directories + ): + top_directories.append(directory) + + files: list[_SelectedContextFile] = [] + for directory in top_directories: + prefix = f"{directory}/" + for path in self._visible_files: + if path.startswith(prefix): + child = path.removeprefix(prefix) + files.append( + _SelectedContextFile( + path, f"{posixpath.basename(directory)}/{child}" + ) + ) + + for path in self._visible_files: + if not _glob_matches(source, path): + continue + if any(path.startswith(f"{directory}/") for directory in top_directories): + continue + files.append(_SelectedContextFile(path, posixpath.basename(path))) + + if not files: + raise DockerContextError(f"context source is ignored: {source!r}") + return _ContextSelection( + source=source, + kind="wildcard", + files=_validate_selection_files(files), + ) + + +class DockerContext(ABC): + """Dockerfile + build context files source abstraction. + + Local directories are the built-in implementation; subclasses can load from + OSS, S3, memory, etc. File access is uniformly exposed as an open() stream + so the runner can materialize and upload regardless of source. + """ + + @abstractmethod + def dockerfile_text(self) -> str: + """Return the Dockerfile content.""" + + @abstractmethod + @contextmanager + def open(self, path: str) -> Iterator[BinaryIO]: + """Open a file by its relative POSIX path inside the context. + + Yields a binary stream. The path is relative to the context root and + uses POSIX separators regardless of host OS. + """ + + @abstractmethod + def walk(self) -> Iterator[str]: + """Enumerate relative POSIX paths of all context files. + + The Dockerfile itself is not part of the enumerated context files. + """ + + +def _to_posix(rel: str) -> str: + """Normalize a relative path to POSIX separators.""" + + return rel.replace(os.sep, "/").lstrip("/") + + +def _validate_walk_paths(paths: tuple[object, ...]) -> None: + """Reject malformed context paths before they can affect selection.""" + + seen: set[str] = set() + for path in paths: + if not isinstance(path, str): + raise DockerContextError("context walk yielded a non-string path") + if not path: + raise DockerContextError("context walk yielded an empty path") + if "\0" in path: + raise DockerContextError("context walk yielded a path with NUL") + if "\\" in path: + raise DockerContextError("context walk yielded a path with backslash") + if posixpath.isabs(path): + raise DockerContextError("context walk yielded an absolute path") + if path.endswith("/"): + raise DockerContextError("context walk yielded a directory path") + segments = path.split("/") + if any(segment in ("", ".", "..") for segment in segments): + raise DockerContextError("context walk yielded an unsafe path") + if posixpath.normpath(path) != path: + raise DockerContextError("context walk yielded a non-normalized path") + if path in seen: + raise DockerContextError(f"context walk yielded a duplicate path: {path!r}") + seen.add(path) + + +def _directories_from_files(paths: tuple[str, ...]) -> set[str]: + """Derive non-empty directories from walked file paths.""" + + directories: set[str] = set() + for path in paths: + segments = path.split("/") + for index in range(1, len(segments)): + directories.add("/".join(segments[:index])) + return directories + + +def _is_reserved_root_path(path: str) -> bool: + """Return whether a root metadata file is never selectable.""" + + return path.lower() == "dockerfile" or path == ".dockerignore" + + +def _read_ignore_spec( + context: DockerContext, raw_files: tuple[str, ...] +) -> GitIgnoreSpec: + """Read and compile the root .dockerignore when it was walked.""" + + if ".dockerignore" not in raw_files: + return GitIgnoreSpec.from_lines(()) + + try: + with context.open(".dockerignore") as stream: + content = stream.read() + lines = _prepare_ignore_lines(content.decode("utf-8-sig")) + return GitIgnoreSpec.from_lines(lines) + except DockerContextError: + raise + except Exception as error: + raise DockerContextError("failed to read .dockerignore") from error + + +def _prepare_ignore_lines(content: str) -> list[str]: + """Normalize common Docker .dockerignore syntax for GitIgnoreSpec.""" + + lines: list[str] = [] + for raw_line in content.splitlines(): + line = raw_line.strip() + if not line or line == ".": + continue + if line.startswith("#"): + continue + + negation = "" + pattern = line + if pattern.startswith("!"): + negation = "!" + pattern = pattern[1:] + + while pattern.startswith("/"): + pattern = pattern[1:] + while pattern.endswith("/") and not _is_escaped(pattern, len(pattern) - 1): + pattern = pattern[:-1] + + if not pattern or pattern == ".": + continue + lines.append(f"{negation}{pattern}") + return lines + + +def _is_escaped(value: str, index: int) -> bool: + """Return whether a character is preceded by an odd number of backslashes.""" + + count = 0 + index -= 1 + while index >= 0 and value[index] == "\\": + count += 1 + index -= 1 + return count % 2 == 1 + + +def _normalize_source(source: str) -> str: + """Normalize a Docker COPY or ADD source without permitting traversal.""" + + if not isinstance(source, str): + raise DockerContextError("context source must be a string") + if not source: + raise DockerContextError("context source is empty") + if "\0" in source: + raise DockerContextError("context source contains NUL") + if "\\" in source: + raise DockerContextError("context source contains backslash") + if posixpath.isabs(source): + raise DockerContextError("context source is absolute") + if any(segment == ".." for segment in source.split("/")): + raise DockerContextError("context source contains '..'") + + segments = [segment for segment in source.split("/") if segment not in ("", ".")] + normalized = "/".join(segments) + if not normalized: + if all(segment in ("", ".") for segment in source.split("/")): + return "." + raise DockerContextError("context source is empty") + return normalized + + +def _has_wildcard(path: str) -> bool: + """Return whether a path contains a POSIX glob metacharacter.""" + + return any(character in path for character in "*?[") + + +def _glob_matches(pattern: str, path: str) -> bool: + """Match a POSIX path where ** spans zero or more path segments.""" + + pattern_segments = pattern.split("/") + path_segments = path.split("/") + cache: dict[tuple[int, int], bool] = {} + + def match(pattern_index: int, path_index: int) -> bool: + key = (pattern_index, path_index) + if key in cache: + return cache[key] + if pattern_index == len(pattern_segments): + result = path_index == len(path_segments) + elif pattern_segments[pattern_index] == "**": + result = any( + match(pattern_index + 1, next_index) + for next_index in range(path_index, len(path_segments) + 1) + ) + elif path_index == len(path_segments): + result = False + else: + result = fnmatch.fnmatchcase( + path_segments[path_index], pattern_segments[pattern_index] + ) and match(pattern_index + 1, path_index + 1) + cache[key] = result + return result + + return match(0, 0) + + +def _validate_selection_files( + files: tuple[_SelectedContextFile, ...] | list[_SelectedContextFile], +) -> tuple[_SelectedContextFile, ...]: + """Sort selection files and reject source or target collisions.""" + + ordered = tuple(sorted(files, key=lambda item: item.source_path)) + source_paths = [item.source_path for item in ordered] + target_paths = [item.relative_target for item in ordered] + if len(source_paths) != len(set(source_paths)): + raise DockerContextError("context selection has duplicate source paths") + if len(target_paths) != len(set(target_paths)): + raise DockerContextError("context selection has colliding target paths") + return ordered + + +class LocalDockerContext(DockerContext): + """Local filesystem backed DockerContext. + + dockerfile may be either an existing path to a Dockerfile or the Dockerfile + content as a string (so callers can build a context purely in memory). + """ + + def __init__( + self, + dockerfile: str | Path, + context_dir: str | Path | None = None, + ) -> None: + dockerfile_str = str(dockerfile) + # If the argument points to an existing file, treat it as a path; + # otherwise treat it as Dockerfile content directly. + if os.path.isfile(dockerfile_str): + with open(dockerfile_str, encoding="utf-8") as handle: + self._dockerfile_text = handle.read() + self._dockerfile_path = os.path.abspath(dockerfile_str) + self._context_dir = os.path.abspath( + os.path.dirname(self._dockerfile_path) + if context_dir is None + else str(context_dir) + ) + else: + self._dockerfile_text = dockerfile_str + self._dockerfile_path = "" + self._context_dir = os.path.abspath( + "." if context_dir is None else str(context_dir) + ) + + def dockerfile_text(self) -> str: + return self._dockerfile_text + + @contextmanager + def open(self, path: str) -> Iterator[BinaryIO]: + """Open a regular context file without following symbolic links. + + Every path component is opened relative to an already-open directory + descriptor. This keeps validation and opening atomic with respect to + symlink replacement. + """ + + normalized = _normalize_source(path) + if normalized == ".": + raise DockerContextError("context file path must name a file") + if not _SECURE_OPEN_SUPPORTED: + raise DockerContextError( + "secure context file opening is not supported on this platform" + ) + + common_flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) + directory_flags = common_flags | os.O_DIRECTORY | os.O_NOFOLLOW + file_flags = common_flags | os.O_NOFOLLOW | getattr(os, "O_NONBLOCK", 0) + directory_fds: list[int] = [] + final_fd: int | None = None + handle: BinaryIO | None = None + try: + root_fd = os.open(self._context_dir, directory_flags) + directory_fds.append(root_fd) + parent_fd = root_fd + segments = normalized.split("/") + for segment in segments[:-1]: + parent_fd = os.open( + segment, + directory_flags, + dir_fd=parent_fd, + ) + directory_fds.append(parent_fd) + final_fd = os.open( + segments[-1], + file_flags, + dir_fd=parent_fd, + ) + if not stat.S_ISREG(os.fstat(final_fd).st_mode): + raise DockerContextError( + f"context path is not a regular file: {normalized!r}" + ) + handle = os.fdopen(final_fd, "rb") + final_fd = None + except DockerContextError: + raise + except (OSError, TypeError, ValueError) as error: + raise DockerContextError( + f"cannot securely open context file: {normalized!r}" + ) from error + finally: + if handle is None: + if final_fd is not None: + os.close(final_fd) + for descriptor in reversed(directory_fds): + os.close(descriptor) + + try: + yield handle + finally: + try: + handle.close() + finally: + for descriptor in reversed(directory_fds): + os.close(descriptor) + + def walk(self) -> Iterator[str]: + if not os.path.isdir(self._context_dir): + return + dockerfile_abs = ( + os.path.normpath(self._dockerfile_path) if self._dockerfile_path else None + ) + for root, dirs, files in os.walk(self._context_dir): + for name in dirs: + if os.path.islink(os.path.join(root, name)): + raise DockerContextError( + f"context contains a symbolic link: {name!r}" + ) + dirs.sort() + for name in sorted(files): + full = os.path.join(root, name) + if os.path.islink(full): + raise DockerContextError( + f"context contains a symbolic link: {name!r}" + ) + if dockerfile_abs and os.path.normpath(full) == dockerfile_abs: + continue + if ( + not self._dockerfile_path + and root == os.path.normpath(self._context_dir) + and name.lower() == "dockerfile" + ): + continue + rel = os.path.relpath(full, self._context_dir) + yield _to_posix(rel) + + @property + def context_dir(self) -> str: + """Absolute path to the local context root.""" + + return self._context_dir diff --git a/sdk/python/akernel_sdk/_dockerfile.py b/sdk/python/akernel_sdk/_dockerfile.py new file mode 100644 index 0000000..057d483 --- /dev/null +++ b/sdk/python/akernel_sdk/_dockerfile.py @@ -0,0 +1,838 @@ +# Copyright (c) 2026 Ant Group Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Dockerfile parsing for the sandbox-launch path. + +Line-level parsing is delegated to ``dockerfile-parse`` (BSD-3-Clause): it +handles comments, instruction case-insensitivity, line continuations and the +``# escape=`` directive. This module performs value-level post-processing and +translates the deliberately narrow direct-launch subset to typed structures. +""" + +from __future__ import annotations + +import json +import re +import shlex +from collections.abc import Sequence +from dataclasses import dataclass, field + +from ._dockercontext import DockerContext + + +@dataclass(frozen=True) +class FromInstruction: + image: str + + +@dataclass(frozen=True) +class RunInstruction: + command: str + + +@dataclass(frozen=True) +class CopyInstruction: + srcs: tuple[str, ...] + dest: str + chown: str | None = None + is_add: bool = False + + +@dataclass(frozen=True) +class EnvInstruction: + envs: dict[str, str] + + +@dataclass(frozen=True) +class WorkdirInstruction: + path: str + + +@dataclass(frozen=True) +class UserInstruction: + user: str + + +@dataclass(frozen=True) +class CmdInstruction: + cmd: tuple[str, ...] + shell_form: bool + + +@dataclass(frozen=True) +class EntrypointInstruction: + cmd: tuple[str, ...] + shell_form: bool + + +@dataclass(frozen=True) +class ExposeInstruction: + ports: tuple[str, ...] + + +@dataclass(frozen=True) +class UnsupportedInstruction: + """Syntax the sandbox-launch path deliberately does not execute.""" + + kind: str + value: str + reason: str = "unsupported_instruction" + + +BuildInstruction = ( + FromInstruction + | RunInstruction + | CopyInstruction + | EnvInstruction + | WorkdirInstruction + | UserInstruction + | CmdInstruction + | EntrypointInstruction + | ExposeInstruction +) + + +@dataclass(frozen=True) +class ParsedDockerfile: + base_image: str + instructions: tuple[BuildInstruction, ...] + unsupported: tuple[UnsupportedInstruction, ...] + envs: dict[str, str] = field(default_factory=dict) + workdir: str = "/" + user: str | None = None + start_cmd: tuple[str, ...] | None = None + entrypoint: tuple[str, ...] | None = None + exposed_ports: tuple[str, ...] = () + warnings: tuple[str, ...] = () + + +def _last_start_instructions( + instructions: Sequence[BuildInstruction], +) -> tuple[EntrypointInstruction | None, CmdInstruction | None]: + """Return the last declared ENTRYPOINT and CMD instructions.""" + entrypoint: EntrypointInstruction | None = None + cmd: CmdInstruction | None = None + for instruction in instructions: + if isinstance(instruction, EntrypointInstruction): + entrypoint = instruction + elif isinstance(instruction, CmdInstruction): + cmd = instruction + return entrypoint, cmd + + +def _as_argv(cmd: tuple[str, ...], *, shell_form: bool) -> tuple[str, ...]: + """Normalize a parsed command to argv without inferring from its text.""" + return ("/bin/sh", "-c", cmd[0]) if shell_form else cmd + + +def resolve_start_cmd( + instructions: Sequence[BuildInstruction], +) -> tuple[str, ...] | None: + """Resolve the effective command to executable argv per OCI semantics.""" + entrypoint, cmd = _last_start_instructions(instructions) + if entrypoint is not None: + if entrypoint.shell_form: + return _as_argv(entrypoint.cmd, shell_form=True) + if cmd is None: + return entrypoint.cmd + return entrypoint.cmd + _as_argv(cmd.cmd, shell_form=cmd.shell_form) + return _as_argv(cmd.cmd, shell_form=cmd.shell_form) if cmd is not None else None + + +def _resolve_entrypoint( + instructions: Sequence[BuildInstruction], +) -> tuple[str, ...] | None: + """Return the last declared ENTRYPOINT as executable argv.""" + entrypoint, _ = _last_start_instructions(instructions) + if entrypoint is None: + return None + return _as_argv(entrypoint.cmd, shell_form=entrypoint.shell_form) + + +_IGNORED_INSTRUCTIONS: dict[str, str] = { + "VOLUME": "not supported; use storage_mb or mounts for persistence", + "LABEL": "not supported", + "HEALTHCHECK": "not supported", + "SHELL": "not supported", + "STOPSIGNAL": "not supported", + "ONBUILD": "not supported", + "MAINTAINER": "not supported (deprecated)", +} + +_CHOWN_VALUE_RE = re.compile( + r"[A-Za-z0-9_][A-Za-z0-9_.-]*(?::[A-Za-z0-9_][A-Za-z0-9_.-]*)?" +) +_VARIABLE_RE = re.compile(r"\$(?:\{[^}]*\}|[A-Za-z_][A-Za-z0-9_]*)") +DIRECT_LAUNCH_ROOTFS_ONLY_WARNING = ( + "FROM supplies only the root filesystem; inherited image ENV, USER, WORKDIR, " + "CMD and ENTRYPOINT are not applied. Declare required runtime settings in this " + "Dockerfile or pre-build and use Sandbox(image=...)." +) + + +class DockerfileParseError(ValueError): + """Raised when a Dockerfile cannot be parsed for direct sandbox launch.""" + + def __init__(self, message: str, *, reason: str = "unsupported_syntax") -> None: + super().__init__(message) + self.reason = reason + + +class DockerfileBuildError(RuntimeError): + """Raised when a build-time instruction fails inside the sandbox.""" + + def __init__( + self, + message: str, + *, + index: int | None = None, + instruction: str | None = None, + stdout: str = "", + stderr: str = "", + exit_code: int | None = None, + ) -> None: + super().__init__(message) + self.index = index + self.instruction = instruction + self.stdout = stdout + self.stderr = stderr + self.exit_code = exit_code + + +@dataclass(frozen=True) +class DockerfileCheckResult: + """Result of :func:`check_direct_launch`.""" + + direct_launchable: bool + base_image: str | None + reasons: tuple[str, ...] + has_build_instructions: bool + ignored_instructions: tuple[str, ...] + warnings: tuple[str, ...] + + +def parse_dockerfile( + context: DockerContext, *, strict: bool = False +) -> ParsedDockerfile: + """Parse a Dockerfile into a :class:`ParsedDockerfile`. + + ``strict=True`` raises at the first unsupported instruction or syntax. + Non-strict parsing is diagnostic only: unsupported input is recorded and is + never translated into an executable instruction. + """ + structure = _load_structure(context.dockerfile_text()) + froms = [node for node in structure if node["instruction"] == "FROM"] + if not froms: + raise DockerfileParseError( + "Dockerfile must contain a FROM instruction", reason="no_from" + ) + if len(froms) > 1: + raise DockerfileParseError( + "Multi-stage Dockerfiles are not supported; pre-build the image " + "and launch with Sandbox(image=...)", + reason="multi_stage", + ) + + unsupported: list[UnsupportedInstruction] = [] + warnings: list[str] = [] + base_image = _parse_from(_value(froms[0]), strict, unsupported, warnings) + instructions: list[BuildInstruction] = [FromInstruction(image=base_image)] + envs: dict[str, str] = {} + workdir = "/" + user: str | None = None + exposed_ports: list[str] = [] + + for node in structure: + kind = node["instruction"] + value = _value(node) + if kind in ("COMMENT", "FROM"): + continue + if kind == "RUN": + if _json_array(value) is not None: + _unsupported( + kind, + value, + "exec-form RUN is not supported; use shell-form RUN", + "unsupported_syntax", + strict, + unsupported, + warnings, + ) + continue + cmd = _fold_run(value) + if not cmd: + continue + args = _split_shell_like(value, kind, strict, unsupported, warnings) + if args is None: + continue + if args and args[0].startswith("--"): + _unsupported( + kind, + value, + "RUN flags are not supported", + "unsupported_syntax", + strict, + unsupported, + warnings, + ) + continue + instructions.append(RunInstruction(command=cmd)) + elif kind in ("COPY", "ADD"): + _parse_copy_add(value, kind, strict, unsupported, warnings, instructions) + elif kind == "ENV": + parsed = _parse_env(value, strict, unsupported, warnings) + if parsed is not None: + envs.update(parsed) + instructions.append(EnvInstruction(envs=dict(parsed))) + elif kind == "ARG": + _unsupported( + kind, + value, + "ARG is not supported; pre-build the image and launch with " + "Sandbox(image=...)", + "unsupported_instruction", + strict, + unsupported, + warnings, + ) + elif kind == "WORKDIR": + wd = value.strip() + if not wd or not wd.startswith("/"): + _unsupported( + kind, + value, + "WORKDIR must be an absolute path", + "unsupported_syntax", + strict, + unsupported, + warnings, + ) + elif _contains_variable(wd): + _unsupported( + kind, + value, + "WORKDIR variable expansion is not supported", + "unsupported_syntax", + strict, + unsupported, + warnings, + ) + else: + workdir = wd + instructions.append(WorkdirInstruction(path=wd)) + elif kind == "USER": + user_value = value.strip() + if not user_value or _contains_variable(user_value): + _unsupported( + kind, + value, + "USER variable expansion is not supported", + "unsupported_syntax", + strict, + unsupported, + warnings, + ) + else: + user = user_value + instructions.append(UserInstruction(user=user_value)) + elif kind in ("CMD", "ENTRYPOINT"): + _parse_cmd_entrypoint( + value, kind, strict, unsupported, warnings, instructions + ) + elif kind == "EXPOSE": + ports = tuple(port for port in value.split() if port) + if ports: + exposed_ports.extend(ports) + instructions.append(ExposeInstruction(ports=ports)) + elif kind in _IGNORED_INSTRUCTIONS: + _unsupported( + kind, + value, + f"{kind} {_IGNORED_INSTRUCTIONS[kind]}", + "unsupported_instruction", + strict, + unsupported, + warnings, + ) + else: + _unsupported( + kind, + value, + f"{kind} is not supported", + "unsupported_instruction", + strict, + unsupported, + warnings, + ) + + return ParsedDockerfile( + base_image=base_image, + instructions=tuple(instructions), + unsupported=tuple(unsupported), + envs=envs, + workdir=workdir, + user=user, + start_cmd=resolve_start_cmd(instructions), + entrypoint=_resolve_entrypoint(instructions), + exposed_ports=tuple(exposed_ports), + warnings=tuple(warnings), + ) + + +def check_direct_launch( + context: DockerContext, *, strict: bool = False +) -> DockerfileCheckResult: + """Check whether ``Sandbox(context=...)`` can execute a Dockerfile safely.""" + try: + parsed = parse_dockerfile(context, strict=strict) + except DockerfileParseError as exc: + return DockerfileCheckResult( + direct_launchable=False, + base_image=None, + reasons=(exc.reason,), + has_build_instructions=False, + ignored_instructions=(), + warnings=(str(exc),), + ) + + has_build = any( + isinstance(instruction, (RunInstruction, CopyInstruction)) + for instruction in parsed.instructions + ) + unsupported_kinds = tuple(sorted({item.kind for item in parsed.unsupported})) + reasons = tuple(dict.fromkeys(item.reason for item in parsed.unsupported)) + warnings = list(parsed.warnings) + if has_build and not parsed.unsupported: + warnings.append( + "Dockerfile contains RUN/COPY/ADD; Sandbox(context=...) re-runs " + "them on every launch (no snapshot). For build-once reuse, " + "pre-build the image and launch with Sandbox(image=...)." + ) + if not parsed.unsupported: + warnings.append(DIRECT_LAUNCH_ROOTFS_ONLY_WARNING) + + return DockerfileCheckResult( + direct_launchable=not parsed.unsupported, + base_image=parsed.base_image if not parsed.unsupported else None, + reasons=reasons, + has_build_instructions=has_build, + ignored_instructions=unsupported_kinds, + warnings=tuple(warnings), + ) + + +# --------------------------------------------------------------------------- +# Value post-processing helpers +# --------------------------------------------------------------------------- + + +def _load_structure(text: str) -> Sequence[dict]: + """Load a Dockerfile's instruction structure via dockerfile-parse.""" + import os + import tempfile + + from dockerfile_parse import DockerfileParser # type: ignore[import-untyped] + + with tempfile.TemporaryDirectory() as tmp: + with open(os.path.join(tmp, "Dockerfile"), "w", encoding="utf-8") as fh: + fh.write(text) + parser = DockerfileParser(path=tmp) + return list(parser.structure) + + +def _value(node: dict) -> str: + return str(node.get("value", "")) + + +def _parse_from( + value: str, + strict: bool, + unsupported: list[UnsupportedInstruction], + warnings: list[str], +) -> str: + args = _split_shell_like(value, "FROM", strict, unsupported, warnings) + if args is None: + return "" + if not args: + _unsupported( + "FROM", + value, + "FROM requires an image", + "unsupported_syntax", + strict, + unsupported, + warnings, + ) + return "" + if args[0].startswith("--"): + _unsupported( + "FROM", + value, + "FROM flags are not supported", + "unsupported_syntax", + strict, + unsupported, + warnings, + ) + return _first_non_flag(args) + image = args[0] + if _contains_variable(image): + _unsupported( + "FROM", + value, + "FROM variable expansion is not supported", + "unsupported_syntax", + strict, + unsupported, + warnings, + ) + elif len(args) not in (1, 3) or (len(args) == 3 and args[1].lower() != "as"): + _unsupported( + "FROM", + value, + "FROM supports only 'FROM image AS alias'", + "unsupported_syntax", + strict, + unsupported, + warnings, + ) + return image + + +def _first_non_flag(args: list[str]) -> str: + for arg in args: + if not arg.startswith("--"): + return arg + return "" + + +def _json_array(value: str) -> list[object] | None: + """Return ``value`` as a JSON array, or ``None`` for shell-form text.""" + try: + parsed = json.loads(value) + except json.JSONDecodeError: + return None + return parsed if isinstance(parsed, list) else None + + +def _fold_run(value: str) -> str: + if not value.strip(): + return "" + return re.sub(r"\\\s*\n\s*", " ", value).strip() + + +def _split_shell_like( + value: str, + kind: str, + strict: bool, + unsupported: list[UnsupportedInstruction], + warnings: list[str], +) -> list[str] | None: + try: + return shlex.split(value, posix=True) + except ValueError: + _unsupported( + kind, + value, + f"{kind} contains malformed quoting", + "unsupported_syntax", + strict, + unsupported, + warnings, + ) + return None + + +def _parse_copy_add( + value: str, + kind: str, + strict: bool, + unsupported: list[UnsupportedInstruction], + warnings: list[str], + out: list[BuildInstruction], +) -> None: + if not value.strip(): + _unsupported( + kind, + value, + f"{kind} requires a source and a destination", + "unsupported_syntax", + strict, + unsupported, + warnings, + ) + return + if _json_array(value) is not None: + _unsupported( + kind, + value, + f"JSON-form {kind} is not supported", + "unsupported_syntax", + strict, + unsupported, + warnings, + ) + return + args = _split_shell_like(value, kind, strict, unsupported, warnings) + if args is None: + return + + chown: str | None = None + paths: list[str] = [] + for arg in args: + if arg.startswith("--from"): + _unsupported( + kind, + value, + "COPY --from is not supported (multi-stage); pre-build " + "the image and launch with Sandbox(image=...)", + "multi_stage", + strict, + unsupported, + warnings, + ) + return + if arg.startswith("--chown="): + chown = arg.split("=", 1)[1] + if not _CHOWN_VALUE_RE.fullmatch(chown) or _contains_variable(chown): + _unsupported( + kind, + value, + "--chown must be a literal user[:group]", + "unsupported_syntax", + strict, + unsupported, + warnings, + ) + return + continue + if arg.startswith("--"): + _unsupported( + kind, + value, + f"{kind} flag {arg!r} is not supported", + "unsupported_syntax", + strict, + unsupported, + warnings, + ) + return + paths.append(arg) + + if len(paths) < 2: + _unsupported( + kind, + value, + f"{kind} requires at least a source and a destination", + "unsupported_syntax", + strict, + unsupported, + warnings, + ) + return + if any(_contains_variable(path) for path in paths): + _unsupported( + kind, + value, + f"{kind} path variable expansion is not supported", + "unsupported_syntax", + strict, + unsupported, + warnings, + ) + return + if kind == "ADD" and any(_is_remote_url(path) for path in paths[:-1]): + _unsupported( + kind, + value, + "ADD remote URLs are not supported; download into the " + "build context first or pre-build and use Sandbox(image=...)", + "remote_add", + strict, + unsupported, + warnings, + ) + return + srcs = tuple(paths[:-1]) + dest = paths[-1] + if len(srcs) > 1 and not dest.endswith("/"): + _unsupported( + kind, + value, + f"{kind} destination must be a directory ending in '/' for " + "multiple sources", + "unsupported_syntax", + strict, + unsupported, + warnings, + ) + return + out.append( + CopyInstruction( + srcs=srcs, dest=dest, chown=chown, is_add=(kind == "ADD") + ) + ) + + +def _parse_env( + value: str, + strict: bool, + unsupported: list[UnsupportedInstruction], + warnings: list[str], +) -> dict[str, str] | None: + value = value.strip() + if not value: + _unsupported( + "ENV", + value, + "ENV requires a key and value", + "unsupported_syntax", + strict, + unsupported, + warnings, + ) + return None + try: + tokens = shlex.split(value, posix=True) + except ValueError: + _unsupported( + "ENV", + value, + "ENV contains malformed quoting", + "unsupported_syntax", + strict, + unsupported, + warnings, + ) + return None + if "=" in tokens[0]: + if not all("=" in token for token in tokens): + _unsupported( + "ENV", + value, + "ENV assignment form is malformed", + "unsupported_syntax", + strict, + unsupported, + warnings, + ) + return None + result = dict(token.split("=", 1) for token in tokens) + elif len(tokens) >= 2: + result = {tokens[0]: " ".join(tokens[1:])} + else: + _unsupported( + "ENV", + value, + "ENV requires a key and value", + "unsupported_syntax", + strict, + unsupported, + warnings, + ) + return None + if any(_contains_variable(item) for item in result.values()): + _unsupported( + "ENV", + value, + "ENV variable expansion is not supported", + "unsupported_syntax", + strict, + unsupported, + warnings, + ) + return None + return result + + +def _parse_cmd_entrypoint( + value: str, + kind: str, + strict: bool, + unsupported: list[UnsupportedInstruction], + warnings: list[str], + out: list[BuildInstruction], +) -> None: + value = value.strip() + if not value: + _unsupported( + kind, + value, + f"{kind} requires a command", + "unsupported_syntax", + strict, + unsupported, + warnings, + ) + return + parsed = _json_array(value) + if parsed is not None: + if not all( + isinstance(item, str) for item in parsed + ): + _unsupported( + kind, + value, + f"{kind} must be a JSON array of strings", + "unsupported_syntax", + strict, + unsupported, + warnings, + ) + return + cmd = tuple(item for item in parsed if isinstance(item, str)) + if not cmd or not cmd[0]: + _unsupported( + kind, + value, + f"{kind} requires a non-empty command name", + "unsupported_syntax", + strict, + unsupported, + warnings, + ) + return + if kind == "CMD": + out.append(CmdInstruction(cmd=cmd, shell_form=False)) + else: + out.append(EntrypointInstruction(cmd=cmd, shell_form=False)) + return + folded = re.sub(r"\\\s*\n\s*", " ", value).strip() + if kind == "CMD": + out.append(CmdInstruction(cmd=(folded,), shell_form=True)) + else: + out.append(EntrypointInstruction(cmd=(folded,), shell_form=True)) + + +def _contains_variable(value: str) -> bool: + return bool(_VARIABLE_RE.search(value)) + + +def _is_remote_url(value: str) -> bool: + return value.lower().startswith(("http://", "https://")) + + +def _unsupported( + kind: str, + value: str, + message: str, + reason: str, + strict: bool, + unsupported: list[UnsupportedInstruction], + warnings: list[str], +) -> None: + if strict: + raise DockerfileParseError(message, reason=reason) + unsupported.append(UnsupportedInstruction(kind=kind, value=value, reason=reason)) + warnings.append(message) diff --git a/sdk/python/akernel_sdk/_dockerfile_runner.py b/sdk/python/akernel_sdk/_dockerfile_runner.py new file mode 100644 index 0000000..815e56a --- /dev/null +++ b/sdk/python/akernel_sdk/_dockerfile_runner.py @@ -0,0 +1,785 @@ +# Copyright (c) 2026 Ant Group Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Runner that translates parsed Dockerfile instructions into sandbox ops. + +The runner walks a +:class:`~akernel_sdk._dockerfile.ParsedDockerfile`, maintaining an accumulated +``{envs, workdir, user}`` context, and drives the existing sandbox operations +(``commands.run``, ``files.copy_from_local``, ``files.make_dir``). No BuildKit, +no docker daemon, no registry push — execution happens inside the sandbox. +""" + +from __future__ import annotations + +import os +import posixpath +import shlex +import shutil +import tarfile +import tempfile +import time +from collections.abc import Iterable +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from ._dockercontext import DockerContext, _ContextManifest +from ._dockerfile import ( + CmdInstruction, + CopyInstruction, + DockerfileBuildError, + EntrypointInstruction, + EnvInstruction, + ExposeInstruction, + FromInstruction, + ParsedDockerfile, + RunInstruction, + UserInstruction, + WorkdirInstruction, + resolve_start_cmd, +) +from .commands import CommandHandle + +if TYPE_CHECKING: + from .sandbox import Sandbox + + +@dataclass(frozen=True) +class DockerfileApplyResult: + start_cmd: tuple[str, ...] | None + startup_command: CommandHandle | None + entrypoint: tuple[str, ...] | None + warnings: tuple[str, ...] + + +@dataclass(frozen=True) +class _PlannedCopy: + """One selected context file and its final sandbox target.""" + + source_path: str + relative_target: str + remote_path: str + local_path: str | None = None + + +@dataclass(frozen=True) +class _TarEntry: + """One validated regular file or directory in a local ADD archive.""" + + path: str + is_dir: bool + + +@dataclass(frozen=True) +class _PreparedCopy: + """A fully materialized COPY or ADD plan with no sandbox side effects.""" + + index: int + instruction: CopyInstruction + workdir: str + dest: str + plans: tuple[_PlannedCopy, ...] + has_directory_selection: bool + extract_tar: bool + tar_entries: tuple[_TarEntry, ...] = () + + +# Default polling cadence for sandbox readiness before dispatching CMD. +_SANDBOX_READY_POLL_INTERVAL = 0.5 +_SANDBOX_READY_POLL_TIMEOUT = 120 +_TAR_SUFFIXES = (".tar", ".tar.gz", ".tgz", ".tar.bz2", ".tar.xz", ".txz") + + +def apply_dockerfile( + sb: Sandbox, + parsed: ParsedDockerfile, + context: DockerContext, + *, + auto_start_cmd: bool = True, + run_timeout: int = 600, +) -> DockerfileApplyResult: + """Execute a parsed Dockerfile's build-time instructions inside ``sb``. + + Walks ``parsed.instructions`` in order, translating each to sandbox + operations. ``RUN``/``COPY``/``ADD`` failures raise + :class:`DockerfileBuildError`. + + Args: + sb: An already-launched sandbox (image = ``parsed.base_image``). + parsed: Output of :func:`~akernel_sdk._dockerfile.parse_dockerfile`. + context: The :class:`DockerContext` the parsed Dockerfile came from + (used to read context files for COPY/ADD). + auto_start_cmd: After build-time instructions, wait for sandbox readiness + and dispatch ``CMD``/``ENTRYPOINT`` in the background. The returned + command handle confirms dispatch only; no application health check is + performed. Default ``True``. + run_timeout: Per-``RUN`` timeout in seconds. Default ``600``. + """ + if parsed.unsupported: + kinds = ", ".join(sorted({item.kind for item in parsed.unsupported})) + raise DockerfileBuildError( + "Dockerfile contains unsupported instruction(s): " + kinds + ) + if not parsed.base_image: + raise ValueError("ParsedDockerfile.base_image is required to apply") + + runner = _Runner(sb, context, run_timeout) + warnings = list(parsed.warnings) + + with tempfile.TemporaryDirectory() as staging_dir: + prepared_copies: dict[int, _PreparedCopy] = {} + planned_workdir = "/" + for index, instruction in enumerate(parsed.instructions): + if isinstance(instruction, WorkdirInstruction): + planned_workdir = instruction.path + elif isinstance(instruction, CopyInstruction): + try: + prepared_copies[index] = runner.prepare_copy( + instruction, planned_workdir, index, staging_dir + ) + except DockerfileBuildError: + raise + except Exception as exc: # noqa: BLE001 — preserve Dockerfile location + raise DockerfileBuildError( + f"Instruction {index} failed: {exc}", + index=index, + instruction="ADD" if instruction.is_add else "COPY", + ) from exc + + # Drive the baseline runtime context after all COPY/ADD inputs are safe. + envs: dict[str, str] = {} + workdir = "/" + user: str | None = None + + for index, instruction in enumerate(parsed.instructions): + try: + if isinstance(instruction, FromInstruction): + continue # already used to launch the sandbox + elif isinstance(instruction, RunInstruction): + runner.run(instruction.command, envs, workdir, user, index) + elif isinstance(instruction, CopyInstruction): + runner.execute_prepared_copy(prepared_copies[index], envs, user) + elif isinstance(instruction, EnvInstruction): + envs.update(instruction.envs) + elif isinstance(instruction, WorkdirInstruction): + workdir = instruction.path + runner.ensure_workdir(workdir) + elif isinstance(instruction, UserInstruction): + user = instruction.user + elif isinstance(instruction, (CmdInstruction, EntrypointInstruction)): + continue # handled after build-time instructions + elif isinstance(instruction, ExposeInstruction): + continue # metadata only + except DockerfileBuildError: + raise + except Exception as exc: # noqa: BLE001 — surface as build error + raise DockerfileBuildError( + f"Instruction {index} failed: {exc}", + index=index, + ) from exc + + # Resolve the effective start command: ENTRYPOINT + CMD per OCI semantics. + start_cmd = _resolve_start_cmd(parsed) + + startup_command: CommandHandle | None = None + + # Sandbox-readiness gate before dispatching the resolved command. + if auto_start_cmd and start_cmd is not None: + runner.wait_sandbox_ready(_SANDBOX_READY_POLL_TIMEOUT) + try: + startup_command = runner.launch_start_cmd( + start_cmd, envs, workdir, user + ) + except Exception as exc: # noqa: BLE001 - attach Dockerfile metadata + raise DockerfileBuildError( + "Failed to dispatch startup command: " + str(exc), instruction="CMD" + ) from exc + + return DockerfileApplyResult( + start_cmd=start_cmd, + startup_command=startup_command, + entrypoint=parsed.entrypoint, + warnings=tuple(warnings), + ) + + +# --------------------------------------------------------------------------- +# Internal runner +# --------------------------------------------------------------------------- + + +class _Runner: + def __init__(self, sb: Sandbox, context: DockerContext, run_timeout: int) -> None: + self._sb = sb + self._context = context + self._run_timeout = run_timeout + self._manifest: _ContextManifest | None = None + + def _context_manifest(self) -> _ContextManifest: + if self._manifest is not None: + return self._manifest + try: + self._manifest = _ContextManifest.from_context(self._context) + except Exception as error: + cause = error.__cause__ + detail = f"{error}: {cause}" if cause is not None else str(error) + raise DockerfileBuildError( + f"Failed to build Docker context manifest for " + f"{type(self._context).__name__}: {detail}" + ) from error + return self._manifest + + # -- RUN -------------------------------------------------------------- + def run( + self, + command: str, + envs: dict[str, str], + workdir: str, + user: str | None, + index: int, + ) -> None: + wrapped = wrap_user(command, user) + result = self._sb.commands.run( + wrapped, + envs=envs or None, + cwd=workdir if workdir != "/" else None, + timeout=self._run_timeout, + ) + if result.exit_code != 0: + raise DockerfileBuildError( + f"RUN failed with exit code {result.exit_code}: {command[:120]}", + index=index, + instruction="RUN", + stdout=result.stdout, + stderr=result.stderr, + exit_code=result.exit_code, + ) + + # -- COPY / ADD ------------------------------------------------------- + def prepare_copy( + self, + ins: CopyInstruction, + workdir: str, + index: int, + staging_dir: str, + ) -> _PreparedCopy: + instruction = "ADD" if ins.is_add else "COPY" + dest = self._copy_destination(ins.dest, workdir, index, instruction) + manifest = self._context_manifest() + selections = [] + for source in ins.srcs: + try: + selections.append((source, manifest.select(source))) + except Exception as error: + raise DockerfileBuildError( + f"{instruction} source {source!r} cannot be selected: {error}", + index=index, + instruction=instruction, + ) from error + + plans: list[_PlannedCopy] = [] + has_directory_selection = False + for source, selection in selections: + directory_selection = selection.kind in ("literal_directory", "dot") + wildcard_directory_expansion = selection.kind == "wildcard" and any( + "/" in selected.relative_target for selected in selection.files + ) + must_use_directory = directory_selection or ( + selection.kind == "wildcard" + and (len(selection.files) > 1 or wildcard_directory_expansion) + ) + if ( + must_use_directory + and not ins.dest.endswith("/") + and not directory_selection + ): + raise DockerfileBuildError( + f"{instruction} source {source!r} expands to multiple paths; " + "destination must end in '/'", + index=index, + instruction=instruction, + ) + has_directory_selection |= directory_selection + for selected in selection.files: + try: + remote_path = self._copy_target( + dest, + selected.relative_target, + directory_selection or ins.dest.endswith("/"), + ) + except Exception as error: + raise DockerfileBuildError( + f"{instruction} source {source!r} has an invalid target: " + f"{error}", + index=index, + instruction=instruction, + ) from error + plans.append( + _PlannedCopy( + source_path=selected.source_path, + relative_target=selected.relative_target, + remote_path=remote_path, + ) + ) + + self._validate_copy_plan(plans, index, instruction) + extract_tar = ( + ins.is_add + and len(ins.srcs) == 1 + and selections[0][1].kind == "literal_file" + and len(selections[0][1].files) == 1 + and selections[0][1].files[0].source_path.lower().endswith(_TAR_SUFFIXES) + ) + + materialized = self._materialize( + plans, os.path.join(staging_dir, f"{index:08d}"), index, instruction + ) + tar_entries = ( + self._read_tar_entries(materialized[0].local_path or "", index) + if extract_tar + else () + ) + return _PreparedCopy( + index=index, + instruction=ins, + workdir=workdir, + dest=dest, + plans=tuple(materialized), + has_directory_selection=has_directory_selection, + extract_tar=extract_tar, + tar_entries=tar_entries, + ) + + def execute_prepared_copy( + self, + prepared: _PreparedCopy, + envs: dict[str, str], + user: str | None, + ) -> None: + ins = prepared.instruction + instruction = "ADD" if ins.is_add else "COPY" + if prepared.extract_tar: + archive = prepared.plans[0] + tar_directories = self._tar_directories(prepared) + self._ensure_tar_paths_not_symlinks(prepared, tar_directories) + new_directories = ( + self._new_directories(tar_directories) if ins.chown else () + ) + self._extract_tar( + archive.local_path or "", + posixpath.basename(archive.source_path), + prepared.dest, + envs, + prepared.workdir, + user, + prepared.index, + ) + chown_targets = ( + tuple( + self._tar_remote_path(prepared.dest, entry.path) + for entry in prepared.tar_entries + if not entry.is_dir + ) + + new_directories + ) + else: + parents = self._parent_directories( + plan.remote_path for plan in prepared.plans + ) + new_directories = self._new_directories(parents) if ins.chown else () + for parent in parents: + self._sb.files.make_dir(parent) + for plan in prepared.plans: + if plan.local_path is None: + raise AssertionError("copy plan was not materialized") + self._sb.files.copy_from_local(plan.local_path, plan.remote_path) + chown_targets = ( + tuple(dict.fromkeys(plan.remote_path for plan in prepared.plans)) + + new_directories + ) + + if ins.chown: + self._chown( + ins.chown, + chown_targets, + prepared.workdir, + prepared.index, + instruction, + ) + + def _copy_destination( + self, dest: str, workdir: str, index: int, instruction: str + ) -> str: + try: + candidate = dest if dest.startswith("/") else _join_posix(workdir, dest) + if "\0" in candidate or any(part == ".." for part in candidate.split("/")): + raise ValueError("destination escapes the sandbox root") + normalized = posixpath.normpath(candidate) + if not posixpath.isabs(normalized): + raise ValueError("destination is not absolute") + return normalized + except Exception as error: + raise DockerfileBuildError( + f"{instruction} destination {dest!r} is invalid: {error}", + index=index, + instruction=instruction, + ) from error + + def _copy_target(self, dest: str, relative_target: str, directory: bool) -> str: + candidate = posixpath.join(dest, relative_target) if directory else dest + target = posixpath.normpath(candidate) + if ( + not posixpath.isabs(target) + or target == "/" + or any(part == ".." for part in candidate.split("/")) + ): + raise ValueError(f"invalid COPY target: {target!r}") + return target + + def _validate_copy_plan( + self, plans: list[_PlannedCopy], index: int, instruction: str + ) -> None: + if not plans: + raise DockerfileBuildError( + f"{instruction} sources select no files", + index=index, + instruction=instruction, + ) + relative_targets = [plan.relative_target for plan in plans] + if len(relative_targets) != len(set(relative_targets)): + raise DockerfileBuildError( + f"{instruction} sources have colliding relative targets", + index=index, + instruction=instruction, + ) + targets = {plan.remote_path for plan in plans} + if len(targets) != len(plans): + raise DockerfileBuildError( + f"{instruction} sources have colliding destination paths", + index=index, + instruction=instruction, + ) + for target in targets: + parent = posixpath.dirname(target) + while parent != "/": + if parent in targets: + raise DockerfileBuildError( + f"{instruction} destination file conflicts with descendant: " + f"{parent!r}", + index=index, + instruction=instruction, + ) + parent = posixpath.dirname(parent) + + def _materialize( + self, + plans: list[_PlannedCopy], + staging_dir: str, + index: int, + instruction: str, + ) -> list[_PlannedCopy]: + materialized: list[_PlannedCopy] = [] + os.makedirs(staging_dir, exist_ok=True) + for sequence, plan in enumerate(plans): + local_path = os.path.join(staging_dir, f"{sequence:08d}") + try: + with self._context.open(plan.source_path) as stream: + with open(local_path, "wb") as output: + shutil.copyfileobj(stream, output) + except Exception as error: + raise DockerfileBuildError( + f"{instruction} source {plan.source_path!r} could not be read: " + f"{error}", + index=index, + instruction=instruction, + ) from error + materialized.append( + _PlannedCopy( + source_path=plan.source_path, + relative_target=plan.relative_target, + remote_path=plan.remote_path, + local_path=local_path, + ) + ) + return materialized + + def _read_tar_entries(self, local_tar: str, index: int) -> tuple[_TarEntry, ...]: + try: + with tarfile.open(local_tar, "r:*") as archive: + entries: list[_TarEntry] = [] + for member in archive.getmembers(): + self._validate_tar_member_type(member, index) + entries.append( + _TarEntry( + path=self._normalize_tar_path(member.name, index), + is_dir=member.isdir(), + ) + ) + validated_entries = tuple(entries) + self._validate_tar_entries(validated_entries, index) + return validated_entries + except DockerfileBuildError: + raise + except (OSError, EOFError, tarfile.TarError) as error: + raise DockerfileBuildError( + f"ADD archive is invalid: {error}", index=index, instruction="ADD" + ) from error + + def _validate_tar_member_type(self, member: tarfile.TarInfo, index: int) -> bool: + if member.isreg() or member.isdir(): + return True + raise DockerfileBuildError( + f"ADD archive contains unsupported member type: {member.name!r}", + index=index, + instruction="ADD", + ) + + def _normalize_tar_path(self, path: str, index: int) -> str: + if ( + not path + or path.startswith("/") + or "\\" in path + or any(ord(character) < 32 or ord(character) == 127 for character in path) + or any(part == ".." for part in path.split("/")) + ): + raise DockerfileBuildError( + f"ADD archive contains unsafe entry: {path!r}", + index=index, + instruction="ADD", + ) + normalized = posixpath.normpath(path) + if normalized in ("", ".") or normalized.startswith("/"): + raise DockerfileBuildError( + f"ADD archive contains unsafe entry: {path!r}", + index=index, + instruction="ADD", + ) + return normalized + + def _validate_tar_entries(self, entries: tuple[_TarEntry, ...], index: int) -> None: + by_path: dict[str, _TarEntry] = {} + for entry in entries: + if entry.path in by_path: + raise DockerfileBuildError( + f"ADD archive contains duplicate entry: {entry.path!r}", + index=index, + instruction="ADD", + ) + by_path[entry.path] = entry + for entry in entries: + parent = posixpath.dirname(entry.path) + while parent not in ("", "."): + ancestor = by_path.get(parent) + if ancestor is not None and not ancestor.is_dir: + raise DockerfileBuildError( + "ADD archive has a file-as-ancestor collision: " + f"{ancestor.path!r}", + index=index, + instruction="ADD", + ) + parent = posixpath.dirname(parent) + if not entry.is_dir and any( + other.path.startswith(entry.path + "/") for other in entries + ): + raise DockerfileBuildError( + f"ADD archive has a file-as-ancestor collision: {entry.path!r}", + index=index, + instruction="ADD", + ) + + def _parent_directories(self, paths: Iterable[str]) -> tuple[str, ...]: + directories: set[str] = set() + for path in paths: + parent = posixpath.dirname(path) + while parent != "/": + directories.add(parent) + parent = posixpath.dirname(parent) + return tuple(sorted(directories, key=lambda path: (path.count("/"), path))) + + def _new_directories(self, directories: tuple[str, ...]) -> tuple[str, ...]: + return tuple( + directory + for directory in directories + if not self._sb.files.exists(directory) + ) + + def _tar_remote_path(self, dest: str, path: str) -> str: + return posixpath.normpath(posixpath.join(dest, path)) + + def _tar_directories(self, prepared: _PreparedCopy) -> tuple[str, ...]: + member_paths = tuple( + self._tar_remote_path(prepared.dest, entry.path) + for entry in prepared.tar_entries + ) + directories = set(self._parent_directories(member_paths)) + directories.update( + self._tar_remote_path(prepared.dest, entry.path) + for entry in prepared.tar_entries + if entry.is_dir + ) + if prepared.dest != "/": + directories.add(prepared.dest) + return tuple(sorted(directories, key=lambda path: (path.count("/"), path))) + + def _ensure_tar_paths_not_symlinks( + self, prepared: _PreparedCopy, directories: tuple[str, ...] + ) -> None: + member_paths = tuple( + self._tar_remote_path(prepared.dest, entry.path) + for entry in prepared.tar_entries + ) + paths = tuple(dict.fromkeys((prepared.dest,) + member_paths + directories)) + command = " && ".join(f"test ! -L {shlex.quote(path)}" for path in paths) + result = self._sb.commands.run( + command, + timeout=self._run_timeout, + ) + if result.exit_code != 0: + raise DockerfileBuildError( + f"ADD archive destination path is a symlink: {result.stderr}", + index=prepared.index, + instruction="ADD", + stderr=result.stderr, + exit_code=result.exit_code, + ) + + def _chown( + self, + owner: str, + targets: tuple[str, ...], + workdir: str, + index: int, + instruction: str, + ) -> None: + for target in targets: + command = f"chown {shlex.quote(owner)} {shlex.quote(target)}" + result = self._sb.commands.run( + wrap_user(command, "root"), + cwd=workdir if workdir != "/" else None, + timeout=self._run_timeout, + ) + if result.exit_code != 0: + raise DockerfileBuildError( + f"{instruction} --chown failed: {result.stderr}", + index=index, + instruction=instruction, + stderr=result.stderr, + exit_code=result.exit_code, + ) + + def _extract_tar( + self, + local_tar: str, + archive_name: str, + dest: str, + envs: dict[str, str], + workdir: str, + user: str | None, + index: int, + ) -> None: + # Docker ADD extraction creates the destination directory before tar xf. + if dest != "/": + self._sb.files.make_dir(dest) + sandbox_tar = f"/tmp/akernel_add_{archive_name}" + self._sb.files.copy_from_local(local_tar, sandbox_tar) + tar_cmd = ( + f"tar xf {shlex.quote(sandbox_tar)} --no-same-owner -C {shlex.quote(dest)}" + ) + result = self._sb.commands.run( + wrap_user(tar_cmd, user), + envs=envs or None, + cwd=workdir if workdir != "/" else None, + timeout=self._run_timeout, + ) + if result.exit_code != 0: + raise DockerfileBuildError( + f"ADD tar extraction failed: {result.stderr}", + index=index, + instruction="ADD", + stderr=result.stderr, + exit_code=result.exit_code, + ) + + # -- WORKDIR ---------------------------------------------------------- + def ensure_workdir(self, workdir: str) -> None: + if workdir and workdir != "/": + self._sb.files.make_dir(workdir) + + # -- Sandbox readiness + CMD dispatch --------------------------------- + def wait_sandbox_ready(self, timeout: int) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if self._sb.is_running(): + return + time.sleep(_SANDBOX_READY_POLL_INTERVAL) + raise DockerfileBuildError( + "Sandbox did not become ready before dispatching startup command " + f"within {timeout}s", + instruction="CMD", + ) + + def launch_start_cmd( + self, + start_cmd: tuple[str, ...], + envs: dict[str, str], + workdir: str, + user: str | None, + ) -> CommandHandle: + # ``start_cmd`` is normalized executable argv by resolve_start_cmd. + wrapped = wrap_user(shlex.join(start_cmd), user) + return self._sb.commands.run( + wrapped, + background=True, + envs=envs or None, + cwd=workdir if workdir != "/" else None, + ) + + +def _join_posix(base: str, rel: str) -> str: + if not base.endswith("/"): + base = base + "/" + return base + rel.lstrip("/") + + +def wrap_user(command: str, user: str | None) -> str: + """Wrap ``command`` so it runs as ``user`` (root passthrough when None). + + Uses ``runuser`` by default. The actual fallback to ``su`` is resolved at + runtime inside the sandbox (see :func:`resolve_user_wrapper`); ``wrap_user`` + emits a wrapper that tries ``runuser`` first and falls back to ``su``. + """ + if not user: + return command + user = user.split(":", 1)[0] # strip :group + if not user or user == "root" or user == "0": + return command + # Try runuser, fall back to su if runuser is absent. + return ( + f"if command -v runuser >/dev/null 2>&1; then " + f"runuser -u {_shq(user)} -- sh -c {_shq(command)}; " + f"else su -s /bin/sh {_shq(user)} -c {_shq(command)}; fi" + ) + + +def _shq(s: str) -> str: + """Single-quote a shell string.""" + return "'" + s.replace("'", "'\\''") + "'" + + +def _resolve_start_cmd(parsed: ParsedDockerfile) -> tuple[str, ...] | None: + """Resolve the effective command through the shared OCI resolver. + + Shell forms are normalized to explicit ``/bin/sh -c`` argv. + """ + return resolve_start_cmd(parsed.instructions) diff --git a/sdk/python/akernel_sdk/sandbox.py b/sdk/python/akernel_sdk/sandbox.py index 02d426d..8bf2ad2 100644 --- a/sdk/python/akernel_sdk/sandbox.py +++ b/sdk/python/akernel_sdk/sandbox.py @@ -26,8 +26,9 @@ from ._addresses import Endpoint, api_endpoint_from_env, gateway_endpoint_from_env from ._backends.base import BackendSession, SandboxSpec from ._backends.registry import load_backend +from ._dockercontext import DockerContext from ._sandbox_resources import normalize_xpu, validate_storage_mb -from .commands import Commands +from .commands import CommandHandle, Commands from .filesystem import Filesystem from .pty import Pty from .types import HttpReverseTunnel, Mount, NetworkPolicy, S3Config, SandboxInfo @@ -133,6 +134,9 @@ def __init__( xpu: str | None = None, storage_mb: int | None = None, network_policy: NetworkPolicy | None = None, + context: DockerContext | None = None, + auto_start_cmd: bool = True, + build_run_timeout: int = 600, ) -> None: """Create and wait for a sandbox to become ready. @@ -164,6 +168,19 @@ def __init__( are validated against the selected runtime by the backend. network_policy: Optional creation-time network policy. Omitting it leaves sandbox networking unrestricted. + context: A :class:`DockerContext` carrying a Dockerfile and build + context. ``FROM`` supplies only the root filesystem; its OCI + ENV, USER, WORKDIR, CMD and ENTRYPOINT configuration is not + inherited. The sandbox applies only state explicitly declared + in this Dockerfile, then executes build-time instructions + in-sandbox. Mutually exclusive with ``image`` and ``rootfs``. + auto_start_cmd: When ``context`` is given, wait for sandbox + readiness and dispatch CMD/ENTRYPOINT in the background after + build-time instructions complete. Construction confirms only + successful dispatch, not that the process remains running or + healthy. Default ``True``. + build_run_timeout: Per-``RUN`` timeout in seconds when ``context`` + is given. Default ``600``. Raises: TypeError: An argument has an invalid type. @@ -175,8 +192,13 @@ def __init__( raise ValueError("image must be a non-empty string") if rootfs is not None and not isinstance(rootfs, S3Config): raise TypeError("rootfs must be an S3Config") - if image is not None and rootfs is not None: - raise ValueError("image and rootfs are mutually exclusive") + if context is not None and not isinstance(context, DockerContext): + raise TypeError("context must be a DockerContext") + if sum(value is not None for value in (image, rootfs, context)) > 1: + raise ValueError( + "image, rootfs and context are mutually exclusive: at most one " + "may be given" + ) if not isinstance(runtime, str): raise TypeError("runtime must be a string") runtime = runtime.strip() @@ -235,7 +257,17 @@ def __init__( f"reverse tunnel ports conflict with port_forwardings: {rendered}" ) + parsed_dockerfile = None + if context is not None: + from ._dockerfile import parse_dockerfile + + parsed_dockerfile = parse_dockerfile(context, strict=True) + image = parsed_dockerfile.base_image + if not isinstance(image, str) or not image.strip(): + raise ValueError("Dockerfile base image must be a non-empty string") + self._session: BackendSession | None = None + self._startup_command: CommandHandle | None = None self._pty: Pty | None = None self._closed = False self._terminated = detached @@ -280,6 +312,17 @@ def __init__( self._files = Filesystem(self._session.files) self._commands = Commands(self._session.commands) self._pty = Pty(self._id) + if context is not None and parsed_dockerfile is not None: + from ._dockerfile_runner import apply_dockerfile + + apply_result = apply_dockerfile( + self, + parsed_dockerfile, + context, + auto_start_cmd=auto_start_cmd, + run_timeout=build_run_timeout, + ) + self._startup_command = apply_result.startup_command except Exception: self._closed = True try: @@ -310,6 +353,19 @@ def commands(self) -> Commands: return self._commands + @property + def startup_command(self) -> CommandHandle | None: + """Background CMD/ENTRYPOINT handle for a context launch, if dispatched. + + The handle is available only when context and auto_start_cmd are used + with a startup command. It is None for normal image/rootfs launches, + auto_start_cmd=False, or Dockerfiles without CMD or ENTRYPOINT. + Sandbox construction does not guarantee that the process remains + running or healthy after dispatch. + """ + + return self._startup_command + @property def pty(self) -> Pty: """Factory for interactive pseudo-terminal sessions.""" diff --git a/sdk/python/examples/dockerfile_launch.py b/sdk/python/examples/dockerfile_launch.py new file mode 100644 index 0000000..40e677c --- /dev/null +++ b/sdk/python/examples/dockerfile_launch.py @@ -0,0 +1,186 @@ +# Copyright (c) 2026 Ant Group Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Launch sandboxes from the strict Dockerfile direct-launch subset. + +Each section builds a separate local context and creates a fresh sandbox. The +``FROM`` image provides only the root filesystem; the Dockerfile explicitly +sets runtime state. RUN/COPY/ADD execute for every launch, without a snapshot +or build cache. + +Sections: + 1. Core path — .dockerignore, COPY ., RUN, ENV, WORKDIR, USER, and CMD + 2. ENTRYPOINT + CMD exec-form combination + 3. COPY --chown ownership + 4. ADD local tar extraction + 5. Fail-closed pre-check without a sandbox +""" + +import tarfile +import tempfile +from pathlib import Path + +from akernel_sdk import LocalDockerContext, Sandbox, check_direct_launch + + +def _precheck(context: LocalDockerContext) -> None: + """Print diagnostics and assert direct launch is available.""" + result = check_direct_launch(context) + for warning in result.warnings: + print(f" precheck warning: {warning}") + assert result.direct_launchable, (result.reasons, result.warnings) + + +def section_core_path() -> None: + """Copy a filtered context, then run a finite CMD with explicit state.""" + print("\n=== Section 1: filtered core path ===") + with tempfile.TemporaryDirectory() as directory: + context_dir = Path(directory) + (context_dir / ".dockerignore").write_text("secret.txt\n", encoding="utf-8") + (context_dir / "secret.txt").write_text("do not upload\n", encoding="utf-8") + (context_dir / "app.py").write_text( + "import os\n" + "open('/tmp/app.started', 'w').write(\n" + " f\"whoami={os.environ['WHOAMI']} cwd={os.getcwd()}\"\n" + ")\n", + encoding="utf-8", + ) + (context_dir / "greeting.txt").write_text("hello\n", encoding="utf-8") + dockerfile = """\ +FROM ubuntu:22.04 +RUN apt-get update && apt-get install -y --no-install-recommends python3 +RUN useradd -m app +ENV WHOAMI=app +WORKDIR /srv +USER app +COPY . /srv/ +CMD ["python3", "/srv/app.py"] +""" + context = LocalDockerContext(dockerfile, context_dir=context_dir) + _precheck(context) + + with Sandbox(context=context, build_run_timeout=300) as sandbox: + startup = sandbox.startup_command + assert startup is not None + result = startup.wait(timeout=60) + assert result.exit_code == 0, result.stderr + + marker = sandbox.commands.run("cat /tmp/app.started") + assert marker.exit_code == 0, marker.stderr + assert marker.stdout.strip() == "whoami=app cwd=/srv", marker.stdout + absent = sandbox.commands.run("test ! -e /srv/secret.txt") + assert absent.exit_code == 0, absent.stderr + print(f" sandbox: {sandbox.id}; marker: {marker.stdout.strip()}") + + +def section_entrypoint_cmd_merge() -> None: + """Wait for an ENTRYPOINT + CMD command before reading its marker.""" + print("\n=== Section 2: ENTRYPOINT + CMD ===") + with tempfile.TemporaryDirectory() as directory: + context_dir = Path(directory) + dockerfile = """\ +FROM ubuntu:22.04 +ENTRYPOINT ["/bin/sh", "-c", "printf %s \\\"$1\\\" > /tmp/ep.out"] +CMD ["ignored-argv-zero", "entrypoint+cmd merged"] +""" + context = LocalDockerContext(dockerfile, context_dir=context_dir) + _precheck(context) + + with Sandbox(context=context) as sandbox: + startup = sandbox.startup_command + assert startup is not None + result = startup.wait(timeout=60) + assert result.exit_code == 0, result.stderr + output = sandbox.commands.run("cat /tmp/ep.out") + assert output.exit_code == 0, output.stderr + assert output.stdout == "entrypoint+cmd merged", output.stdout + print(f" sandbox: {sandbox.id}; output: {output.stdout}") + + +def section_copy_chown() -> None: + """Verify COPY --chown without dispatching a startup command.""" + print("\n=== Section 3: COPY --chown ===") + with tempfile.TemporaryDirectory() as directory: + context_dir = Path(directory) + (context_dir / "payload.txt").write_text("owned\n", encoding="utf-8") + dockerfile = """\ +FROM ubuntu:22.04 +RUN useradd -m myuser +COPY --chown=myuser:myuser payload.txt /data/payload.txt +""" + context = LocalDockerContext(dockerfile, context_dir=context_dir) + _precheck(context) + + with Sandbox(context=context) as sandbox: + assert sandbox.startup_command is None + owner = sandbox.commands.run("stat -c '%U:%G' /data/payload.txt") + assert owner.exit_code == 0, owner.stderr + assert owner.stdout.strip() == "myuser:myuser", owner.stdout + print(f" sandbox: {sandbox.id}; owner: {owner.stdout.strip()}") + + +def section_add_tar() -> None: + """Verify literal local ADD tar extraction without a startup command.""" + print("\n=== Section 4: ADD local tar ===") + with tempfile.TemporaryDirectory() as directory: + context_dir = Path(directory) + payload = context_dir / "payload" + nested = payload / "nested" + nested.mkdir(parents=True) + (payload / "top.txt").write_text("top\n", encoding="utf-8") + (nested / "child.txt").write_text("child\n", encoding="utf-8") + archive = context_dir / "app.tar.gz" + with tarfile.open(archive, "w:gz") as tar: + tar.add(payload / "top.txt", arcname="top.txt") + tar.add(nested, arcname="nested") + + context = LocalDockerContext( + "FROM ubuntu:22.04\nADD app.tar.gz /opt/app/\n", + context_dir=context_dir, + ) + _precheck(context) + + with Sandbox(context=context) as sandbox: + assert sandbox.startup_command is None + listing = sandbox.commands.run("find /opt/app -type f -printf '%P\n'") + assert listing.exit_code == 0, listing.stderr + assert set(listing.stdout.splitlines()) == {"nested/child.txt", "top.txt"} + print(f" sandbox: {sandbox.id}; files: {listing.stdout.strip()}") + + +def section_fail_closed_precheck() -> None: + """Reject remote ADD before a sandbox is created.""" + print("\n=== Section 5: fail-closed precheck ===") + with tempfile.TemporaryDirectory() as directory: + context = LocalDockerContext( + "FROM ubuntu:22.04\nADD https://example.test/app.tar /opt/app/\n", + context_dir=directory, + ) + result = check_direct_launch(context) + assert not result.direct_launchable + assert "remote_add" in result.reasons, result + print(f" rejected reasons: {', '.join(result.reasons)}") + + +def main() -> None: + section_core_path() + section_entrypoint_cmd_merge() + section_copy_chown() + section_add_tar() + section_fail_closed_precheck() + print("\nAll Dockerfile direct-launch sections passed.") + + +if __name__ == "__main__": + main() diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index 6b21031..683ca14 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -30,6 +30,8 @@ classifiers = [ dependencies = [ "openyuanrong-sandbox==0.9.7", "websockets>=10.0", + "dockerfile-parse>=2.0.1", + "pathspec>=0.12,<2", ] [project.optional-dependencies] diff --git a/sdk/python/tests/unit/test_dockercontext.py b/sdk/python/tests/unit/test_dockercontext.py new file mode 100644 index 0000000..f5d2466 --- /dev/null +++ b/sdk/python/tests/unit/test_dockercontext.py @@ -0,0 +1,321 @@ +"""Unit tests for Docker build context manifests and source selection.""" + +from __future__ import annotations + +import io +import os +import tempfile +import unittest +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import BinaryIO +from unittest.mock import patch + +from akernel_sdk._dockercontext import ( + DockerContext, + DockerContextError, + LocalDockerContext, + _ContextManifest, +) + +NL = bytes([10]) +CRLF = bytes([13, 10]) + + +class MemoryDockerContext(DockerContext): + """A context fixture that records manifest file opens.""" + + def __init__(self, files: dict[str, bytes], paths: list[object] | None = None): + self.files = files + self.paths = list(files) if paths is None else paths + self.open_paths: list[str] = [] + + def dockerfile_text(self) -> str: + return "FROM scratch" + + @contextmanager + def open(self, path: str) -> Iterator[BinaryIO]: + self.open_paths.append(path) + yield io.BytesIO(self.files[path]) + + def walk(self) -> Iterator[str]: + yield from self.paths # type: ignore[misc] + + +def paths(selection) -> list[tuple[str, str]]: + return [(item.source_path, item.relative_target) for item in selection.files] + + +class TestContextManifest(unittest.TestCase): + def test_deterministic_literal_file_directory_and_dot(self) -> None: + context = MemoryDockerContext( + {"src/lib/b.py": b"", "root.txt": b"", "src/a.py": b""}, + ["src/lib/b.py", "root.txt", "src/a.py"], + ) + manifest = _ContextManifest.from_context(context) + self.assertEqual(context.open_paths, []) + self.assertEqual(paths(manifest.select("src/a.py")), [("src/a.py", "a.py")]) + self.assertEqual( + paths(manifest.select("./src//")), + [("src/a.py", "a.py"), ("src/lib/b.py", "lib/b.py")], + ) + self.assertEqual( + paths(manifest.select(".")), + [ + ("root.txt", "root.txt"), + ("src/a.py", "src/a.py"), + ("src/lib/b.py", "src/lib/b.py"), + ], + ) + self.assertEqual(context.open_paths, []) + + def test_dockerignore_semantics_and_reserved_files(self) -> None: + ignore = ( + bytes([0xEF, 0xBB, 0xBF]) + + b"# comment" + + CRLF + + CRLF + + b"." + + CRLF + + b".env" + + CRLF + + b".git/" + + CRLF + + b"*.txt" + + CRLF + + b"!keep.txt" + + CRLF + + b"**/generated.py" + + CRLF + + b"/anchored/" + + CRLF + + b"tail/" + + CRLF + ) + context = MemoryDockerContext( + { + ".dockerignore": ignore, + "Dockerfile": b"", + ".env": b"", + ".git/config": b"", + "drop.txt": b"", + "keep.txt": b"", + "src/generated.py": b"", + "anchored/a": b"", + "tail/a": b"", + "src/ok.py": b"", + } + ) + manifest = _ContextManifest.from_context(context) + self.assertEqual(context.open_paths, [".dockerignore"]) + self.assertEqual( + paths(manifest.select(".")), + [("keep.txt", "keep.txt"), ("src/ok.py", "src/ok.py")], + ) + for source in (".dockerignore", "Dockerfile", ".env", ".git", "drop.txt"): + with self.subTest(source=source): + with self.assertRaisesRegex(DockerContextError, "ignored"): + manifest.select(source) + self.assertEqual(context.open_paths, [".dockerignore"]) + + def test_escaped_ignore_patterns_and_invalid_utf8(self) -> None: + slash = chr(92).encode() + escaped = MemoryDockerContext( + { + ".dockerignore": slash + b"#secret" + NL + slash + b"!secret" + NL, + "#secret": b"", + "!secret": b"", + "visible": b"", + } + ) + self.assertEqual( + paths(_ContextManifest.from_context(escaped).select(".")), + [("visible", "visible")], + ) + invalid = MemoryDockerContext({".dockerignore": bytes([0xFF]), "ok": b""}) + with self.assertRaisesRegex(DockerContextError, "dockerignore"): + _ContextManifest.from_context(invalid) + + def test_wildcards_directories_and_collisions(self) -> None: + manifest = _ContextManifest.from_context( + MemoryDockerContext( + { + "top.py": b"", + "src/a.py": b"", + "src/lib/b.py": b"", + "modules/one/x.py": b"", + "modules/two/y.txt": b"", + "one/x.py": b"", + "two/x.py": b"", + } + ) + ) + self.assertEqual(paths(manifest.select("*.py")), [("top.py", "top.py")]) + self.assertEqual( + paths(manifest.select("src/**/*.py")), + [("src/a.py", "a.py"), ("src/lib/b.py", "b.py")], + ) + self.assertEqual( + paths(manifest.select("modules/*")), + [("modules/one/x.py", "one/x.py"), ("modules/two/y.txt", "two/y.txt")], + ) + self.assertEqual( + paths(manifest.select("modules/**")), + [ + ("modules/one/x.py", "modules/one/x.py"), + ("modules/two/y.txt", "modules/two/y.txt"), + ], + ) + with self.assertRaisesRegex(DockerContextError, "colliding"): + manifest.select("*/*.py") + + def test_wildcard_no_match_and_only_ignored(self) -> None: + manifest = _ContextManifest.from_context( + MemoryDockerContext( + {".dockerignore": b"*.py" + NL, "hidden.py": b"", "visible.txt": b""} + ) + ) + with self.assertRaisesRegex(DockerContextError, "ignored"): + manifest.select("*.py") + with self.assertRaisesRegex(DockerContextError, "no match"): + manifest.select("*.go") + + def test_dot_distinguishes_empty_and_fully_filtered_contexts(self) -> None: + cases = ( + ({}, "no match"), + ({"Dockerfile": b""}, "ignored"), + ({".dockerignore": b"*.txt" + NL, "hidden.txt": b""}, "ignored"), + ) + for files, message in cases: + with self.subTest(files=files): + manifest = _ContextManifest.from_context(MemoryDockerContext(files)) + with self.assertRaisesRegex(DockerContextError, message): + manifest.select(".") + + def test_malicious_walk_paths_fail_closed(self) -> None: + invalid: list[object] = [ + 1, + "", + "/absolute", + "directory/", + "../parent", + "a/../b", + "./file", + "a//b", + "a" + chr(92) + "b", + "a" + chr(0) + "b", + ] + for value in invalid: + with self.subTest(value=repr(value)): + with self.assertRaises(DockerContextError): + _ContextManifest.from_context(MemoryDockerContext({}, [value])) + with self.assertRaises(DockerContextError): + _ContextManifest.from_context(MemoryDockerContext({}, ["same", "same"])) + + def test_malicious_sources_fail_closed(self) -> None: + manifest = _ContextManifest.from_context(MemoryDockerContext({"safe": b""})) + for source in ( + "", + "/absolute", + "../parent", + "a/../b", + "a" + chr(92) + "b", + "a" + chr(0), + ): + with self.subTest(source=repr(source)): + with self.assertRaises(DockerContextError): + manifest.select(source) + + def test_local_and_memory_contexts_match(self) -> None: + files = { + ".dockerignore": b"*.tmp" + NL + b"!keep.tmp" + NL, + "Dockerfile": b"", + "src/a.py": b"", + "src/cache.tmp": b"", + "keep.tmp": b"", + } + expected = _ContextManifest.from_context(MemoryDockerContext(files)).select(".") + with tempfile.TemporaryDirectory() as directory: + for name, content in files.items(): + path = Path(directory, name) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + actual = _ContextManifest.from_context( + LocalDockerContext("FROM scratch", context_dir=directory) + ).select(".") + self.assertEqual(expected, actual) + + def test_local_walk_determinism_and_symlink_protection(self) -> None: + with tempfile.TemporaryDirectory() as directory: + Path(directory, "z").write_bytes(b"") + Path(directory, "a").write_bytes(b"") + local = LocalDockerContext("FROM scratch", context_dir=directory) + self.assertEqual(list(local.walk()), ["a", "z"]) + outside = Path(directory).parent / "dockercontext-outside" + outside.write_bytes(b"outside") + try: + os.symlink(outside, Path(directory, "link")) + with self.assertRaisesRegex(DockerContextError, "symbolic link"): + list(local.walk()) + with self.assertRaisesRegex(DockerContextError, "securely open"): + with local.open("link"): + pass + finally: + outside.unlink(missing_ok=True) + + def test_local_open_rejects_symlink_replacement_race(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + context_dir = root / "context" + context_dir.mkdir() + selected = context_dir / "selected" + selected.write_bytes(b"safe") + (root / "host-secret").write_bytes(b"host-secret") + local = LocalDockerContext("FROM scratch", context_dir=context_dir) + real_open = os.open + replaced = False + + def racing_open( + path: str | bytes, + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + nonlocal replaced + if path == "selected" and dir_fd is not None and not replaced: + replaced = True + selected.unlink() + selected.symlink_to("../host-secret") + if dir_fd is None: + return real_open(path, flags, mode) + return real_open(path, flags, mode, dir_fd=dir_fd) + + with ( + patch( + "akernel_sdk._dockercontext.os.open", + side_effect=racing_open, + ), + self.assertRaisesRegex(DockerContextError, "securely open"), + ): + with local.open("selected") as stream: + self.fail(f"escaped context: {stream.read()!r}") + self.assertTrue(replaced) + + def test_local_open_accepts_nested_regular_files_only(self) -> None: + with tempfile.TemporaryDirectory() as directory: + nested = Path(directory, "nested") + nested.mkdir() + Path(nested, "file").write_bytes(b"ok") + local = LocalDockerContext("FROM scratch", context_dir=directory) + with local.open("nested/file") as stream: + self.assertEqual(stream.read(), b"ok") + with self.assertRaisesRegex(DockerContextError, "regular file"): + with local.open("nested"): + pass + for unsafe in ("/absolute", "../parent", "a\\b", "nul\0path"): + with self.subTest(path=unsafe): + with self.assertRaises(DockerContextError): + with local.open(unsafe): + pass diff --git a/sdk/python/tests/unit/test_dockerfile.py b/sdk/python/tests/unit/test_dockerfile.py new file mode 100644 index 0000000..0db1282 --- /dev/null +++ b/sdk/python/tests/unit/test_dockerfile.py @@ -0,0 +1,1283 @@ +# Copyright (c) 2026 Ant Group Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for the Dockerfile sandbox-launch path (RFC §8).""" + +from __future__ import annotations + +import io +import tarfile +import unittest +from collections.abc import Iterator +from contextlib import contextmanager +from typing import BinaryIO + +from akernel_sdk._dockercontext import ( + DockerContext, + DockerContextError, + LocalDockerContext, +) +from akernel_sdk._dockerfile import ( + DIRECT_LAUNCH_ROOTFS_ONLY_WARNING, + CmdInstruction, + CopyInstruction, + DockerfileBuildError, + DockerfileParseError, + RunInstruction, + _json_array, + check_direct_launch, + parse_dockerfile, +) +from akernel_sdk._dockerfile_runner import ( + _resolve_start_cmd, + _Runner, + apply_dockerfile, + wrap_user, +) + + +class _MockResult: + def __init__(self, exit_code=0, stdout="", stderr=""): + self.exit_code = exit_code + self.stdout = stdout + self.stderr = stderr + + +class _MockHandle: + pass + + +class _MockFiles: + def __init__(self, existing_paths: set[str] | None = None): + self.ops: list[tuple] = [] + self.existing_paths = existing_paths or set() + self.exists_calls: list[str] = [] + + def exists(self, path): + self.exists_calls.append(path) + return path in self.existing_paths + + def copy_from_local(self, local, remote): + self.ops.append(("cp", local, remote)) + + def make_dir(self, path): + self.ops.append(("mkdir", path)) + return True + + +class _MockCommands: + def __init__(self): + self.ops: list[tuple] = [] + + def run(self, cmd, background=False, envs=None, cwd=None, timeout=60, stdin=False): + self.ops.append(("run", cmd, background, envs, cwd, timeout)) + return _MockResult() + + +class _MockSandbox: + def __init__( + self, + failing_cmd=None, + startup_error=None, + existing_paths: set[str] | None = None, + ): + self.files = _MockFiles(existing_paths) + self.commands = _MockCommands( + failing=failing_cmd, + startup_error=startup_error, + ) + self._running = True + self.is_running_calls = 0 + + def is_running(self): + self.is_running_calls += 1 + return self._running + + +class _MockCommands(_MockCommands): + def __init__(self, failing=None, startup_error=None): + super().__init__() + self._failing = failing + self._startup_error = startup_error + self.startup_handle = _MockHandle() + + def run(self, cmd, background=False, envs=None, cwd=None, timeout=60, stdin=False): + self.ops.append(("run", cmd, background, envs, cwd, timeout)) + if background: + if self._startup_error is not None: + raise self._startup_error + return self.startup_handle + if self._failing and self._failing in cmd: + return _MockResult(exit_code=1, stderr=f"boom at {self._failing}") + return _MockResult() + + +def _archive_bytes( + members: list[tuple[tarfile.TarInfo, bytes | None]], +) -> bytes: + output = io.BytesIO() + with tarfile.open(fileobj=output, mode="w") as archive: + for info, content in members: + archive.addfile(info, io.BytesIO(content) if content is not None else None) + return output.getvalue() + + +def _regular_member(name: str, content: bytes = b"x") -> tuple[tarfile.TarInfo, bytes]: + info = tarfile.TarInfo(name) + info.size = len(content) + return info, content + + +class _MemoryDockerContext(DockerContext): + def __init__( + self, + dockerfile: str, + files: dict[str, bytes], + *, + fail_path: str | None = None, + ) -> None: + self._dockerfile = dockerfile + self._files = files + self._fail_path = fail_path + self.open_paths: list[str] = [] + + def dockerfile_text(self) -> str: + return self._dockerfile + + @contextmanager + def open(self, path: str) -> Iterator[BinaryIO]: + self.open_paths.append(path) + if path == self._fail_path: + raise OSError(f"cannot open {path}") + yield io.BytesIO(self._files[path]) + + def walk(self) -> Iterator[str]: + yield from self._files + + +class TestParseDockerfile(unittest.TestCase): + def _parse(self, content, strict=False): + return parse_dockerfile(LocalDockerContext(content), strict=strict) + + def test_basic_instructions(self): + parsed = self._parse( + "FROM ubuntu:22.04\n" + "RUN apt-get update\n" + "COPY app.py /app/\n" + "ENV K=v\n" + "WORKDIR /srv\n" + "USER app\n" + 'CMD ["python3", "app.py"]\n' + ) + self.assertEqual(parsed.base_image, "ubuntu:22.04") + self.assertEqual(parsed.envs, {"K": "v"}) + self.assertEqual(parsed.workdir, "/srv") + self.assertEqual(parsed.user, "app") + self.assertEqual(parsed.start_cmd, ("python3", "app.py")) + + def test_from_alias_unaffected(self): + parsed = self._parse("FROM node:20 AS builder\nRUN echo hi\n") + self.assertEqual(parsed.base_image, "node:20") + + def test_multi_stage_rejected(self): + with self.assertRaisesRegex(DockerfileParseError, "Multi-stage"): + self._parse("FROM ubuntu AS a\nFROM node:20\n") + + def test_missing_from(self): + with self.assertRaisesRegex(DockerfileParseError, "FROM"): + self._parse("RUN echo hi\n") + + def test_env_double_form(self): + parsed = self._parse( + "FROM ubuntu\nENV K1=v1 K2=v2\nENV SINGLE some value here\n" + ) + self.assertEqual( + parsed.envs, + {"K1": "v1", "K2": "v2", "SINGLE": "some value here"}, + ) + + def test_copy_chown_parsed(self): + parsed = self._parse("FROM ubuntu\nCOPY --chown=app:app src.py /app/\n") + ins = [i for i in parsed.instructions if isinstance(i, CopyInstruction)][0] + self.assertEqual(ins.chown, "app:app") + self.assertEqual(ins.srcs, ("src.py",)) + self.assertEqual(ins.dest, "/app/") + + def test_cmd_shell_form(self): + parsed = self._parse("FROM ubuntu\nCMD python3 app.py\n") + ins = [i for i in parsed.instructions if isinstance(i, CmdInstruction)][0] + self.assertTrue(ins.shell_form) + + def test_parsed_start_cmd_merged_with_entrypoint(self): + parsed = self._parse('FROM ubuntu\nENTRYPOINT ["python3"]\nCMD ["app.py"]\n') + self.assertEqual(parsed.start_cmd, ("python3", "app.py")) + + def test_comment_skipped(self): + parsed = self._parse("FROM ubuntu:22.04\n# a comment\nRUN echo hi\n") + self.assertEqual(parsed.unsupported, ()) + self.assertFalse(any("COMMENT" in warning for warning in parsed.warnings)) + result = check_direct_launch( + LocalDockerContext("FROM ubuntu:22.04\n# a comment\nRUN echo hi\n") + ) + self.assertNotIn("COMMENT", result.ignored_instructions) + + def test_copy_multiple_sources(self): + parsed = self._parse("FROM ubuntu\nCOPY a.py b.py /app/\n") + ins = [i for i in parsed.instructions if isinstance(i, CopyInstruction)][0] + self.assertEqual(ins.srcs, ("a.py", "b.py")) + self.assertEqual(ins.dest, "/app/") + + def test_run_leading_flags_fail_closed(self): + for command in ( + "--mount=type=secret,id=x; touch /tmp/x", + "--network=none echo x", + "--security=insecure echo x", + "--device=nvidia.com/gpu=all echo x", + "--future-flag=enabled echo x", + ): + with self.subTest(command=command): + dockerfile = f"FROM ubuntu\nRUN {command}\n" + parsed = self._parse(dockerfile) + self.assertEqual(parsed.unsupported[0].reason, "unsupported_syntax") + self.assertFalse( + any( + isinstance(item, RunInstruction) for item in parsed.instructions + ) + ) + self.assertFalse( + check_direct_launch( + LocalDockerContext(dockerfile) + ).direct_launchable + ) + with self.assertRaises(DockerfileParseError): + self._parse(dockerfile, strict=True) + + def test_run_nonleading_double_dash_and_bracket_command_are_supported(self): + parsed = self._parse( + "FROM ubuntu\nRUN printf -- '%s' x\nRUN [ -f /x ]\n", strict=True + ) + commands = [ + item.command + for item in parsed.instructions + if isinstance(item, RunInstruction) + ] + self.assertEqual(commands, ["printf -- '%s' x", "[ -f /x ]"]) + + def test_run_malformed_quoting_fails_closed(self): + dockerfile = 'FROM ubuntu\nRUN echo "unterminated\n' + parsed = self._parse(dockerfile) + self.assertEqual(parsed.unsupported[0].kind, "RUN") + self.assertFalse( + any(isinstance(item, RunInstruction) for item in parsed.instructions) + ) + with self.assertRaisesRegex(DockerfileParseError, "malformed quoting"): + self._parse(dockerfile, strict=True) + + +class TestCheckDirectLaunch(unittest.TestCase): + def test_simple_direct_launchable(self): + r = check_direct_launch(LocalDockerContext("FROM ubuntu:22.04\nRUN echo hi\n")) + self.assertTrue(r.direct_launchable) + self.assertEqual(r.reasons, ()) + self.assertTrue(r.has_build_instructions) + self.assertEqual(r.base_image, "ubuntu:22.04") + self.assertIn(DIRECT_LAUNCH_ROOTFS_ONLY_WARNING, r.warnings) + + def test_no_build_instructions(self): + r = check_direct_launch(LocalDockerContext("FROM ubuntu\nENV K=v\n")) + self.assertTrue(r.direct_launchable) + self.assertFalse(r.has_build_instructions) + + def test_check_multi_stage_reason_unchanged(self): + r = check_direct_launch(LocalDockerContext("FROM ubuntu AS a\nFROM node:20\n")) + self.assertFalse(r.direct_launchable) + self.assertEqual(r.reasons, ("multi_stage",)) + + def test_no_from_not_launchable(self): + r = check_direct_launch(LocalDockerContext("RUN echo hi\n")) + self.assertFalse(r.direct_launchable) + self.assertEqual(r.reasons, ("no_from",)) + + def test_check_direct_launch_no_comment_warning(self): + r = check_direct_launch( + LocalDockerContext("FROM ubuntu:22.04\n# a comment\nRUN echo hi\n") + ) + self.assertEqual(r.ignored_instructions, ()) + self.assertFalse(any("COMMENT" in warning for warning in r.warnings)) + + def test_run_does_not_break_launchable(self): + r = check_direct_launch( + LocalDockerContext("FROM ubuntu\nRUN apt-get install -y curl\nCOPY x /y\n") + ) + self.assertTrue(r.direct_launchable) + self.assertTrue(r.has_build_instructions) + + def test_warnings_mention_no_snapshot(self): + r = check_direct_launch(LocalDockerContext("FROM ubuntu\nRUN echo hi\n")) + self.assertTrue(any("no snapshot" in w for w in r.warnings)) + + +class TestWrapUser(unittest.TestCase): + def test_no_user_passthrough(self): + self.assertEqual(wrap_user("whoami", None), "whoami") + self.assertEqual(wrap_user("whoami", "root"), "whoami") + self.assertEqual(wrap_user("whoami", "0"), "whoami") + + def test_user_wraps_runuser_with_su_fallback(self): + wrapped = wrap_user("whoami", "app") + self.assertIn("runuser", wrapped) + self.assertIn("su -s /bin/sh", wrapped) + self.assertIn("app", wrapped) + + def test_group_stripped(self): + wrapped = wrap_user("whoami", "app:grp") + # Should reference user "app" not "app:grp" + self.assertIn("app", wrapped) + self.assertNotIn("app:grp", wrapped) + + +class TestResolveStartCmd(unittest.TestCase): + def _parse(self, content): + return parse_dockerfile(LocalDockerContext(content)) + + def test_cmd_only(self): + parsed = self._parse('FROM ubuntu\nCMD ["a", "b"]\n') + self.assertEqual(_resolve_start_cmd(parsed), ("a", "b")) + + def test_entrypoint_only(self): + parsed = self._parse('FROM ubuntu\nENTRYPOINT ["python3"]\n') + self.assertEqual(_resolve_start_cmd(parsed), ("python3",)) + + def test_entrypoint_and_cmd_concat(self): + parsed = self._parse('FROM ubuntu\nENTRYPOINT ["python3"]\nCMD ["app.py"]\n') + self.assertEqual(_resolve_start_cmd(parsed), ("python3", "app.py")) + + def test_no_cmd_no_entrypoint(self): + parsed = self._parse("FROM ubuntu\nRUN echo hi\n") + self.assertIsNone(_resolve_start_cmd(parsed)) + + +class TestApplyDockerfile(unittest.TestCase): + def _apply(self, dockerfile, sb, auto_start_cmd=False, run_timeout=60): + ctx = LocalDockerContext(dockerfile) + parsed = parse_dockerfile(ctx) + return apply_dockerfile( + sb, + parsed, + ctx, + auto_start_cmd=auto_start_cmd, + run_timeout=run_timeout, + ) + + def test_run_uses_accumulated_envs_cwd_user(self): + sb = _MockSandbox() + self._apply( + "FROM ubuntu\nENV K=v\nWORKDIR /app\nUSER app\nRUN whoami\n", + sb, + auto_start_cmd=False, + ) + run_ops = [o for o in sb.commands.ops if o[0] == "run"] + # whoami should be wrapped with runuser + carry envs and cwd + last_run = run_ops[-1] + self.assertIn("runuser", last_run[1]) + self.assertIn("app", last_run[1]) + self.assertEqual(last_run[3], {"K": "v"}) # envs + self.assertEqual(last_run[4], "/app") # cwd + # make_dir called for WORKDIR + self.assertIn(("mkdir", "/app"), sb.files.ops) + + def test_run_failure_raises_build_error(self): + sb = _MockSandbox(failing_cmd="whoami") + with self.assertRaises(DockerfileBuildError) as cm: + self._apply("FROM ubuntu\nRUN whoami\n", sb, auto_start_cmd=False) + self.assertIn("exit code", str(cm.exception)) + + def test_auto_start_launches_background(self): + sb = _MockSandbox() + res = self._apply( + 'FROM ubuntu\nRUN echo hi\nCMD ["python3", "app.py"]\n', + sb, + auto_start_cmd=True, + ) + bg_ops = [o for o in sb.commands.ops if o[0] == "run" and o[2] is True] + self.assertEqual(len(bg_ops), 1) + self.assertEqual(res.start_cmd, ("python3", "app.py")) + self.assertIs(res.startup_command, sb.commands.startup_handle) + self.assertEqual(sb.is_running_calls, 1) + + def test_no_auto_start_returns_cmd(self): + sb = _MockSandbox() + res = self._apply( + 'FROM ubuntu\nRUN echo hi\nCMD ["python3", "app.py"]\n', + sb, + auto_start_cmd=False, + ) + bg_ops = [o for o in sb.commands.ops if o[0] == "run" and o[2] is True] + self.assertEqual(len(bg_ops), 0) + self.assertEqual(res.start_cmd, ("python3", "app.py")) + self.assertIsNone(res.startup_command) + + def test_no_start_command_returns_no_startup_handle(self): + sb = _MockSandbox() + res = self._apply("FROM ubuntu\nRUN echo hi\n", sb, auto_start_cmd=True) + self.assertIsNone(res.start_cmd) + self.assertIsNone(res.startup_command) + self.assertEqual(sb.is_running_calls, 0) + + def test_startup_dispatch_failure_includes_instruction_metadata(self): + startup_error = RuntimeError("start RPC failed") + sb = _MockSandbox(startup_error=startup_error) + with self.assertRaises(DockerfileBuildError) as raised: + self._apply('FROM ubuntu\nCMD ["server"]\n', sb, auto_start_cmd=True) + self.assertEqual(raised.exception.instruction, "CMD") + self.assertIs(raised.exception.__cause__, startup_error) + + def test_workdir_make_dir(self): + sb = _MockSandbox() + self._apply("FROM ubuntu\nWORKDIR /srv/app\nRUN pwd\n", sb) + self.assertIn(("mkdir", "/srv/app"), sb.files.ops) + + def test_copy_chown_runs_chown(self): + import os + import tempfile + + with tempfile.TemporaryDirectory() as d: + with open(os.path.join(d, "app.py"), "w") as _f: + _f.write("print(1)") + sb = _MockSandbox() + ctx = LocalDockerContext( + "FROM ubuntu\nCOPY --chown=app:app app.py /app/\n", + context_dir=d, + ) + parsed = parse_dockerfile(ctx) + apply_dockerfile(sb, parsed, ctx, auto_start_cmd=False) + # Should have: copy_from_local + a chown command run + cp_ops = [o for o in sb.files.ops if o[0] == "cp"] + self.assertTrue(cp_ops) + run_ops = [o for o in sb.commands.ops if o[0] == "run"] + chown_ops = [o for o in run_ops if "chown" in o[1]] + self.assertTrue(chown_ops) + self.assertIn("chown app:app /app/app.py", chown_ops[0][1]) + + def test_copy_to_dir_dest_appends_basename(self): + import os + import tempfile + + with tempfile.TemporaryDirectory() as d: + with open(os.path.join(d, "app.py"), "w") as _f: + _f.write("print(1)") + sb = _MockSandbox() + ctx = LocalDockerContext("FROM ubuntu\nCOPY app.py /srv/\n", context_dir=d) + parsed = parse_dockerfile(ctx) + apply_dockerfile(sb, parsed, ctx, auto_start_cmd=False) + cp_ops = [o for o in sb.files.ops if o[0] == "cp"] + # dir-form dest (trailing /) places the file inside by basename + self.assertEqual(cp_ops[-1][2], "/srv/app.py") + + def test_add_tar_creates_dest_dir_before_extract(self): + import os + import tempfile + + with tempfile.TemporaryDirectory() as d: + import tarfile + + archive = os.path.join(d, "app.tar.gz") + with tarfile.open(archive, "w:gz") as tar: + inner = os.path.join(d, "top.txt") + with open(inner, "w") as _f: + _f.write("x") + tar.add(inner, arcname="top.txt") + sb = _MockSandbox() + ctx = LocalDockerContext( + "FROM ubuntu\nUSER app\nADD app.tar.gz /opt/app/\n", + context_dir=d, + ) + parsed = parse_dockerfile(ctx) + apply_dockerfile(sb, parsed, ctx, auto_start_cmd=False) + # dest dir created before extraction + self.assertTrue( + any( + o[0] == "mkdir" and o[1].rstrip("/") == "/opt/app" + for o in sb.files.ops + ), + sb.files.ops, + ) + # tar extraction command was issued + run_ops = [o for o in sb.commands.ops if o[0] == "run"] + self.assertTrue( + any("tar xf" in o[1] and "/opt/app" in o[1] for o in run_ops) + ) + self.assertTrue(any("--no-same-owner" in o[1] for o in run_ops)) + self.assertTrue( + any("tar xf" in o[1] and "runuser" in o[1] for o in run_ops) + ) + + def test_add_tar_rejects_path_traversal(self): + import io + import os + import tarfile + import tempfile + + with tempfile.TemporaryDirectory() as d: + archive = os.path.join(d, "unsafe.tar") + with tarfile.open(archive, "w") as tar: + content = b"evil" + info = tarfile.TarInfo("../evil.txt") + info.size = len(content) + tar.addfile(info, io.BytesIO(content)) + sb = _MockSandbox() + ctx = LocalDockerContext( + "FROM ubuntu\nADD unsafe.tar /opt/app/\n", context_dir=d + ) + parsed = parse_dockerfile(ctx) + with self.assertRaisesRegex(DockerfileBuildError, "unsafe"): + apply_dockerfile(sb, parsed, ctx, auto_start_cmd=False) + self.assertEqual(sb.files.ops, []) + self.assertEqual(sb.commands.ops, []) + + +class TestDockerfileManifestCopies(unittest.TestCase): + def _apply_memory( + self, + dockerfile: str, + files: dict[str, bytes], + *, + fail_path: str | None = None, + existing_paths: set[str] | None = None, + ) -> tuple[_MockSandbox, _MemoryDockerContext]: + context = _MemoryDockerContext(dockerfile, files, fail_path=fail_path) + sandbox = _MockSandbox(existing_paths=existing_paths) + apply_dockerfile( + sandbox, + parse_dockerfile(context), + context, + auto_start_cmd=False, + ) + return sandbox, context + + def test_dot_honors_dockerignore_and_skips_reserved_files(self) -> None: + files = { + ".dockerignore": b".env\n.git/**\n", + "Dockerfile": b"FROM scratch\n", + ".env": b"secret", + ".git/config": b"git", + "allowed": b"ok", + "sub/x": b"x", + } + sandbox, context = self._apply_memory("FROM ubuntu\nCOPY . /app/\n", files) + copies = [ + operation[2] for operation in sandbox.files.ops if operation[0] == "cp" + ] + self.assertEqual(copies, ["/app/allowed", "/app/sub/x"]) + self.assertEqual(context.open_paths, [".dockerignore", "allowed", "sub/x"]) + + def test_local_and_memory_contexts_have_same_copy_targets(self) -> None: + import os + import tempfile + + dockerfile = "FROM ubuntu\nCOPY src /app\n" + memory_sandbox, _ = self._apply_memory( + dockerfile, {"src/a.py": b"a", "src/sub/b.py": b"b"} + ) + with tempfile.TemporaryDirectory() as directory: + os.makedirs(os.path.join(directory, "src", "sub")) + with open(os.path.join(directory, "src", "a.py"), "wb") as output: + output.write(b"a") + with open(os.path.join(directory, "src", "sub", "b.py"), "wb") as output: + output.write(b"b") + context = LocalDockerContext(dockerfile, context_dir=directory) + local_sandbox = _MockSandbox() + apply_dockerfile( + local_sandbox, + parse_dockerfile(context), + context, + auto_start_cmd=False, + ) + memory_targets = [op[2] for op in memory_sandbox.files.ops if op[0] == "cp"] + local_targets = [op[2] for op in local_sandbox.files.ops if op[0] == "cp"] + self.assertEqual(local_targets, memory_targets) + + def test_literal_directory_dot_and_wildcard_targets(self) -> None: + files = { + "root.py": b"root", + "src/a.py": b"a", + "src/sub/b.py": b"b", + "other.txt": b"other", + } + cases = ( + ("COPY src/a.py /one/\n", ["/one/a.py"]), + ("COPY src /two\n", ["/two/a.py", "/two/sub/b.py"]), + ( + "COPY . /three/\n", + [ + "/three/other.txt", + "/three/root.py", + "/three/src/a.py", + "/three/src/sub/b.py", + ], + ), + ("COPY *.py /four/\n", ["/four/root.py"]), + ("COPY src/**/*.py /five/\n", ["/five/a.py", "/five/b.py"]), + ) + for instruction, expected in cases: + with self.subTest(instruction=instruction): + sandbox, _ = self._apply_memory("FROM ubuntu\n" + instruction, files) + copies = [op[2] for op in sandbox.files.ops if op[0] == "cp"] + self.assertEqual(copies, expected) + + def test_selection_and_target_errors_have_no_sandbox_operations(self) -> None: + cases = ( + ("COPY no-match /dest/\n", {"ok": b""}), + ( + "COPY hidden.py /dest/\n", + {".dockerignore": b"hidden.py\n", "hidden.py": b""}, + ), + ("COPY /absolute /dest/\n", {"absolute": b""}), + ("COPY ../parent /dest/\n", {"parent": b""}), + ("COPY *.py /dest\n", {"a.py": b"", "b.py": b""}), + ("COPY one/a two/a /dest/\n", {"one/a": b"", "two/a": b""}), + ("COPY . /dest/\n", {}), + ( + "COPY . /dest/\n", + {".dockerignore": b"*.txt\n", "hidden.txt": b""}, + ), + ("COPY . /dest/\n", {"a": b"", "a-foo": b"", "a/b": b""}), + ) + for instruction, files in cases: + with self.subTest(instruction=instruction): + context = _MemoryDockerContext("FROM ubuntu\n" + instruction, files) + sandbox = _MockSandbox() + with self.assertRaises(DockerfileBuildError) as raised: + apply_dockerfile( + sandbox, + parse_dockerfile(context), + context, + auto_start_cmd=False, + ) + self.assertEqual(raised.exception.index, 1) + self.assertEqual(raised.exception.instruction, "COPY") + self.assertEqual(sandbox.files.ops, []) + self.assertEqual(sandbox.commands.ops, []) + if ".dockerignore" not in files: + self.assertEqual(context.open_paths, []) + + def test_empty_copy_plan_has_no_sandbox_operations(self) -> None: + sandbox = _MockSandbox() + runner = _Runner( + sandbox, + _MemoryDockerContext("FROM ubuntu\n", {"visible": b""}), + run_timeout=60, + ) + with self.assertRaisesRegex(DockerfileBuildError, "select no files"): + runner._validate_copy_plan([], index=1, instruction="COPY") + self.assertEqual(sandbox.files.ops, []) + self.assertEqual(sandbox.commands.ops, []) + + def test_context_open_failure_has_no_sandbox_operations(self) -> None: + context = _MemoryDockerContext( + "FROM ubuntu\nCOPY a b /dest/\n", + {"a": b"a", "b": b"b"}, + fail_path="b", + ) + sandbox = _MockSandbox() + with self.assertRaisesRegex( + DockerfileBuildError, "source 'b'.*cannot open" + ) as raised: + apply_dockerfile( + sandbox, parse_dockerfile(context), context, auto_start_cmd=False + ) + self.assertEqual(raised.exception.index, 1) + self.assertEqual(raised.exception.instruction, "COPY") + self.assertEqual(context.open_paths, ["a", "b"]) + self.assertEqual(sandbox.files.ops, []) + self.assertEqual(sandbox.commands.ops, []) + + def test_later_missing_copy_prevents_earlier_run(self) -> None: + context = _MemoryDockerContext( + "FROM ubuntu\nRUN echo side-effect\nCOPY missing /dst/\n", {} + ) + sandbox = _MockSandbox() + with self.assertRaises(DockerfileBuildError) as raised: + apply_dockerfile( + sandbox, parse_dockerfile(context), context, auto_start_cmd=False + ) + self.assertEqual(raised.exception.index, 2) + self.assertEqual(raised.exception.instruction, "COPY") + self.assertEqual(context.open_paths, []) + self.assertEqual(sandbox.files.ops, []) + self.assertEqual(sandbox.commands.ops, []) + + def test_later_copy_open_failure_prevents_earlier_run(self) -> None: + context = _MemoryDockerContext( + "FROM ubuntu\nRUN echo side-effect\nCOPY a /one/\nCOPY b /two/\n", + {"a": b"a", "b": b"b"}, + fail_path="b", + ) + sandbox = _MockSandbox() + with self.assertRaises(DockerfileBuildError) as raised: + apply_dockerfile( + sandbox, parse_dockerfile(context), context, auto_start_cmd=False + ) + self.assertEqual(raised.exception.index, 3) + self.assertEqual(raised.exception.instruction, "COPY") + self.assertEqual(context.open_paths, ["a", "b"]) + self.assertEqual(sandbox.files.ops, []) + self.assertEqual(sandbox.commands.ops, []) + + def test_all_sources_materialize_before_first_sandbox_operation(self) -> None: + context = _MemoryDockerContext( + "FROM ubuntu\nRUN echo side-effect\nCOPY a /one/\nCOPY b /two/\n", + {"a": b"a", "b": b"b"}, + ) + sandbox = _MockSandbox() + original_open = context.open + events: list[str] = [] + original_run = sandbox.commands.run + original_copy = sandbox.files.copy_from_local + + def traced_run(*args, **kwargs): + events.append("run") + return original_run(*args, **kwargs) + + def traced_copy(local: str, remote: str) -> None: + events.append(f"copy:{remote}") + original_copy(local, remote) + + sandbox.commands.run = traced_run # type: ignore[method-assign] + sandbox.files.copy_from_local = traced_copy # type: ignore[method-assign] + + @contextmanager + def checked_open(path: str) -> Iterator[BinaryIO]: + self.assertEqual(sandbox.files.ops, []) + self.assertEqual(sandbox.commands.ops, []) + with original_open(path) as stream: + yield stream + + context.open = checked_open # type: ignore[method-assign] + apply_dockerfile( + sandbox, parse_dockerfile(context), context, auto_start_cmd=False + ) + self.assertEqual(context.open_paths, ["a", "b"]) + self.assertEqual( + [operation[2] for operation in sandbox.files.ops if operation[0] == "cp"], + ["/one/a", "/two/b"], + ) + self.assertEqual(events, ["run", "copy:/one/a", "copy:/two/b"]) + + def test_manifest_failure_is_wrapped_before_sandbox_operations(self) -> None: + class BrokenContext(_MemoryDockerContext): + def walk(self) -> Iterator[str]: + raise OSError("walk failed") + yield "unreachable" + + context = BrokenContext("FROM ubuntu\nRUN echo never\nCOPY a /dest/\n", {}) + sandbox = _MockSandbox() + parsed = parse_dockerfile(context) + with self.assertRaisesRegex( + DockerfileBuildError, "Docker context manifest.*BrokenContext.*walk failed" + ): + apply_dockerfile(sandbox, parsed, context, auto_start_cmd=False) + self.assertEqual(sandbox.files.ops, []) + self.assertEqual(sandbox.commands.ops, []) + + def test_relative_destination_resolves_against_workdir(self) -> None: + sandbox, _ = self._apply_memory( + "FROM ubuntu\nWORKDIR /work\nCOPY app.py output/\n", + {"app.py": b"app"}, + ) + copies = [ + operation[2] for operation in sandbox.files.ops if operation[0] == "cp" + ] + self.assertEqual(copies, ["/work/output/app.py"]) + + def test_workdir_copy_is_prepared_before_workdir_operation(self) -> None: + context = _MemoryDockerContext( + "FROM ubuntu\nWORKDIR /work\nCOPY app.py output/\n", {"app.py": b"app"} + ) + sandbox = _MockSandbox() + original_open = context.open + + @contextmanager + def checked_open(path: str) -> Iterator[BinaryIO]: + self.assertEqual(sandbox.files.ops, []) + self.assertEqual(sandbox.commands.ops, []) + with original_open(path) as stream: + yield stream + + context.open = checked_open # type: ignore[method-assign] + apply_dockerfile( + sandbox, parse_dockerfile(context), context, auto_start_cmd=False + ) + self.assertEqual(context.open_paths, ["app.py"]) + self.assertEqual(sandbox.files.ops[0], ("mkdir", "/work")) + self.assertEqual(sandbox.files.ops[-1][2], "/work/output/app.py") + + def test_add_extracts_only_explicit_literal_tar(self) -> None: + archive = _archive_bytes([_regular_member("app.txt")]) + literal, _ = self._apply_memory( + "FROM ubuntu\nADD app.tar /out/\n", {"app.tar": archive} + ) + self.assertTrue(any("tar xf" in op[1] for op in literal.commands.ops)) + + wildcard, _ = self._apply_memory( + "FROM ubuntu\nADD *.tar /out/\n", {"app.tar": b"tar"} + ) + self.assertEqual( + [op[2] for op in wildcard.files.ops if op[0] == "cp"], ["/out/app.tar"] + ) + self.assertFalse(any("tar xf" in op[1] for op in wildcard.commands.ops)) + + directory, _ = self._apply_memory( + "FROM ubuntu\nADD src /out/\n", {"src/app.tar": b"tar"} + ) + self.assertEqual( + [op[2] for op in directory.files.ops if op[0] == "cp"], ["/out/app.tar"] + ) + self.assertFalse(any("tar xf" in op[1] for op in directory.commands.ops)) + + def test_directory_copy_chown_skips_existing_destination(self) -> None: + sandbox, _ = self._apply_memory( + "FROM ubuntu\nCOPY --chown=app:app src /app\n", + {"src/a": b"a", "src/sub/b": b"b"}, + existing_paths={"/app"}, + ) + chown = [op[1] for op in sandbox.commands.ops if "chown" in op[1]] + self.assertEqual( + chown, + [ + "chown app:app /app/a", + "chown app:app /app/sub/b", + "chown app:app /app/sub", + ], + ) + self.assertEqual(sandbox.files.exists_calls, ["/app", "/app/sub"]) + self.assertNotIn("chown app:app /app", chown) + self.assertFalse(any("-R" in command for command in chown)) + + def test_directory_copy_chown_includes_only_new_parent_directories(self) -> None: + sandbox, _ = self._apply_memory( + "FROM ubuntu\nCOPY --chown=app:app src /app\n", + {"src/a": b"a", "src/sub/b": b"b"}, + ) + chown = [op[1] for op in sandbox.commands.ops if "chown" in op[1]] + self.assertEqual( + set(chown), + { + "chown app:app /app/a", + "chown app:app /app/sub/b", + "chown app:app /app", + "chown app:app /app/sub", + }, + ) + self.assertFalse(any("-R" in command for command in chown)) + + def test_multiple_copy_chown_uses_exact_file_targets(self) -> None: + sandbox, _ = self._apply_memory( + "FROM ubuntu\nCOPY --chown=app:app a b /app/\n", + {"a": b"a", "b": b"b"}, + existing_paths={"/app"}, + ) + chown = [op[1] for op in sandbox.commands.ops if "chown" in op[1]] + self.assertEqual( + chown, + ["chown app:app /app/a", "chown app:app /app/b"], + ) + + def test_tar_chown_skips_existing_destination_and_uses_exact_targets(self) -> None: + archive = _archive_bytes([_regular_member("a"), _regular_member("sub/b")]) + sandbox, _ = self._apply_memory( + "FROM ubuntu\nADD --chown=app:app app.tar /app/\n", + {"app.tar": archive}, + existing_paths={"/app"}, + ) + chown = [op[1] for op in sandbox.commands.ops if "chown" in op[1]] + self.assertEqual( + chown, + [ + "chown app:app /app/a", + "chown app:app /app/sub/b", + "chown app:app /app/sub", + ], + ) + self.assertNotIn("chown app:app /app", chown) + self.assertFalse(any("-R" in command for command in chown)) + + def test_invalid_tar_members_fail_before_sandbox_operations(self) -> None: + symlink = tarfile.TarInfo("link") + symlink.type = tarfile.SYMTYPE + symlink.linkname = "target" + hardlink = tarfile.TarInfo("hard") + hardlink.type = tarfile.LNKTYPE + hardlink.linkname = "target" + fifo = tarfile.TarInfo("pipe") + fifo.type = tarfile.FIFOTYPE + device = tarfile.TarInfo("device") + device.type = tarfile.CHRTYPE + cases = { + "symlink": [(symlink, None)], + "hardlink": [(hardlink, None)], + "fifo": [(fifo, None)], + "device": [(device, None)], + "absolute": [_regular_member("/absolute")], + "control": [_regular_member("bad\x01name")], + "backslash": [_regular_member(r"dir\file")], + "traversal": [_regular_member("../traversal")], + "duplicate": [_regular_member("same"), _regular_member("same")], + "file ancestor": [_regular_member("file"), _regular_member("file/child")], + } + for name, members in cases.items(): + with self.subTest(name=name): + sandbox = _MockSandbox() + context = _MemoryDockerContext( + "FROM ubuntu\nADD invalid.tar /app/\n", + {"invalid.tar": _archive_bytes(members)}, + ) + with self.assertRaises(DockerfileBuildError) as raised: + apply_dockerfile( + sandbox, + parse_dockerfile(context), + context, + auto_start_cmd=False, + ) + self.assertEqual(raised.exception.index, 1) + self.assertEqual(raised.exception.instruction, "ADD") + self.assertEqual(sandbox.files.ops, []) + self.assertEqual(sandbox.commands.ops, []) + + def test_tar_symlink_precheck_prevents_write_operations(self) -> None: + sandbox = _MockSandbox(failing_cmd="test ! -L") + context = _MemoryDockerContext( + "FROM ubuntu\nADD --chown=app:app app.tar /app/\n", + {"app.tar": _archive_bytes([_regular_member("app.txt")])}, + ) + with self.assertRaisesRegex(DockerfileBuildError, "symlink"): + apply_dockerfile( + sandbox, + parse_dockerfile(context), + context, + auto_start_cmd=False, + ) + self.assertEqual(sandbox.files.ops, []) + self.assertEqual(len(sandbox.commands.ops), 1) + self.assertIn("test ! -L", sandbox.commands.ops[0][1]) + self.assertFalse(any("tar xf" in op[1] for op in sandbox.commands.ops)) + self.assertFalse(any("chown" in op[1] for op in sandbox.commands.ops)) + + +class TestLocalDockerContext(unittest.TestCase): + def test_content_string(self): + ctx = LocalDockerContext("FROM ubuntu\n") + self.assertEqual(ctx.dockerfile_text(), "FROM ubuntu\n") + + def test_file_path( + self, + ): + import os + import tempfile + + with tempfile.TemporaryDirectory() as d: + p = os.path.join(d, "Dockerfile") + with open(p, "w") as _f: + _f.write("FROM node:20\n") + ctx = LocalDockerContext(p) + self.assertEqual(ctx.dockerfile_text(), "FROM node:20\n") + self.assertEqual(ctx.context_dir, os.path.abspath(d)) + + def test_walk_excludes_dockerfile(self): + import os + import tempfile + + with tempfile.TemporaryDirectory() as d: + with open(os.path.join(d, "Dockerfile"), "w") as _f: + _f.write("FROM ubuntu\n") + with open(os.path.join(d, "app.py"), "w") as _f: + _f.write("print(1)") + ctx = LocalDockerContext("FROM ubuntu\n", context_dir=d) + self.assertEqual(list(ctx.walk()), ["app.py"]) + + def test_open_path_escape_rejected(self): + import tempfile + + with tempfile.TemporaryDirectory() as d: + ctx = LocalDockerContext("FROM ubuntu\n", context_dir=d) + with self.assertRaises(DockerContextError): + with ctx.open("../x"): + pass + + def test_open_path_inside_context(self): + import os + import tempfile + + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "app.py") + with open(path, "wb") as handle: + handle.write(b"print(1)") + ctx = LocalDockerContext("FROM ubuntu\n", context_dir=d) + with ctx.open("app.py") as handle: + self.assertEqual(handle.read(), b"print(1)") + + +class TestStrictDockerfileSemantics(unittest.TestCase): + def _parse(self, dockerfile, strict=False): + return parse_dockerfile(LocalDockerContext(dockerfile), strict=strict) + + def test_remote_add_is_never_executable(self): + dockerfile = "FROM ubuntu\nADD https://example.test/app.tar /opt/\n" + parsed = self._parse(dockerfile) + self.assertEqual(parsed.unsupported[0].kind, "ADD") + self.assertEqual(parsed.unsupported[0].reason, "remote_add") + self.assertFalse( + any(isinstance(item, CopyInstruction) for item in parsed.instructions) + ) + with self.assertRaisesRegex( + DockerfileParseError, "download into the build context" + ): + self._parse(dockerfile, strict=True) + result = check_direct_launch(LocalDockerContext(dockerfile)) + self.assertFalse(result.direct_launchable) + self.assertEqual(result.reasons, ("remote_add",)) + + def test_apply_rejects_unsupported_before_any_operation(self): + context = LocalDockerContext("FROM ubuntu\nADD http://example.test/a /opt/\n") + parsed = parse_dockerfile(context) + sandbox = _MockSandbox() + with self.assertRaisesRegex(DockerfileBuildError, "unsupported instruction"): + apply_dockerfile(sandbox, parsed, context) + self.assertEqual(sandbox.files.ops, []) + self.assertEqual(sandbox.commands.ops, []) + + def test_supported_shell_run_absolute_workdir_and_chown(self): + parsed = self._parse( + "FROM ubuntu\nRUN echo hi\nWORKDIR /app\nCOPY --chown=app:app x /app/\n", + strict=True, + ) + self.assertFalse(parsed.unsupported) + self.assertEqual(parsed.workdir, "/app") + copy = next( + item for item in parsed.instructions if isinstance(item, CopyInstruction) + ) + self.assertEqual(copy.chown, "app:app") + + def test_unsupported_syntax_is_not_executable_and_not_launchable(self): + cases = { + "json COPY": 'COPY ["a b", "/dest/"]', + "json ADD": 'ADD ["a b", "/dest/"]', + "exec RUN": 'RUN ["printf", "%s", "x"]', + "relative WORKDIR": "WORKDIR app", + "ARG": "ARG VERSION=1", + "chmod": "COPY --chmod=755 a /dest/", + "link": "COPY --link a /dest/", + "unknown flag": "COPY --parents a /dest/", + "FROM platform": "FROM --platform=linux/amd64 ubuntu", + } + for name, instruction in cases.items(): + with self.subTest(name=name): + dockerfile = ( + instruction + if instruction.startswith("FROM") + else f"FROM ubuntu\n{instruction}\n" + ) + parsed = self._parse(dockerfile) + self.assertTrue(parsed.unsupported) + self.assertFalse( + any( + isinstance(item, (CopyInstruction, RunInstruction)) + for item in parsed.instructions + ) + ) + self.assertFalse( + check_direct_launch( + LocalDockerContext(dockerfile) + ).direct_launchable + ) + with self.assertRaises(DockerfileParseError): + self._parse(dockerfile, strict=True) + + def test_ignored_and_unknown_instructions_fail_closed(self): + for kind, value in ( + ("VOLUME", "/data"), + ("LABEL", "k=v"), + ("HEALTHCHECK", "CMD true"), + ("SHELL", '["/bin/bash", "-c"]'), + ("STOPSIGNAL", "SIGTERM"), + ("ONBUILD", "RUN echo x"), + ("MAINTAINER", "someone"), + ("FUTURE", "value"), + ): + with self.subTest(kind=kind): + dockerfile = f"FROM ubuntu\n{kind} {value}\n" + self.assertEqual(self._parse(dockerfile).unsupported[0].kind, kind) + result = check_direct_launch(LocalDockerContext(dockerfile)) + self.assertFalse(result.direct_launchable) + self.assertEqual(result.reasons, ("unsupported_instruction",)) + with self.assertRaises(DockerfileParseError): + self._parse(dockerfile, strict=True) + + def test_malformed_quotes_do_not_fall_back_to_whitespace_splitting(self): + dockerfile = 'FROM ubuntu\nCOPY "unterminated /dest/\n' + parsed = self._parse(dockerfile) + self.assertEqual(parsed.unsupported[0].kind, "COPY") + self.assertFalse( + any(isinstance(item, CopyInstruction) for item in parsed.instructions) + ) + self.assertFalse( + check_direct_launch(LocalDockerContext(dockerfile)).direct_launchable + ) + with self.assertRaisesRegex(DockerfileParseError, "malformed quoting"): + self._parse(dockerfile, strict=True) + + def test_build_time_variable_expansion_fails_closed(self): + cases = ( + "FROM ${IMAGE}", + "FROM ubuntu\nWORKDIR /app/$NAME", + "FROM ubuntu\nCOPY $SRC /app/", + "FROM ubuntu\nADD ${SRC} /app/", + "FROM ubuntu\nUSER $USER", + "FROM ubuntu\nENV K=$VALUE", + ) + for dockerfile in cases: + with self.subTest(dockerfile=dockerfile): + self.assertFalse( + check_direct_launch( + LocalDockerContext(dockerfile) + ).direct_launchable + ) + with self.assertRaises(DockerfileParseError): + self._parse(dockerfile, strict=True) + + def test_copy_from_uses_multi_stage_reason(self): + dockerfile = "FROM ubuntu\nCOPY --from=build x /y\n" + result = check_direct_launch(LocalDockerContext(dockerfile)) + self.assertFalse(result.direct_launchable) + self.assertEqual(result.reasons, ("multi_stage",)) + with self.assertRaises(DockerfileParseError): + self._parse(dockerfile, strict=True) + + def test_cmd_and_entrypoint_resolve_to_executable_argv(self): + cases = ( + ('CMD ["a", "b"]', ("a", "b")), + ("CMD echo hello", ("/bin/sh", "-c", "echo hello")), + ('ENTRYPOINT ["python3"]', ("python3",)), + ('ENTRYPOINT ["python3"]\nCMD ["app.py"]', ("python3", "app.py")), + ( + 'ENTRYPOINT ["python3"]\nCMD echo hello', + ("python3", "/bin/sh", "-c", "echo hello"), + ), + ( + 'ENTRYPOINT echo entry\nCMD ["ignored"]', + ("/bin/sh", "-c", "echo entry"), + ), + ) + for body, expected in cases: + with self.subTest(body=body): + parsed = self._parse(f"FROM ubuntu\n{body}\n", strict=True) + self.assertEqual(parsed.start_cmd, expected) + shell_entrypoint = self._parse( + 'FROM ubuntu\nENTRYPOINT echo entry\nCMD ["ignored"]\n', strict=True + ) + self.assertEqual(shell_entrypoint.entrypoint, ("/bin/sh", "-c", "echo entry")) + + def test_json_array_detection_requires_a_complete_json_array(self): + self.assertEqual(_json_array('["echo", "ok"]'), ["echo", "ok"]) + self.assertIsNone(_json_array("[ -f /tmp/x ] && echo yes")) + self.assertIsNone(_json_array("[file /dest/")) + + def test_bracket_prefixed_shell_forms_are_supported(self): + run = self._parse("FROM ubuntu\nRUN [ -f /tmp/x ] && echo yes\n", strict=True) + instruction = next( + item for item in run.instructions if isinstance(item, RunInstruction) + ) + self.assertEqual(instruction.command, "[ -f /tmp/x ] && echo yes") + + cmd = self._parse("FROM ubuntu\nCMD [ -f /tmp/x ] && echo yes\n", strict=True) + self.assertEqual( + cmd.start_cmd, + ("/bin/sh", "-c", "[ -f /tmp/x ] && echo yes"), + ) + + copy = self._parse("FROM ubuntu\nCOPY [file /dest/\n", strict=True) + copy_instruction = next( + item for item in copy.instructions if isinstance(item, CopyInstruction) + ) + self.assertEqual(copy_instruction.srcs, ("[file",)) + self.assertEqual(copy_instruction.dest, "/dest/") + + def test_legacy_env_uses_shlex_processed_value(self): + parsed = self._parse('FROM ubuntu\nENV FOO "bar baz"\n', strict=True) + self.assertEqual(parsed.envs, {"FOO": "bar baz"}) + + empty_assignment = self._parse("FROM ubuntu\nENV FOO=\n", strict=True) + self.assertEqual(empty_assignment.envs, {"FOO": ""}) + + def test_env_without_value_is_unsupported(self): + dockerfile = "FROM ubuntu\nENV FOO\n" + result = check_direct_launch(LocalDockerContext(dockerfile)) + self.assertFalse(result.direct_launchable) + self.assertEqual(result.reasons, ("unsupported_syntax",)) + with self.assertRaisesRegex(DockerfileParseError, "requires a key and value"): + self._parse(dockerfile, strict=True) + + def test_multi_source_copy_destination_must_end_in_slash(self): + rejected = "FROM ubuntu\nCOPY a b /dest\n" + result = check_direct_launch(LocalDockerContext(rejected)) + self.assertFalse(result.direct_launchable) + self.assertEqual(result.reasons, ("unsupported_syntax",)) + with self.assertRaisesRegex( + DockerfileParseError, "destination must be a directory" + ): + self._parse(rejected, strict=True) + + parsed = self._parse("FROM ubuntu\nCOPY a b /dest/\n", strict=True) + instruction = next( + item for item in parsed.instructions if isinstance(item, CopyInstruction) + ) + self.assertEqual(instruction.srcs, ("a", "b")) + self.assertEqual(instruction.dest, "/dest/") + + def test_empty_cmd_and_entrypoint_are_unsupported(self): + cases = ( + "CMD []", + "ENTRYPOINT []", + 'CMD [""]', + 'ENTRYPOINT [""]', + "CMD", + "ENTRYPOINT", + ) + for instruction in cases: + with self.subTest(instruction=instruction): + dockerfile = f"FROM ubuntu\n{instruction}\n" + result = check_direct_launch(LocalDockerContext(dockerfile)) + self.assertFalse(result.direct_launchable) + self.assertEqual(result.reasons, ("unsupported_syntax",)) + with self.assertRaises(DockerfileParseError): + self._parse(dockerfile, strict=True) + + def test_launcher_uses_shlex_join_for_single_exec_argument_with_spaces(self): + context = LocalDockerContext('FROM ubuntu\nCMD ["hello world"]\n') + sandbox = _MockSandbox() + result = apply_dockerfile( + sandbox, + parse_dockerfile(context, strict=True), + context, + auto_start_cmd=True, + ) + self.assertEqual(result.start_cmd, ("hello world",)) + background = next( + operation for operation in sandbox.commands.ops if operation[2] + ) + self.assertEqual(background[1], "'hello world'") + + +if __name__ == "__main__": + unittest.main() diff --git a/sdk/python/tests/unit/test_sandbox.py b/sdk/python/tests/unit/test_sandbox.py index 3aec282..8a4ee1e 100644 --- a/sdk/python/tests/unit/test_sandbox.py +++ b/sdk/python/tests/unit/test_sandbox.py @@ -18,6 +18,9 @@ from akernel_sdk import HttpReverseTunnel, NetworkPolicy, S3Config, Sandbox from akernel_sdk import sandbox as sandbox_module +from akernel_sdk._dockercontext import LocalDockerContext +from akernel_sdk._dockerfile import DockerfileBuildError, DockerfileParseError +from akernel_sdk._dockerfile_runner import DockerfileApplyResult from akernel_sdk.types import SandboxInfo @@ -54,6 +57,7 @@ def test_default_constructor_and_info(self): self.assertEqual(sandbox.get_info().cpu, 2000) self.assertIsNone(sandbox.get_info().xpu) self.assertIsNone(sandbox.get_info().storage_mb) + self.assertIsNone(sandbox.startup_command) spec = self.backend.create.call_args.args[0] self.assertEqual(spec.cpu, 2000) @@ -320,6 +324,127 @@ def test_partial_facade_cleanup_preserves_initialization_error(self): self.session.terminate.assert_called_once_with() self.session.close.assert_called_once_with() + def test_dockerfile_context_is_strict_before_backend_creation(self): + unsupported = ( + "ADD https://example.test/a /opt/", + 'COPY ["a b", "/dest/"]', + 'ADD ["a b", "/dest/"]', + 'RUN ["printf", "%s", "x"]', + "WORKDIR app", + "ARG VERSION=1", + "COPY --chmod=755 a /dest/", + "COPY --link a /dest/", + "COPY --parents a /dest/", + "FROM --platform=linux/amd64 ubuntu", + ) + for instruction in unsupported: + with self.subTest(instruction=instruction): + dockerfile = ( + instruction + if instruction.startswith("FROM") + else f"FROM ubuntu\n{instruction}\n" + ) + with self.assertRaises(DockerfileParseError): + Sandbox(context=LocalDockerContext(dockerfile)) + self.backend.create.assert_not_called() + + def test_dockerfile_context_uses_base_image_and_applies_after_facades(self): + context = LocalDockerContext("FROM ubuntu:24.04\nRUN true\n") + apply_result = DockerfileApplyResult( + start_cmd=None, + startup_command=None, + entrypoint=None, + warnings=(), + ) + with patch( + "akernel_sdk._dockerfile_runner.apply_dockerfile", + return_value=apply_result, + ) as apply: + sandbox = Sandbox(context=context) + + spec = self.backend.create.call_args.args[0] + self.assertEqual(spec.image, "ubuntu:24.04") + self.assertIs(apply.call_args.args[0], sandbox) + self.assertIsNotNone(sandbox._files) + self.assertIsNotNone(sandbox._commands) + self.assertIsNotNone(sandbox._pty) + self.assertIs(apply.call_args.args[2], context) + self.assertTrue(apply.call_args.kwargs["auto_start_cmd"]) + self.assertEqual(apply.call_args.kwargs["run_timeout"], 600) + self.assertIsNone(sandbox.startup_command) + sandbox.kill() + + def test_dockerfile_context_exposes_startup_command_handle(self): + context = LocalDockerContext('FROM ubuntu:24.04\nCMD ["server"]\n') + startup_handle = object() + apply_result = DockerfileApplyResult( + start_cmd=("server",), + startup_command=startup_handle, + entrypoint=None, + warnings=(), + ) + with patch( + "akernel_sdk._dockerfile_runner.apply_dockerfile", + return_value=apply_result, + ): + sandbox = Sandbox(context=context) + + self.assertIs(sandbox.startup_command, startup_handle) + sandbox.kill() + + def test_dockerfile_context_without_dispatched_command_has_no_startup_handle(self): + context = LocalDockerContext("FROM ubuntu:24.04\n") + for auto_start_cmd in (False, True): + with self.subTest(auto_start_cmd=auto_start_cmd): + apply_result = DockerfileApplyResult( + start_cmd=None, + startup_command=None, + entrypoint=None, + warnings=(), + ) + with patch( + "akernel_sdk._dockerfile_runner.apply_dockerfile", + return_value=apply_result, + ): + sandbox = Sandbox(context=context, auto_start_cmd=auto_start_cmd) + self.assertIsNone(sandbox.startup_command) + sandbox.kill() + + def test_dockerfile_startup_dispatch_failure_terminates_and_closes_session(self): + startup_error = DockerfileBuildError( + "Failed to dispatch startup command", + instruction="CMD", + ) + with patch( + "akernel_sdk._dockerfile_runner.apply_dockerfile", + side_effect=startup_error, + ): + with self.assertRaises(DockerfileBuildError) as raised: + Sandbox( + context=LocalDockerContext('FROM ubuntu\nCMD ["server"]\n'), + detached=True, + ) + + self.assertIs(raised.exception, startup_error) + self.session.terminate.assert_called_once_with() + self.session.close.assert_called_once_with() + + def test_dockerfile_build_failure_terminates_and_closes_detached_session(self): + build_error = DockerfileBuildError("RUN failed") + with patch( + "akernel_sdk._dockerfile_runner.apply_dockerfile", + side_effect=build_error, + ): + with self.assertRaises(DockerfileBuildError) as raised: + Sandbox( + context=LocalDockerContext("FROM ubuntu\nRUN false\n"), + detached=True, + ) + + self.assertIs(raised.exception, build_error) + self.session.terminate.assert_called_once_with() + self.session.close.assert_called_once_with() + def test_get_port_url(self): with patch.dict( os.environ, diff --git a/sdk/python/tests/unit/test_types.py b/sdk/python/tests/unit/test_types.py index 13ac5cc..6bc3f5c 100644 --- a/sdk/python/tests/unit/test_types.py +++ b/sdk/python/tests/unit/test_types.py @@ -70,6 +70,17 @@ def test_public_exports_are_minimal(self): "BackendNotInstalledError", "UnsupportedBackendFeatureError", "BackendOperationError", + "DockerContext", + "LocalDockerContext", + "parse_dockerfile", + "check_direct_launch", + "apply_dockerfile", + "ParsedDockerfile", + "DockerfileApplyResult", + "DockerfileCheckResult", + "DockerfileBuildError", + "DockerfileParseError", + "BuildInstruction", }, ) for removed in ( From 517829a62b64e7d2c0e573d22da86381a2293d80 Mon Sep 17 00:00:00 2001 From: Peng-YM Date: Wed, 12 Aug 2026 20:34:47 +0800 Subject: [PATCH 02/12] fix(sdk): align direct launch semantics Preserve the explicit root working directory and represent Docker build context entries with file type and permission metadata so COPY retains executable modes and empty directories. Run archive extraction with builder ownership and reject USER forms that cannot be represented faithfully by the current command facade. Add focused regressions for directory targets, modes, ownership, strict USER validation, and live-example coverage for the corrected behavior. Signed-off-by: Peng-YM --- AGENTS.md | 10 +- sdk/python/README.md | 32 ++- sdk/python/akernel_sdk/__init__.py | 2 + sdk/python/akernel_sdk/_dockercontext.py | 274 ++++++++++++------- sdk/python/akernel_sdk/_dockerfile.py | 19 +- sdk/python/akernel_sdk/_dockerfile_runner.py | 187 +++++++++---- sdk/python/examples/dockerfile_launch.py | 72 +++-- sdk/python/tests/unit/test_dockercontext.py | 114 +++++++- sdk/python/tests/unit/test_dockerfile.py | 251 +++++++++++++++-- sdk/python/tests/unit/test_types.py | 9 +- 10 files changed, 766 insertions(+), 204 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1d80518..1ff6fc7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -349,10 +349,12 @@ of both native packages; all native conversions belong under The SDK also supports a strict Dockerfile direct-launch subset through `DockerContext` and `Sandbox(context=...)`. `FROM` supplies only the root -filesystem; inherited OCI image configuration is not applied. Context transfer -must remain backend-neutral, honor `.dockerignore`, and fail closed for syntax -outside the documented subset. Keep the contract, security boundaries, public -types, unit tests, SDK README, and `examples/dockerfile_launch.py` in sync. +filesystem; inherited OCI image configuration is not applied. `DockerContext.walk()` +uses public structured entries for files and directories, including permission +modes and empty directories. Context transfer must remain backend-neutral, honor +`.dockerignore`, and fail closed for syntax outside the documented subset. Keep +the contract, security boundaries, public types, unit tests, SDK README, and +`examples/dockerfile_launch.py` in sync. When changing a public SDK method, update its type annotations and docstring, add or update unit coverage, and keep the SDK README and maintained examples in diff --git a/sdk/python/README.md b/sdk/python/README.md index bb5c821..a59a426 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -156,21 +156,29 @@ rejects a parsed result containing unsupported syntax. | --- | --- | | Exactly one literal `FROM`, optionally `AS alias`; shell-form `RUN` | Multiple stages, `COPY --from`, `FROM` flags or variables, and exec-form `RUN` | | Shell-form local `COPY` and `ADD` paths: files, directories, `.`, and wildcards; literal `--chown`; literal local tar extraction for `ADD` | JSON-form `COPY`/`ADD`, remote `ADD` URLs, `--chmod`, `--link`, unknown flags, or build-time variable expansion | -| Literal `ENV`, absolute `WORKDIR`, literal `USER`, and `EXPOSE` metadata | Any `ARG`, relative `WORKDIR`, and `VOLUME`, `LABEL`, `HEALTHCHECK`, `SHELL`, `STOPSIGNAL`, `ONBUILD`, `MAINTAINER`, or unknown instructions | +| Literal `ENV`, absolute `WORKDIR`, named `USER` values such as `app` or `root`, and `EXPOSE` metadata | Any `ARG`, relative `WORKDIR`, `USER user:group` or numeric UID/GID values, and `VOLUME`, `LABEL`, `HEALTHCHECK`, `SHELL`, `STOPSIGNAL`, `ONBUILD`, `MAINTAINER`, or unknown instructions | | Exec- or shell-form `CMD` and `ENTRYPOINT`, normalized and combined | — | -A `DockerContext` exposes Dockerfile text plus context files. `LocalDockerContext` -uses a local directory; custom contexts may implement the same abstraction. -Before any sandbox operation, direct launch walks and validates the context -manifest, applies root `.dockerignore` rules, excludes `Dockerfile` and +A `DockerContext` exposes Dockerfile text plus structured `DockerContextEntry` +values from `walk()`. Each entry has a relative POSIX `path`, `kind` of `file` +or `directory`, and a permission-bit `mode` from `0o000` through `0o777`; +custom contexts must expose files and every ancestor or empty directory. +`LocalDockerContext` uses a local directory and rejects symbolic links. Before +any sandbox operation, direct launch walks and validates the manifest, applies +root `.dockerignore` rules to files and directories, excludes `Dockerfile` and `.dockerignore` from selection, plans every `COPY`/`ADD`, and materializes all -selected files. Local context symbolic links are rejected, and paths and target -collisions fail closed. `--chown` affects only files and directories created by -the current instruction. Local tar `ADD` accepts only regular files and -directories with safe paths. Remote `ADD` URLs are rejected; the SDK never -fetches them. Secure local context opening requires platform support for -directory-relative file descriptors and no-follow flags; unsupported platforms -fail closed. +selected files. Selected directory entries create empty and nested directories, +while each copied child file or directory has its own non-recursive mode +restored. For a literal directory source, the source root itself is only a +content container: the destination is created when needed, but does not inherit +that source-root mode. Paths and target collisions fail closed. `--chown` +affects only files and directories +created by the current instruction. Local tar `ADD` accepts only regular files +and directories with safe paths, preserves tar member metadata, and always +extracts as the builder/root identity before applying `--chown`. Remote `ADD` +URLs are rejected; the SDK never fetches them. Secure local context opening +requires platform support for directory-relative file descriptors and no-follow +flags; unsupported platforms fail closed. Each launch executes `RUN`, `COPY`, and `ADD` again in a new sandbox. There is no snapshot or build cache. After instructions finish, the SDK polls **sandbox diff --git a/sdk/python/akernel_sdk/__init__.py b/sdk/python/akernel_sdk/__init__.py index eacdffd..ea69876 100644 --- a/sdk/python/akernel_sdk/__init__.py +++ b/sdk/python/akernel_sdk/__init__.py @@ -57,6 +57,7 @@ "UnsupportedBackendFeatureError", "BackendOperationError", "DockerContext", + "DockerContextEntry", "LocalDockerContext", "parse_dockerfile", "check_direct_launch", @@ -77,6 +78,7 @@ "PtyError": (".pty", "PtyError"), "resources": ("._resources", "resources"), "DockerContext": ("._dockercontext", "DockerContext"), + "DockerContextEntry": ("._dockercontext", "DockerContextEntry"), "LocalDockerContext": ("._dockercontext", "LocalDockerContext"), "parse_dockerfile": ("._dockerfile", "parse_dockerfile"), "check_direct_launch": ("._dockerfile", "check_direct_launch"), diff --git a/sdk/python/akernel_sdk/_dockercontext.py b/sdk/python/akernel_sdk/_dockercontext.py index 626be14..314444d 100644 --- a/sdk/python/akernel_sdk/_dockercontext.py +++ b/sdk/python/akernel_sdk/_dockercontext.py @@ -48,65 +48,97 @@ class DockerContextError(ValueError): @dataclass(frozen=True) -class _SelectedContextFile: - """A source context file and its relative destination target.""" +class DockerContextEntry: + """One file or directory exposed by a :class:`DockerContext`. + + ``path`` is relative to the context root and uses POSIX separators. + ``mode`` contains only permission bits, from ``0`` through ``0o777``. + """ + + path: str + kind: Literal["file", "directory"] + mode: int + + def __post_init__(self) -> None: + if self.kind not in ("file", "directory"): + raise ValueError("context entry kind must be 'file' or 'directory'") + if ( + type(self.mode) is not int + or self.mode < 0 + or self.mode > 0o777 + ): + raise ValueError("context entry mode must contain only permission bits") + + +@dataclass(frozen=True) +class _SelectedContextEntry: + """A source context entry and its relative destination target.""" source_path: str relative_target: str + kind: Literal["file", "directory"] + mode: int @dataclass(frozen=True) class _ContextSelection: - """A deterministic file selection for one COPY or ADD source.""" + """A deterministic entry selection for one COPY or ADD source.""" source: str kind: Literal["literal_file", "literal_directory", "dot", "wildcard"] - files: tuple[_SelectedContextFile, ...] + entries: tuple[_SelectedContextEntry, ...] + + @property + def has_directories(self) -> bool: + """Return whether this selection contains an explicit directory.""" + + return any(entry.kind == "directory" for entry in self.entries) @dataclass(frozen=True) class _ContextManifest: """Validated, ignored-filtered context entries used to plan selections.""" - _raw_files: tuple[str, ...] - _visible_files: tuple[str, ...] - _raw_directories: frozenset[str] + _raw_entries: tuple[DockerContextEntry, ...] + _visible_entries: tuple[DockerContextEntry, ...] @classmethod def from_context(cls, context: DockerContext) -> _ContextManifest: """Build a manifest without opening selectable context files.""" try: - raw_files = tuple(context.walk()) + raw_entries = tuple(context.walk()) except Exception as error: raise DockerContextError("failed to walk Docker context") from error - _validate_walk_paths(raw_files) - raw_files = tuple(sorted(raw_files)) - raw_directories = _directories_from_files(raw_files) + _validate_walk_entries(raw_entries) + raw_entries = tuple(sorted(raw_entries, key=lambda entry: entry.path)) + raw_files = tuple( + entry.path for entry in raw_entries if entry.kind == "file" + ) ignore_spec = _read_ignore_spec(context, raw_files) - - reserved = {path for path in raw_files if _is_reserved_root_path(path)} - visible_files = tuple( - path - for path in raw_files - if path not in reserved and not ignore_spec.match_file(path) + visible_entries = tuple( + entry + for entry in raw_entries + if not _is_reserved_root_path(entry.path) + and not ignore_spec.match_file(entry.path) ) - return cls(raw_files, visible_files, frozenset(raw_directories)) + return cls(raw_entries, visible_entries) def select(self, source: str) -> _ContextSelection: """Plan one normalized Docker COPY or ADD source without opening files.""" normalized = _normalize_source(source) if normalized == ".": - if not self._visible_files: - reason = "ignored" if self._raw_files else "no match" + if not self._visible_entries: + reason = "ignored" if self._raw_entries else "no match" raise DockerContextError(f"context source is {reason}: {normalized!r}") return _ContextSelection( source=normalized, kind="dot", - files=tuple( - _SelectedContextFile(path, path) for path in self._visible_files + entries=tuple( + _selected_entry(entry, entry.path) + for entry in self._visible_entries ), ) if _has_wildcard(normalized): @@ -114,78 +146,100 @@ def select(self, source: str) -> _ContextSelection: return self._select_literal(normalized) def _select_literal(self, source: str) -> _ContextSelection: - if source in self._raw_files: - if source not in self._visible_files: - raise DockerContextError(f"context source is ignored: {source!r}") + raw_by_path = {entry.path: entry for entry in self._raw_entries} + visible_by_path = {entry.path: entry for entry in self._visible_entries} + entry = raw_by_path.get(source) + if entry is None: + raise DockerContextError(f"context source has no match: {source!r}") + if source not in visible_by_path: + raise DockerContextError(f"context source is ignored: {source!r}") + if entry.kind == "file": return _ContextSelection( source=source, kind="literal_file", - files=(_SelectedContextFile(source, posixpath.basename(source)),), + entries=(_selected_entry(entry, posixpath.basename(source)),), ) - if source in self._raw_directories: - files = tuple( - _SelectedContextFile(path, path.removeprefix(f"{source}/")) - for path in self._visible_files - if path.startswith(f"{source}/") - ) - if not files: - raise DockerContextError(f"context source is ignored: {source!r}") - return _ContextSelection( - source=source, - kind="literal_directory", - files=_validate_selection_files(files), + entries = tuple( + _selected_entry( + item, + "" if item.path == source else item.path.removeprefix(f"{source}/"), ) - - raise DockerContextError(f"context source has no match: {source!r}") + for item in self._visible_entries + if item.path == source or item.path.startswith(f"{source}/") + ) + if not entries: + raise DockerContextError(f"context source is ignored: {source!r}") + return _ContextSelection( + source=source, + kind="literal_directory", + entries=_validate_selection_entries(entries), + ) def _select_wildcard(self, source: str) -> _ContextSelection: - matched_files = [ - path for path in self._raw_files if _glob_matches(source, path) - ] - matched_directories = [ - path for path in self._raw_directories if _glob_matches(source, path) + matched_entries = [ + entry for entry in self._raw_entries if _glob_matches(source, entry.path) ] - if not matched_files and not matched_directories: + if not matched_entries: raise DockerContextError(f"context source has no match: {source!r}") top_directories: list[str] = [] - for directory in sorted( - matched_directories, key=lambda item: (item.count("/"), item) + for entry in sorted( + (entry for entry in matched_entries if entry.kind == "directory"), + key=lambda item: (item.path.count("/"), item.path), ): if not any( - directory.startswith(f"{parent}/") for parent in top_directories + entry.path.startswith(f"{parent}/") for parent in top_directories ): - top_directories.append(directory) + top_directories.append(entry.path) - files: list[_SelectedContextFile] = [] + entries: list[_SelectedContextEntry] = [] for directory in top_directories: prefix = f"{directory}/" - for path in self._visible_files: - if path.startswith(prefix): - child = path.removeprefix(prefix) - files.append( - _SelectedContextFile( - path, f"{posixpath.basename(directory)}/{child}" - ) + for entry in self._visible_entries: + if entry.path == directory or entry.path.startswith(prefix): + child = ( + "" + if entry.path == directory + else entry.path.removeprefix(prefix) ) + target = posixpath.basename(directory) + if child: + target = f"{target}/{child}" + entries.append(_selected_entry(entry, target)) - for path in self._visible_files: - if not _glob_matches(source, path): + for entry in self._visible_entries: + if not _glob_matches(source, entry.path): continue - if any(path.startswith(f"{directory}/") for directory in top_directories): + if any( + entry.path == directory or entry.path.startswith(f"{directory}/") + for directory in top_directories + ): continue - files.append(_SelectedContextFile(path, posixpath.basename(path))) + entries.append(_selected_entry(entry, posixpath.basename(entry.path))) - if not files: + if not entries: raise DockerContextError(f"context source is ignored: {source!r}") return _ContextSelection( source=source, kind="wildcard", - files=_validate_selection_files(files), + entries=_validate_selection_entries(entries), ) +def _selected_entry( + entry: DockerContextEntry, relative_target: str +) -> _SelectedContextEntry: + """Attach a destination-relative path to a context entry.""" + + return _SelectedContextEntry( + source_path=entry.path, + relative_target=relative_target, + kind=entry.kind, + mode=entry.mode, + ) + + class DockerContext(ABC): """Dockerfile + build context files source abstraction. @@ -208,10 +262,12 @@ def open(self, path: str) -> Iterator[BinaryIO]: """ @abstractmethod - def walk(self) -> Iterator[str]: - """Enumerate relative POSIX paths of all context files. + def walk(self) -> Iterator[DockerContextEntry]: + """Enumerate every context file and directory as structured entries. - The Dockerfile itself is not part of the enumerated context files. + Entries use relative POSIX paths. The Dockerfile itself is not included. + File entries must be readable through :meth:`open`; directory entries + make empty directories and their permission modes representable. """ @@ -221,11 +277,14 @@ def _to_posix(rel: str) -> str: return rel.replace(os.sep, "/").lstrip("/") -def _validate_walk_paths(paths: tuple[object, ...]) -> None: - """Reject malformed context paths before they can affect selection.""" +def _validate_walk_entries(entries: tuple[object, ...]) -> None: + """Reject malformed context entries before they can affect selection.""" - seen: set[str] = set() - for path in paths: + seen: dict[str, DockerContextEntry] = {} + for entry in entries: + if not isinstance(entry, DockerContextEntry): + raise DockerContextError("context walk yielded a non-entry") + path = entry.path if not isinstance(path, str): raise DockerContextError("context walk yielded a non-string path") if not path: @@ -245,19 +304,21 @@ def _validate_walk_paths(paths: tuple[object, ...]) -> None: raise DockerContextError("context walk yielded a non-normalized path") if path in seen: raise DockerContextError(f"context walk yielded a duplicate path: {path!r}") - seen.add(path) - - -def _directories_from_files(paths: tuple[str, ...]) -> set[str]: - """Derive non-empty directories from walked file paths.""" - - directories: set[str] = set() - for path in paths: - segments = path.split("/") - for index in range(1, len(segments)): - directories.add("/".join(segments[:index])) - return directories + seen[path] = entry + for path in seen: + parent = posixpath.dirname(path) + while parent not in ("", "."): + ancestor = seen.get(parent) + if ancestor is None: + raise DockerContextError( + f"context walk omitted directory entry: {parent!r}" + ) + if ancestor.kind == "file": + raise DockerContextError( + f"context walk yielded a file-as-ancestor path: {parent!r}" + ) + parent = posixpath.dirname(parent) def _is_reserved_root_path(path: str) -> bool: """Return whether a root metadata file is never selectable.""" @@ -384,12 +445,12 @@ def match(pattern_index: int, path_index: int) -> bool: return match(0, 0) -def _validate_selection_files( - files: tuple[_SelectedContextFile, ...] | list[_SelectedContextFile], -) -> tuple[_SelectedContextFile, ...]: - """Sort selection files and reject source or target collisions.""" +def _validate_selection_entries( + entries: tuple[_SelectedContextEntry, ...] | list[_SelectedContextEntry], +) -> tuple[_SelectedContextEntry, ...]: + """Sort selection entries and reject source or target collisions.""" - ordered = tuple(sorted(files, key=lambda item: item.source_path)) + ordered = tuple(sorted(entries, key=lambda item: item.source_path)) source_paths = [item.source_path for item in ordered] target_paths = [item.relative_target for item in ordered] if len(source_paths) != len(set(source_paths)): @@ -501,25 +562,47 @@ def open(self, path: str) -> Iterator[BinaryIO]: for descriptor in reversed(directory_fds): os.close(descriptor) - def walk(self) -> Iterator[str]: + def walk(self) -> Iterator[DockerContextEntry]: + """Deterministically enumerate regular files and all directories. + + Symlinks and non-regular files fail closed. Keeping directory entries is + required to preserve empty directories and their modes during COPY. + """ + if not os.path.isdir(self._context_dir): return dockerfile_abs = ( os.path.normpath(self._dockerfile_path) if self._dockerfile_path else None ) + entries: list[DockerContextEntry] = [] for root, dirs, files in os.walk(self._context_dir): + dirs.sort() for name in dirs: - if os.path.islink(os.path.join(root, name)): + full = os.path.join(root, name) + info = os.lstat(full) + if stat.S_ISLNK(info.st_mode): raise DockerContextError( f"context contains a symbolic link: {name!r}" ) - dirs.sort() + if not stat.S_ISDIR(info.st_mode): + raise DockerContextError( + f"context path is not a directory: {name!r}" + ) + rel = _to_posix(os.path.relpath(full, self._context_dir)) + entries.append( + DockerContextEntry(rel, "directory", stat.S_IMODE(info.st_mode)) + ) for name in sorted(files): full = os.path.join(root, name) - if os.path.islink(full): + info = os.lstat(full) + if stat.S_ISLNK(info.st_mode): raise DockerContextError( f"context contains a symbolic link: {name!r}" ) + if not stat.S_ISREG(info.st_mode): + raise DockerContextError( + f"context path is not a regular file: {name!r}" + ) if dockerfile_abs and os.path.normpath(full) == dockerfile_abs: continue if ( @@ -528,8 +611,11 @@ def walk(self) -> Iterator[str]: and name.lower() == "dockerfile" ): continue - rel = os.path.relpath(full, self._context_dir) - yield _to_posix(rel) + rel = _to_posix(os.path.relpath(full, self._context_dir)) + entries.append( + DockerContextEntry(rel, "file", stat.S_IMODE(info.st_mode)) + ) + yield from sorted(entries, key=lambda entry: entry.path) @property def context_dir(self) -> str: diff --git a/sdk/python/akernel_sdk/_dockerfile.py b/sdk/python/akernel_sdk/_dockerfile.py index 057d483..74b852b 100644 --- a/sdk/python/akernel_sdk/_dockerfile.py +++ b/sdk/python/akernel_sdk/_dockerfile.py @@ -173,6 +173,7 @@ def _resolve_entrypoint( _CHOWN_VALUE_RE = re.compile( r"[A-Za-z0-9_][A-Za-z0-9_.-]*(?::[A-Za-z0-9_][A-Za-z0-9_.-]*)?" ) +_USER_NAME_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_.-]*\Z") _VARIABLE_RE = re.compile(r"\$(?:\{[^}]*\}|[A-Za-z_][A-Za-z0-9_]*)") DIRECT_LAUNCH_ROOTFS_ONLY_WARNING = ( "FROM supplies only the root filesystem; inherited image ENV, USER, WORKDIR, " @@ -333,7 +334,7 @@ def parse_dockerfile( instructions.append(WorkdirInstruction(path=wd)) elif kind == "USER": user_value = value.strip() - if not user_value or _contains_variable(user_value): + if _contains_variable(user_value): _unsupported( kind, value, @@ -343,6 +344,16 @@ def parse_dockerfile( unsupported, warnings, ) + elif not _is_named_user(user_value): + _unsupported( + kind, + value, + "USER must be a literal named user without a group or numeric ID", + "unsupported_syntax", + strict, + unsupported, + warnings, + ) else: user = user_value instructions.append(UserInstruction(user=user_value)) @@ -815,6 +826,12 @@ def _parse_cmd_entrypoint( out.append(EntrypointInstruction(cmd=(folded,), shell_form=True)) +def _is_named_user(value: str) -> bool: + """Return whether ``value`` is within the direct-launch USER subset.""" + + return bool(_USER_NAME_RE.fullmatch(value)) + + def _contains_variable(value: str) -> bool: return bool(_VARIABLE_RE.search(value)) diff --git a/sdk/python/akernel_sdk/_dockerfile_runner.py b/sdk/python/akernel_sdk/_dockerfile_runner.py index 815e56a..678f9ef 100644 --- a/sdk/python/akernel_sdk/_dockerfile_runner.py +++ b/sdk/python/akernel_sdk/_dockerfile_runner.py @@ -32,7 +32,7 @@ import time from collections.abc import Iterable from dataclasses import dataclass -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Literal from ._dockercontext import DockerContext, _ContextManifest from ._dockerfile import ( @@ -65,11 +65,14 @@ class DockerfileApplyResult: @dataclass(frozen=True) class _PlannedCopy: - """One selected context file and its final sandbox target.""" + """One selected context entry and its final sandbox target.""" source_path: str relative_target: str remote_path: str + kind: Literal["file", "directory"] + mode: int + is_destination_marker: bool = False local_path: str | None = None @@ -90,7 +93,6 @@ class _PreparedCopy: workdir: str dest: str plans: tuple[_PlannedCopy, ...] - has_directory_selection: bool extract_tar: bool tar_entries: tuple[_TarEntry, ...] = () @@ -169,7 +171,7 @@ def apply_dockerfile( elif isinstance(instruction, RunInstruction): runner.run(instruction.command, envs, workdir, user, index) elif isinstance(instruction, CopyInstruction): - runner.execute_prepared_copy(prepared_copies[index], envs, user) + runner.execute_prepared_copy(prepared_copies[index], envs) elif isinstance(instruction, EnvInstruction): envs.update(instruction.envs) elif isinstance(instruction, WorkdirInstruction): @@ -253,7 +255,7 @@ def run( result = self._sb.commands.run( wrapped, envs=envs or None, - cwd=workdir if workdir != "/" else None, + cwd=workdir, timeout=self._run_timeout, ) if result.exit_code != 0: @@ -289,20 +291,17 @@ def prepare_copy( ) from error plans: list[_PlannedCopy] = [] - has_directory_selection = False for source, selection in selections: - directory_selection = selection.kind in ("literal_directory", "dot") - wildcard_directory_expansion = selection.kind == "wildcard" and any( - "/" in selected.relative_target for selected in selection.files - ) - must_use_directory = directory_selection or ( - selection.kind == "wildcard" - and (len(selection.files) > 1 or wildcard_directory_expansion) + directory_target = ( + selection.kind in ("literal_directory", "dot") + or selection.has_directories + or ins.dest.endswith("/") ) + must_use_directory = selection.has_directories or len(selection.entries) > 1 if ( must_use_directory and not ins.dest.endswith("/") - and not directory_selection + and not directory_target ): raise DockerfileBuildError( f"{instruction} source {source!r} expands to multiple paths; " @@ -310,13 +309,19 @@ def prepare_copy( index=index, instruction=instruction, ) - has_directory_selection |= directory_selection - for selected in selection.files: + for selected in selection.entries: + is_destination_marker = ( + selection.kind == "literal_directory" + and selected.kind == "directory" + and selected.relative_target == "" + ) try: - remote_path = self._copy_target( - dest, - selected.relative_target, - directory_selection or ins.dest.endswith("/"), + remote_path = ( + dest + if is_destination_marker + else self._copy_target( + dest, selected.relative_target, directory_target + ) ) except Exception as error: raise DockerfileBuildError( @@ -330,6 +335,9 @@ def prepare_copy( source_path=selected.source_path, relative_target=selected.relative_target, remote_path=remote_path, + kind=selected.kind, + mode=selected.mode, + is_destination_marker=is_destination_marker, ) ) @@ -338,10 +346,9 @@ def prepare_copy( ins.is_add and len(ins.srcs) == 1 and selections[0][1].kind == "literal_file" - and len(selections[0][1].files) == 1 - and selections[0][1].files[0].source_path.lower().endswith(_TAR_SUFFIXES) + and len(selections[0][1].entries) == 1 + and selections[0][1].entries[0].source_path.lower().endswith(_TAR_SUFFIXES) ) - materialized = self._materialize( plans, os.path.join(staging_dir, f"{index:08d}"), index, instruction ) @@ -356,7 +363,6 @@ def prepare_copy( workdir=workdir, dest=dest, plans=tuple(materialized), - has_directory_selection=has_directory_selection, extract_tar=extract_tar, tar_entries=tar_entries, ) @@ -365,7 +371,6 @@ def execute_prepared_copy( self, prepared: _PreparedCopy, envs: dict[str, str], - user: str | None, ) -> None: ins = prepared.instruction instruction = "ADD" if ins.is_add else "COPY" @@ -382,7 +387,6 @@ def execute_prepared_copy( prepared.dest, envs, prepared.workdir, - user, prepared.index, ) chown_targets = ( @@ -394,18 +398,36 @@ def execute_prepared_copy( + new_directories ) else: - parents = self._parent_directories( - plan.remote_path for plan in prepared.plans - ) - new_directories = self._new_directories(parents) if ins.chown else () - for parent in parents: - self._sb.files.make_dir(parent) + directories = self._copy_directories(prepared.plans) + new_directories = self._new_directories(directories) if ins.chown else () + for directory in directories: + self._sb.files.make_dir(directory) for plan in prepared.plans: + if plan.kind == "directory": + continue if plan.local_path is None: raise AssertionError("copy plan was not materialized") self._sb.files.copy_from_local(plan.local_path, plan.remote_path) + self._chmod( + tuple( + dict.fromkeys( + (plan.remote_path, plan.mode) + for plan in prepared.plans + if not plan.is_destination_marker + ) + ), + prepared.workdir, + prepared.index, + instruction, + ) chown_targets = ( - tuple(dict.fromkeys(plan.remote_path for plan in prepared.plans)) + tuple( + dict.fromkeys( + plan.remote_path + for plan in prepared.plans + if plan.kind == "file" + ) + ) + new_directories ) @@ -456,24 +478,33 @@ def _validate_copy_plan( index=index, instruction=instruction, ) - relative_targets = [plan.relative_target for plan in plans] + relative_targets = [ + plan.relative_target for plan in plans if not plan.is_destination_marker + ] if len(relative_targets) != len(set(relative_targets)): raise DockerfileBuildError( f"{instruction} sources have colliding relative targets", index=index, instruction=instruction, ) - targets = {plan.remote_path for plan in plans} - if len(targets) != len(plans): + by_target: dict[str, _PlannedCopy] = {} + for plan in plans: + existing = by_target.get(plan.remote_path) + if existing is None: + by_target[plan.remote_path] = plan + continue + if existing.is_destination_marker and plan.is_destination_marker: + continue raise DockerfileBuildError( f"{instruction} sources have colliding destination paths", index=index, instruction=instruction, ) - for target in targets: + for target, _plan in by_target.items(): parent = posixpath.dirname(target) while parent != "/": - if parent in targets: + ancestor = by_target.get(parent) + if ancestor is not None and ancestor.kind == "file": raise DockerfileBuildError( f"{instruction} destination file conflicts with descendant: " f"{parent!r}", @@ -482,6 +513,19 @@ def _validate_copy_plan( ) parent = posixpath.dirname(parent) + def _copy_directories(self, plans: tuple[_PlannedCopy, ...]) -> tuple[str, ...]: + """Return parent and explicitly selected directories in creation order.""" + + directories = set( + self._parent_directories(plan.remote_path for plan in plans) + ) + directories.update( + plan.remote_path + for plan in plans + if plan.kind == "directory" and plan.remote_path != "/" + ) + return tuple(sorted(directories, key=lambda path: (path.count("/"), path))) + def _materialize( self, plans: list[_PlannedCopy], @@ -492,6 +536,9 @@ def _materialize( materialized: list[_PlannedCopy] = [] os.makedirs(staging_dir, exist_ok=True) for sequence, plan in enumerate(plans): + if plan.kind == "directory": + materialized.append(plan) + continue local_path = os.path.join(staging_dir, f"{sequence:08d}") try: with self._context.open(plan.source_path) as stream: @@ -509,6 +556,9 @@ def _materialize( source_path=plan.source_path, relative_target=plan.relative_target, remote_path=plan.remote_path, + kind=plan.kind, + mode=plan.mode, + is_destination_marker=plan.is_destination_marker, local_path=local_path, ) ) @@ -643,6 +693,7 @@ def _ensure_tar_paths_not_symlinks( command = " && ".join(f"test ! -L {shlex.quote(path)}" for path in paths) result = self._sb.commands.run( command, + cwd=prepared.workdir, timeout=self._run_timeout, ) if result.exit_code != 0: @@ -666,7 +717,7 @@ def _chown( command = f"chown {shlex.quote(owner)} {shlex.quote(target)}" result = self._sb.commands.run( wrap_user(command, "root"), - cwd=workdir if workdir != "/" else None, + cwd=workdir, timeout=self._run_timeout, ) if result.exit_code != 0: @@ -678,6 +729,31 @@ def _chown( exit_code=result.exit_code, ) + def _chmod( + self, + targets: tuple[tuple[str, int], ...], + workdir: str, + index: int, + instruction: str, + ) -> None: + """Restore selected context entry modes without touching parent paths.""" + + for target, mode in targets: + command = f"chmod {mode:04o} {shlex.quote(target)}" + result = self._sb.commands.run( + wrap_user(command, "root"), + cwd=workdir, + timeout=self._run_timeout, + ) + if result.exit_code != 0: + raise DockerfileBuildError( + f"{instruction} chmod failed: {result.stderr}", + index=index, + instruction=instruction, + stderr=result.stderr, + exit_code=result.exit_code, + ) + def _extract_tar( self, local_tar: str, @@ -685,7 +761,6 @@ def _extract_tar( dest: str, envs: dict[str, str], workdir: str, - user: str | None, index: int, ) -> None: # Docker ADD extraction creates the destination directory before tar xf. @@ -697,9 +772,9 @@ def _extract_tar( f"tar xf {shlex.quote(sandbox_tar)} --no-same-owner -C {shlex.quote(dest)}" ) result = self._sb.commands.run( - wrap_user(tar_cmd, user), + wrap_user(tar_cmd, "root"), envs=envs or None, - cwd=workdir if workdir != "/" else None, + cwd=workdir, timeout=self._run_timeout, ) if result.exit_code != 0: @@ -742,7 +817,7 @@ def launch_start_cmd( wrapped, background=True, envs=envs or None, - cwd=workdir if workdir != "/" else None, + cwd=workdir, ) @@ -753,18 +828,20 @@ def _join_posix(base: str, rel: str) -> str: def wrap_user(command: str, user: str | None) -> str: - """Wrap ``command`` so it runs as ``user`` (root passthrough when None). + """Wrap ``command`` for a supported named user. - Uses ``runuser`` by default. The actual fallback to ``su`` is resolved at - runtime inside the sandbox (see :func:`resolve_user_wrapper`); ``wrap_user`` - emits a wrapper that tries ``runuser`` first and falls back to ``su``. + The parser already rejects unsupported USER syntax. This defensive check + keeps direct callers from silently changing a ``user:group`` or numeric + value into a different command identity. """ - if not user: + if user is None: return command - user = user.split(":", 1)[0] # strip :group - if not user or user == "root" or user == "0": + if user == "root": return command - # Try runuser, fall back to su if runuser is absent. + if not _is_named_user(user): + raise ValueError( + "USER must be a literal named user without a group or numeric ID" + ) return ( f"if command -v runuser >/dev/null 2>&1; then " f"runuser -u {_shq(user)} -- sh -c {_shq(command)}; " @@ -772,6 +849,14 @@ def wrap_user(command: str, user: str | None) -> str: ) +def _is_named_user(value: str) -> bool: + alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_" + characters = alphabet + "0123456789.-" + return bool(value) and value[0] in alphabet and all( + character in characters for character in value + ) + + def _shq(s: str) -> str: """Single-quote a shell string.""" return "'" + s.replace("'", "'\\''") + "'" diff --git a/sdk/python/examples/dockerfile_launch.py b/sdk/python/examples/dockerfile_launch.py index 40e677c..fe85057 100644 --- a/sdk/python/examples/dockerfile_launch.py +++ b/sdk/python/examples/dockerfile_launch.py @@ -20,10 +20,10 @@ or build cache. Sections: - 1. Core path — .dockerignore, COPY ., RUN, ENV, WORKDIR, USER, and CMD - 2. ENTRYPOINT + CMD exec-form combination + 1. Core path — filtered COPY ., modes, empty directories, explicit state + 2. Root cwd plus ENTRYPOINT + CMD exec-form combination 3. COPY --chown ownership - 4. ADD local tar extraction + 4. Builder/root ADD after USER 5. Fail-closed pre-check without a sandbox """ @@ -57,6 +57,15 @@ def section_core_path() -> None: encoding="utf-8", ) (context_dir / "greeting.txt").write_text("hello\n", encoding="utf-8") + executable = context_dir / "entrypoint.sh" + executable.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + executable.chmod(0o755) + empty = context_dir / "empty" + nested_empty = context_dir / "tree" / "nested-empty" + empty.mkdir() + nested_empty.mkdir(parents=True) + empty.chmod(0o711) + nested_empty.chmod(0o750) dockerfile = """\ FROM ubuntu:22.04 RUN apt-get update && apt-get install -y --no-install-recommends python3 @@ -64,6 +73,7 @@ def section_core_path() -> None: ENV WHOAMI=app WORKDIR /srv USER app +COPY empty/ /srv/literal-empty/ COPY . /srv/ CMD ["python3", "/srv/app.py"] """ @@ -81,7 +91,21 @@ def section_core_path() -> None: assert marker.stdout.strip() == "whoami=app cwd=/srv", marker.stdout absent = sandbox.commands.run("test ! -e /srv/secret.txt") assert absent.exit_code == 0, absent.stderr - print(f" sandbox: {sandbox.id}; marker: {marker.stdout.strip()}") + modes = sandbox.commands.run( + "stat -c '%a' /srv/entrypoint.sh /srv/empty " + "/srv/tree/nested-empty /srv/literal-empty" + ) + assert modes.exit_code == 0, modes.stderr + assert modes.stdout.splitlines() == [ + "755", + "711", + "750", + "755", + ], modes.stdout + print( + f" sandbox: {sandbox.id}; marker: {marker.stdout.strip()}; " + f"modes: {modes.stdout.splitlines()}" + ) def section_entrypoint_cmd_merge() -> None: @@ -91,7 +115,8 @@ def section_entrypoint_cmd_merge() -> None: context_dir = Path(directory) dockerfile = """\ FROM ubuntu:22.04 -ENTRYPOINT ["/bin/sh", "-c", "printf %s \\\"$1\\\" > /tmp/ep.out"] +RUN test "$(pwd)" = / +ENTRYPOINT ["/bin/sh", "-c", "printf %s \\\"$1\\\" > /tmp/ep.out; pwd > /tmp/cwd.out"] CMD ["ignored-argv-zero", "entrypoint+cmd merged"] """ context = LocalDockerContext(dockerfile, context_dir=context_dir) @@ -105,7 +130,13 @@ def section_entrypoint_cmd_merge() -> None: output = sandbox.commands.run("cat /tmp/ep.out") assert output.exit_code == 0, output.stderr assert output.stdout == "entrypoint+cmd merged", output.stdout - print(f" sandbox: {sandbox.id}; output: {output.stdout}") + cwd = sandbox.commands.run("cat /tmp/cwd.out") + assert cwd.exit_code == 0, cwd.stderr + assert cwd.stdout.strip() == "/", cwd.stdout + print( + f" sandbox: {sandbox.id}; output: {output.stdout}; " + f"startup cwd: {cwd.stdout.strip()}" + ) def section_copy_chown() -> None: @@ -146,7 +177,10 @@ def section_add_tar() -> None: tar.add(nested, arcname="nested") context = LocalDockerContext( - "FROM ubuntu:22.04\nADD app.tar.gz /opt/app/\n", + "FROM ubuntu:22.04\n" + "RUN useradd -m app\n" + "USER app\n" + "ADD app.tar.gz /opt/app/\n", context_dir=context_dir, ) _precheck(context) @@ -160,17 +194,23 @@ def section_add_tar() -> None: def section_fail_closed_precheck() -> None: - """Reject remote ADD before a sandbox is created.""" + """Reject remote ADD and unsupported USER forms before sandbox creation.""" print("\n=== Section 5: fail-closed precheck ===") - with tempfile.TemporaryDirectory() as directory: - context = LocalDockerContext( + cases = ( + ( "FROM ubuntu:22.04\nADD https://example.test/app.tar /opt/app/\n", - context_dir=directory, - ) - result = check_direct_launch(context) - assert not result.direct_launchable - assert "remote_add" in result.reasons, result - print(f" rejected reasons: {', '.join(result.reasons)}") + "remote_add", + ), + ("FROM ubuntu:22.04\nUSER app:staff\n", "unsupported_syntax"), + ("FROM ubuntu:22.04\nUSER 1000:1001\n", "unsupported_syntax"), + ) + with tempfile.TemporaryDirectory() as directory: + for dockerfile, reason in cases: + context = LocalDockerContext(dockerfile, context_dir=directory) + result = check_direct_launch(context) + assert not result.direct_launchable + assert reason in result.reasons, result + print(f" rejected reasons: {', '.join(result.reasons)}") def main() -> None: diff --git a/sdk/python/tests/unit/test_dockercontext.py b/sdk/python/tests/unit/test_dockercontext.py index f5d2466..20c5a0f 100644 --- a/sdk/python/tests/unit/test_dockercontext.py +++ b/sdk/python/tests/unit/test_dockercontext.py @@ -14,6 +14,7 @@ from akernel_sdk._dockercontext import ( DockerContext, + DockerContextEntry, DockerContextError, LocalDockerContext, _ContextManifest, @@ -28,7 +29,7 @@ class MemoryDockerContext(DockerContext): def __init__(self, files: dict[str, bytes], paths: list[object] | None = None): self.files = files - self.paths = list(files) if paths is None else paths + self.paths = _entries(files) if paths is None else paths self.open_paths: list[str] = [] def dockerfile_text(self) -> str: @@ -39,19 +40,34 @@ def open(self, path: str) -> Iterator[BinaryIO]: self.open_paths.append(path) yield io.BytesIO(self.files[path]) - def walk(self) -> Iterator[str]: + def walk(self) -> Iterator[DockerContextEntry]: yield from self.paths # type: ignore[misc] +def _entries(files: dict[str, bytes]) -> list[DockerContextEntry]: + directories = { + "/".join(path.split("/")[:index]) + for path in files + for index in range(1, len(path.split("/"))) + } + return [ + *(DockerContextEntry(path, "directory", 0o755) for path in sorted(directories)), + *(DockerContextEntry(path, "file", 0o644) for path in sorted(files)), + ] + + def paths(selection) -> list[tuple[str, str]]: - return [(item.source_path, item.relative_target) for item in selection.files] + return [ + (item.source_path, item.relative_target) + for item in selection.entries + if item.kind == "file" + ] class TestContextManifest(unittest.TestCase): def test_deterministic_literal_file_directory_and_dot(self) -> None: context = MemoryDockerContext( {"src/lib/b.py": b"", "root.txt": b"", "src/a.py": b""}, - ["src/lib/b.py", "root.txt", "src/a.py"], ) manifest = _ContextManifest.from_context(context) self.assertEqual(context.open_paths, []) @@ -209,9 +225,35 @@ def test_malicious_walk_paths_fail_closed(self) -> None: for value in invalid: with self.subTest(value=repr(value)): with self.assertRaises(DockerContextError): - _ContextManifest.from_context(MemoryDockerContext({}, [value])) + _ContextManifest.from_context( + MemoryDockerContext( + {}, + [ + DockerContextEntry( # type: ignore[arg-type] + value, "file", 0o644 + ) + ], + ) + ) with self.assertRaises(DockerContextError): - _ContextManifest.from_context(MemoryDockerContext({}, ["same", "same"])) + duplicate = DockerContextEntry("same", "file", 0o644) + _ContextManifest.from_context( + MemoryDockerContext({}, [duplicate, duplicate]) + ) + + def test_walk_requires_explicit_directory_ancestors(self) -> None: + missing_parent = [ + DockerContextEntry("nested/file", "file", 0o644), + ] + with self.assertRaisesRegex(DockerContextError, "omitted directory"): + _ContextManifest.from_context(MemoryDockerContext({}, missing_parent)) + + file_ancestor = [ + DockerContextEntry("nested", "file", 0o644), + DockerContextEntry("nested/file", "file", 0o644), + ] + with self.assertRaisesRegex(DockerContextError, "file-as-ancestor"): + _ContextManifest.from_context(MemoryDockerContext({}, file_ancestor)) def test_malicious_sources_fail_closed(self) -> None: manifest = _ContextManifest.from_context(MemoryDockerContext({"safe": b""})) @@ -250,8 +292,16 @@ def test_local_walk_determinism_and_symlink_protection(self) -> None: with tempfile.TemporaryDirectory() as directory: Path(directory, "z").write_bytes(b"") Path(directory, "a").write_bytes(b"") + os.chmod(Path(directory, "z"), 0o640) + os.chmod(Path(directory, "a"), 0o600) local = LocalDockerContext("FROM scratch", context_dir=directory) - self.assertEqual(list(local.walk()), ["a", "z"]) + self.assertEqual( + list(local.walk()), + [ + DockerContextEntry("a", "file", 0o600), + DockerContextEntry("z", "file", 0o640), + ], + ) outside = Path(directory).parent / "dockercontext-outside" outside.write_bytes(b"outside") try: @@ -303,6 +353,56 @@ def racing_open( self.fail(f"escaped context: {stream.read()!r}") self.assertTrue(replaced) + def test_entries_preserve_empty_directories_and_modes(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + empty = root / "empty" + nested = empty / "nested" + nested.mkdir(parents=True) + executable = root / "run.sh" + executable.write_bytes(b"#!/bin/sh\n") + os.chmod(empty, 0o711) + os.chmod(nested, 0o750) + os.chmod(executable, 0o755) + + self.assertEqual( + list(LocalDockerContext("FROM scratch", context_dir=root).walk()), + [ + DockerContextEntry("empty", "directory", 0o711), + DockerContextEntry("empty/nested", "directory", 0o750), + DockerContextEntry("run.sh", "file", 0o755), + ], + ) + + def test_dockerignore_filters_empty_directories(self) -> None: + entries = [ + DockerContextEntry(".dockerignore", "file", 0o644), + DockerContextEntry("ignored", "directory", 0o755), + DockerContextEntry("visible", "directory", 0o711), + DockerContextEntry("visible/nested", "directory", 0o750), + ] + context = MemoryDockerContext({".dockerignore": b"ignored/" + NL}, entries) + manifest = _ContextManifest.from_context(context) + self.assertEqual( + [ + (entry.source_path, entry.relative_target) + for entry in manifest.select(".").entries + ], + [("visible", "visible"), ("visible/nested", "visible/nested")], + ) + with self.assertRaisesRegex(DockerContextError, "ignored"): + manifest.select("ignored") + + def test_context_entry_rejects_invalid_mode(self) -> None: + class MaliciousMode(int): + def __format__(self, format_spec: str) -> str: + return "0000; injected" + + for mode in (-1, 0o1000, True, MaliciousMode(0o644)): + with self.subTest(mode=mode): + with self.assertRaisesRegex(ValueError, "permission bits"): + DockerContextEntry("path", "file", mode) + def test_local_open_accepts_nested_regular_files_only(self) -> None: with tempfile.TemporaryDirectory() as directory: nested = Path(directory, "nested") diff --git a/sdk/python/tests/unit/test_dockerfile.py b/sdk/python/tests/unit/test_dockerfile.py index 0db1282..9e34d88 100644 --- a/sdk/python/tests/unit/test_dockerfile.py +++ b/sdk/python/tests/unit/test_dockerfile.py @@ -25,6 +25,7 @@ from akernel_sdk._dockercontext import ( DockerContext, + DockerContextEntry, DockerContextError, LocalDockerContext, ) @@ -35,6 +36,7 @@ DockerfileBuildError, DockerfileParseError, RunInstruction, + UserInstruction, _json_array, check_direct_launch, parse_dockerfile, @@ -146,10 +148,12 @@ def __init__( files: dict[str, bytes], *, fail_path: str | None = None, + entries: tuple[DockerContextEntry, ...] | None = None, ) -> None: self._dockerfile = dockerfile self._files = files self._fail_path = fail_path + self._entries = entries self.open_paths: list[str] = [] def dockerfile_text(self) -> str: @@ -162,8 +166,23 @@ def open(self, path: str) -> Iterator[BinaryIO]: raise OSError(f"cannot open {path}") yield io.BytesIO(self._files[path]) - def walk(self) -> Iterator[str]: - yield from self._files + def walk(self) -> Iterator[DockerContextEntry]: + if self._entries is not None: + yield from self._entries + return + directories = { + "/".join(path.split("/")[:index]) + for path in self._files + for index in range(1, len(path.split("/"))) + } + yield from ( + DockerContextEntry(path, "directory", 0o755) + for path in sorted(directories) + ) + yield from ( + DockerContextEntry(path, "file", 0o644) + for path in sorted(self._files) + ) class TestParseDockerfile(unittest.TestCase): @@ -329,10 +348,9 @@ def test_warnings_mention_no_snapshot(self): class TestWrapUser(unittest.TestCase): - def test_no_user_passthrough(self): + def test_no_user_and_root_passthrough(self): self.assertEqual(wrap_user("whoami", None), "whoami") self.assertEqual(wrap_user("whoami", "root"), "whoami") - self.assertEqual(wrap_user("whoami", "0"), "whoami") def test_user_wraps_runuser_with_su_fallback(self): wrapped = wrap_user("whoami", "app") @@ -340,11 +358,11 @@ def test_user_wraps_runuser_with_su_fallback(self): self.assertIn("su -s /bin/sh", wrapped) self.assertIn("app", wrapped) - def test_group_stripped(self): - wrapped = wrap_user("whoami", "app:grp") - # Should reference user "app" not "app:grp" - self.assertIn("app", wrapped) - self.assertNotIn("app:grp", wrapped) + def test_unsupported_user_is_not_silently_rewritten(self): + for user in ("app:grp", "0", "1000:1001", ""): + with self.subTest(user=user): + with self.assertRaisesRegex(ValueError, "named user"): + wrap_user("whoami", user) class TestResolveStartCmd(unittest.TestCase): @@ -519,9 +537,10 @@ def test_add_tar_creates_dest_dir_before_extract(self): any("tar xf" in o[1] and "/opt/app" in o[1] for o in run_ops) ) self.assertTrue(any("--no-same-owner" in o[1] for o in run_ops)) - self.assertTrue( - any("tar xf" in o[1] and "runuser" in o[1] for o in run_ops) - ) + tar_op = next(o for o in run_ops if "tar xf" in o[1]) + self.assertNotIn("runuser", tar_op[1]) + self.assertNotIn("su -s", tar_op[1]) + self.assertEqual(tar_op[4], "/") def test_add_tar_rejects_path_traversal(self): import io @@ -555,8 +574,11 @@ def _apply_memory( *, fail_path: str | None = None, existing_paths: set[str] | None = None, + entries: tuple[DockerContextEntry, ...] | None = None, ) -> tuple[_MockSandbox, _MemoryDockerContext]: - context = _MemoryDockerContext(dockerfile, files, fail_path=fail_path) + context = _MemoryDockerContext( + dockerfile, files, fail_path=fail_path, entries=entries + ) sandbox = _MockSandbox(existing_paths=existing_paths) apply_dockerfile( sandbox, @@ -652,7 +674,6 @@ def test_selection_and_target_errors_have_no_sandbox_operations(self) -> None: "COPY . /dest/\n", {".dockerignore": b"*.txt\n", "hidden.txt": b""}, ), - ("COPY . /dest/\n", {"a": b"", "a-foo": b"", "a/b": b""}), ) for instruction, files in cases: with self.subTest(instruction=instruction): @@ -773,13 +794,16 @@ def checked_open(path: str) -> Iterator[BinaryIO]: [operation[2] for operation in sandbox.files.ops if operation[0] == "cp"], ["/one/a", "/two/b"], ) - self.assertEqual(events, ["run", "copy:/one/a", "copy:/two/b"]) + self.assertEqual( + events, + ["run", "copy:/one/a", "run", "copy:/two/b", "run"], + ) def test_manifest_failure_is_wrapped_before_sandbox_operations(self) -> None: class BrokenContext(_MemoryDockerContext): - def walk(self) -> Iterator[str]: + def walk(self) -> Iterator[DockerContextEntry]: raise OSError("walk failed") - yield "unreachable" + yield DockerContextEntry("unreachable", "file", 0o644) context = BrokenContext("FROM ubuntu\nRUN echo never\nCOPY a /dest/\n", {}) sandbox = _MockSandbox() @@ -975,6 +999,173 @@ def test_tar_symlink_precheck_prevents_write_operations(self) -> None: self.assertFalse(any("chown" in op[1] for op in sandbox.commands.ops)) + def test_structured_context_copies_empty_directories_and_modes(self) -> None: + entries = ( + DockerContextEntry("empty", "directory", 0o711), + DockerContextEntry("top", "directory", 0o755), + DockerContextEntry("top/nested", "directory", 0o750), + DockerContextEntry("plain.txt", "file", 0o640), + DockerContextEntry("run.sh", "file", 0o755), + ) + context = _MemoryDockerContext( + "FROM ubuntu\nCOPY empty/ /srv/empty/\nCOPY . /tree/\n", + {"plain.txt": b"plain", "run.sh": b"#!/bin/sh\n"}, + entries=entries, + ) + sandbox = _MockSandbox() + apply_dockerfile( + sandbox, parse_dockerfile(context), context, auto_start_cmd=False + ) + copied = [op[2] for op in sandbox.files.ops if op[0] == "cp"] + self.assertEqual(copied, ["/tree/plain.txt", "/tree/run.sh"]) + made_dirs = [op[1] for op in sandbox.files.ops if op[0] == "mkdir"] + self.assertEqual( + made_dirs, + [ + "/srv", + "/srv/empty", + "/tree", + "/tree/empty", + "/tree/top", + "/tree/top/nested", + ], + ) + chmod = [op[1] for op in sandbox.commands.ops if "chmod" in op[1]] + self.assertEqual( + chmod, + [ + "chmod 0711 /tree/empty", + "chmod 0640 /tree/plain.txt", + "chmod 0755 /tree/run.sh", + "chmod 0755 /tree/top", + "chmod 0750 /tree/top/nested", + ], + ) + self.assertEqual(context.open_paths, ["plain.txt", "run.sh"]) + + def test_literal_directory_copy_to_root_copies_contents_and_subdirectories( + self, + ) -> None: + entries = ( + DockerContextEntry("src", "directory", 0o700), + DockerContextEntry("src/a", "file", 0o640), + DockerContextEntry("src/empty", "directory", 0o711), + ) + sandbox, context = self._apply_memory( + "FROM ubuntu\nCOPY src /\n", + {"src/a": b"a"}, + entries=entries, + ) + self.assertEqual( + [operation[2] for operation in sandbox.files.ops if operation[0] == "cp"], + ["/a"], + ) + self.assertEqual( + [ + operation[1] + for operation in sandbox.files.ops + if operation[0] == "mkdir" + ], + ["/empty"], + ) + self.assertEqual( + [ + operation[1] + for operation in sandbox.commands.ops + if "chmod" in operation[1] + ], + ["chmod 0640 /a", "chmod 0711 /empty"], + ) + self.assertEqual(context.open_paths, ["src/a"]) + + def test_empty_literal_directory_copy_to_root_has_no_sandbox_operations( + self, + ) -> None: + context = _MemoryDockerContext( + "FROM ubuntu\nCOPY empty /\n", + {}, + entries=(DockerContextEntry("empty", "directory", 0o700),), + ) + sandbox = _MockSandbox() + apply_dockerfile( + sandbox, parse_dockerfile(context), context, auto_start_cmd=False + ) + self.assertEqual(sandbox.files.ops, []) + self.assertEqual(sandbox.commands.ops, []) + self.assertEqual(context.open_paths, []) + + def test_multiple_literal_directories_share_destination_marker(self) -> None: + entries = ( + DockerContextEntry("one", "directory", 0o700), + DockerContextEntry("one/a", "file", 0o640), + DockerContextEntry("two", "directory", 0o711), + DockerContextEntry("two/b", "file", 0o600), + ) + sandbox, _ = self._apply_memory( + "FROM ubuntu\nCOPY one two /target/\n", + {"one/a": b"a", "two/b": b"b"}, + entries=entries, + ) + self.assertEqual( + [operation[2] for operation in sandbox.files.ops if operation[0] == "cp"], + ["/target/a", "/target/b"], + ) + self.assertNotIn( + "chmod 0700 /target", + [operation[1] for operation in sandbox.commands.ops], + ) + self.assertNotIn( + "chmod 0711 /target", + [operation[1] for operation in sandbox.commands.ops], + ) + + def test_literal_directory_root_marker_does_not_chmod_destination(self) -> None: + entries = ( + DockerContextEntry("src", "directory", 0o700), + DockerContextEntry("src/a", "file", 0o640), + ) + sandbox, _ = self._apply_memory( + "FROM ubuntu\nCOPY src /target/\n", + {"src/a": b"a"}, + existing_paths={"/target"}, + entries=entries, + ) + chmod = [ + operation[1] + for operation in sandbox.commands.ops + if "chmod" in operation[1] + ] + self.assertEqual(chmod, ["chmod 0640 /target/a"]) + self.assertNotIn("chmod 0700 /target", chmod) + + def test_runner_commands_always_receive_root_cwd(self) -> None: + archive = _archive_bytes([_regular_member("inside")]) + context = _MemoryDockerContext( + "FROM ubuntu\n" + "USER app\n" + "RUN echo build\n" + "COPY input /copy/\n" + "ADD --chown=app:app app.tar /extract/\n" + 'CMD ["server"]\n', + {"input": b"input", "app.tar": archive}, + ) + sandbox = _MockSandbox() + apply_dockerfile( + sandbox, parse_dockerfile(context), context, auto_start_cmd=True + ) + commands = [op for op in sandbox.commands.ops if op[0] == "run"] + self.assertTrue(any("test ! -L" in op[1] for op in commands)) + self.assertTrue(any("tar xf" in op[1] for op in commands)) + self.assertTrue(any("chown app:app" in op[1] for op in commands)) + self.assertTrue(any("chmod 0644 /copy/input" in op[1] for op in commands)) + self.assertTrue(any(op[2] for op in commands)) + self.assertTrue(all(op[4] == "/" for op in commands)) + tar = next(op[1] for op in commands if "tar xf" in op[1]) + self.assertNotIn("runuser", tar) + self.assertNotIn("su -s", tar) + + + class TestLocalDockerContext(unittest.TestCase): def test_content_string(self): ctx = LocalDockerContext("FROM ubuntu\n") @@ -1003,8 +1194,12 @@ def test_walk_excludes_dockerfile(self): _f.write("FROM ubuntu\n") with open(os.path.join(d, "app.py"), "w") as _f: _f.write("print(1)") + os.chmod(os.path.join(d, "app.py"), 0o640) ctx = LocalDockerContext("FROM ubuntu\n", context_dir=d) - self.assertEqual(list(ctx.walk()), ["app.py"]) + self.assertEqual( + list(ctx.walk()), + [DockerContextEntry("app.py", "file", 0o640)], + ) def test_open_path_escape_rejected(self): import tempfile @@ -1032,6 +1227,26 @@ class TestStrictDockerfileSemantics(unittest.TestCase): def _parse(self, dockerfile, strict=False): return parse_dockerfile(LocalDockerContext(dockerfile), strict=strict) + def test_user_accepts_only_literal_names(self): + accepted = self._parse("FROM ubuntu\nUSER app\n", strict=True) + self.assertEqual(accepted.user, "app") + for value in ("app:staff", "1000", "1000:1001", "app staff", ""): + with self.subTest(value=value): + dockerfile = f"FROM ubuntu\nUSER {value}\n" + parsed = self._parse(dockerfile) + self.assertEqual(parsed.user, None) + self.assertFalse( + any( + isinstance(item, UserInstruction) + for item in parsed.instructions + ) + ) + result = check_direct_launch(LocalDockerContext(dockerfile)) + self.assertFalse(result.direct_launchable) + self.assertEqual(result.reasons, ("unsupported_syntax",)) + with self.assertRaises(DockerfileParseError): + self._parse(dockerfile, strict=True) + def test_remote_add_is_never_executable(self): dockerfile = "FROM ubuntu\nADD https://example.test/app.tar /opt/\n" parsed = self._parse(dockerfile) diff --git a/sdk/python/tests/unit/test_types.py b/sdk/python/tests/unit/test_types.py index 6bc3f5c..0c9eefc 100644 --- a/sdk/python/tests/unit/test_types.py +++ b/sdk/python/tests/unit/test_types.py @@ -19,7 +19,7 @@ from pathlib import Path import akernel_sdk -from akernel_sdk import HttpReverseTunnel, Mount, S3Config +from akernel_sdk import DockerContextEntry, HttpReverseTunnel, Mount, S3Config class PublicTypesTest(unittest.TestCase): @@ -71,6 +71,7 @@ def test_public_exports_are_minimal(self): "UnsupportedBackendFeatureError", "BackendOperationError", "DockerContext", + "DockerContextEntry", "LocalDockerContext", "parse_dockerfile", "check_direct_launch", @@ -94,6 +95,12 @@ def test_public_exports_are_minimal(self): ): self.assertFalse(hasattr(akernel_sdk, removed), removed) + def test_docker_context_entry_is_public_and_immutable(self): + entry = DockerContextEntry("empty", "directory", 0o755) + self.assertEqual(entry.path, "empty") + with self.assertRaisesRegex(AttributeError, "cannot assign"): + entry.mode = 0o700 # type: ignore[misc] + def test_s3_config_serialization(self): config = S3Config( endpoint="https://s3.example.com", From 6f39d1dd24be11f901e5371cc41f7c328938c8e0 Mon Sep 17 00:00:00 2001 From: Peng-YM Date: Thu, 13 Aug 2026 13:04:48 +0800 Subject: [PATCH 03/12] fix(sdk): align wildcard context handling Match Docker source wildcards one path segment at a time and copy the contents of matched directories without retaining their root name. Fail closed when a local context traversal cannot descend into an entry, and keep context parity tests independent of the caller's umask. Add regressions for wildcard directory targets, filepath-style double-star matching, traversal failures, destination ownership, and the maintained live example. Signed-off-by: Peng-YM --- sdk/python/README.md | 7 +- sdk/python/akernel_sdk/_dockercontext.py | 68 ++++---- sdk/python/akernel_sdk/_dockerfile_runner.py | 8 +- sdk/python/examples/dockerfile_launch.py | 24 ++- sdk/python/tests/unit/test_dockercontext.py | 69 +++++++- sdk/python/tests/unit/test_dockerfile.py | 159 ++++++++++++++++++- 6 files changed, 279 insertions(+), 56 deletions(-) diff --git a/sdk/python/README.md b/sdk/python/README.md index a59a426..eefce9e 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -171,8 +171,11 @@ selected files. Selected directory entries create empty and nested directories, while each copied child file or directory has its own non-recursive mode restored. For a literal directory source, the source root itself is only a content container: the destination is created when needed, but does not inherit -that source-root mode. Paths and target collisions fail closed. `--chown` -affects only files and directories +that source-root mode. A wildcard that matches a directory likewise copies its +contents, rather than the matched directory name. Wildcard source patterns +follow Docker filepath-style segment matching, so `**` has the same one-segment +behavior as `*`. Paths and target collisions fail closed. `--chown` affects only +files and directories created by the current instruction. Local tar `ADD` accepts only regular files and directories with safe paths, preserves tar member metadata, and always extracts as the builder/root identity before applying `--chown`. Remote `ADD` diff --git a/sdk/python/akernel_sdk/_dockercontext.py b/sdk/python/akernel_sdk/_dockercontext.py index 314444d..58dfed6 100644 --- a/sdk/python/akernel_sdk/_dockercontext.py +++ b/sdk/python/akernel_sdk/_dockercontext.py @@ -198,14 +198,11 @@ def _select_wildcard(self, source: str) -> _ContextSelection: prefix = f"{directory}/" for entry in self._visible_entries: if entry.path == directory or entry.path.startswith(prefix): - child = ( + target = ( "" if entry.path == directory else entry.path.removeprefix(prefix) ) - target = posixpath.basename(directory) - if child: - target = f"{target}/{child}" entries.append(_selected_entry(entry, target)) for entry in self._visible_entries: @@ -416,33 +413,16 @@ def _has_wildcard(path: str) -> bool: def _glob_matches(pattern: str, path: str) -> bool: - """Match a POSIX path where ** spans zero or more path segments.""" + """Match Docker source patterns one POSIX path segment at a time.""" pattern_segments = pattern.split("/") path_segments = path.split("/") - cache: dict[tuple[int, int], bool] = {} - - def match(pattern_index: int, path_index: int) -> bool: - key = (pattern_index, path_index) - if key in cache: - return cache[key] - if pattern_index == len(pattern_segments): - result = path_index == len(path_segments) - elif pattern_segments[pattern_index] == "**": - result = any( - match(pattern_index + 1, next_index) - for next_index in range(path_index, len(path_segments) + 1) - ) - elif path_index == len(path_segments): - result = False - else: - result = fnmatch.fnmatchcase( - path_segments[path_index], pattern_segments[pattern_index] - ) and match(pattern_index + 1, path_index + 1) - cache[key] = result - return result - - return match(0, 0) + return len(pattern_segments) == len(path_segments) and all( + fnmatch.fnmatchcase(path_segment, pattern_segment) + for pattern_segment, path_segment in zip( + pattern_segments, path_segments, strict=True + ) + ) def _validate_selection_entries( @@ -452,7 +432,11 @@ def _validate_selection_entries( ordered = tuple(sorted(entries, key=lambda item: item.source_path)) source_paths = [item.source_path for item in ordered] - target_paths = [item.relative_target for item in ordered] + target_paths = [ + item.relative_target + for item in ordered + if not (item.kind == "directory" and item.relative_target == "") + ] if len(source_paths) != len(set(source_paths)): raise DockerContextError("context selection has duplicate source paths") if len(target_paths) != len(set(target_paths)): @@ -574,12 +558,29 @@ def walk(self) -> Iterator[DockerContextEntry]: dockerfile_abs = ( os.path.normpath(self._dockerfile_path) if self._dockerfile_path else None ) + + def traversal_error(error: OSError, path: str) -> DockerContextError: + candidate = error.filename or path + try: + detail = _to_posix(os.path.relpath(candidate, self._context_dir)) + except (TypeError, ValueError): + detail = str(candidate) + return DockerContextError( + f"cannot traverse Docker context directory: {detail!r}" + ) + + def onerror(error: OSError) -> None: + raise traversal_error(error, self._context_dir) from error + entries: list[DockerContextEntry] = [] - for root, dirs, files in os.walk(self._context_dir): + for root, dirs, files in os.walk(self._context_dir, onerror=onerror): dirs.sort() for name in dirs: full = os.path.join(root, name) - info = os.lstat(full) + try: + info = os.lstat(full) + except OSError as error: + raise traversal_error(error, full) from error if stat.S_ISLNK(info.st_mode): raise DockerContextError( f"context contains a symbolic link: {name!r}" @@ -594,7 +595,10 @@ def walk(self) -> Iterator[DockerContextEntry]: ) for name in sorted(files): full = os.path.join(root, name) - info = os.lstat(full) + try: + info = os.lstat(full) + except OSError as error: + raise traversal_error(error, full) from error if stat.S_ISLNK(info.st_mode): raise DockerContextError( f"context contains a symbolic link: {name!r}" diff --git a/sdk/python/akernel_sdk/_dockerfile_runner.py b/sdk/python/akernel_sdk/_dockerfile_runner.py index 678f9ef..787dd1c 100644 --- a/sdk/python/akernel_sdk/_dockerfile_runner.py +++ b/sdk/python/akernel_sdk/_dockerfile_runner.py @@ -311,9 +311,7 @@ def prepare_copy( ) for selected in selection.entries: is_destination_marker = ( - selection.kind == "literal_directory" - and selected.kind == "directory" - and selected.relative_target == "" + selected.kind == "directory" and selected.relative_target == "" ) try: remote_path = ( @@ -399,7 +397,9 @@ def execute_prepared_copy( ) else: directories = self._copy_directories(prepared.plans) - new_directories = self._new_directories(directories) if ins.chown else () + new_directories = ( + self._new_directories(directories) if ins.chown else () + ) for directory in directories: self._sb.files.make_dir(directory) for plan in prepared.plans: diff --git a/sdk/python/examples/dockerfile_launch.py b/sdk/python/examples/dockerfile_launch.py index fe85057..2d51b9a 100644 --- a/sdk/python/examples/dockerfile_launch.py +++ b/sdk/python/examples/dockerfile_launch.py @@ -20,7 +20,7 @@ or build cache. Sections: - 1. Core path — filtered COPY ., modes, empty directories, explicit state + 1. Core path — filtered COPY ., wildcard directory COPY, modes, empty dirs 2. Root cwd plus ENTRYPOINT + CMD exec-form combination 3. COPY --chown ownership 4. Builder/root ADD after USER @@ -43,7 +43,7 @@ def _precheck(context: LocalDockerContext) -> None: def section_core_path() -> None: - """Copy a filtered context, then run a finite CMD with explicit state.""" + """Copy a filtered context and wildcard directory, then run a finite CMD.""" print("\n=== Section 1: filtered core path ===") with tempfile.TemporaryDirectory() as directory: context_dir = Path(directory) @@ -62,8 +62,11 @@ def section_core_path() -> None: executable.chmod(0o755) empty = context_dir / "empty" nested_empty = context_dir / "tree" / "nested-empty" + wildcard_file = context_dir / "wild" / "dir1" / "dir2" / "foo" empty.mkdir() nested_empty.mkdir(parents=True) + wildcard_file.parent.mkdir(parents=True) + wildcard_file.write_text("wildcard\n", encoding="utf-8") empty.chmod(0o711) nested_empty.chmod(0o750) dockerfile = """\ @@ -73,9 +76,10 @@ def section_core_path() -> None: ENV WHOAMI=app WORKDIR /srv USER app -COPY empty/ /srv/literal-empty/ -COPY . /srv/ -CMD ["python3", "/srv/app.py"] +COPY empty/ /srv/core/literal-empty/ +COPY . /srv/core/ +COPY wild/* /srv/wild/ +CMD ["python3", "/srv/core/app.py"] """ context = LocalDockerContext(dockerfile, context_dir=context_dir) _precheck(context) @@ -89,11 +93,11 @@ def section_core_path() -> None: marker = sandbox.commands.run("cat /tmp/app.started") assert marker.exit_code == 0, marker.stderr assert marker.stdout.strip() == "whoami=app cwd=/srv", marker.stdout - absent = sandbox.commands.run("test ! -e /srv/secret.txt") + absent = sandbox.commands.run("test ! -e /srv/core/secret.txt") assert absent.exit_code == 0, absent.stderr modes = sandbox.commands.run( - "stat -c '%a' /srv/entrypoint.sh /srv/empty " - "/srv/tree/nested-empty /srv/literal-empty" + "stat -c '%a' /srv/core/entrypoint.sh /srv/core/empty " + "/srv/core/tree/nested-empty /srv/core/literal-empty" ) assert modes.exit_code == 0, modes.stderr assert modes.stdout.splitlines() == [ @@ -102,6 +106,10 @@ def section_core_path() -> None: "750", "755", ], modes.stdout + wildcard = sandbox.commands.run( + "test -f /srv/wild/dir2/foo && test ! -e /srv/wild/dir1" + ) + assert wildcard.exit_code == 0, wildcard.stderr print( f" sandbox: {sandbox.id}; marker: {marker.stdout.strip()}; " f"modes: {modes.stdout.splitlines()}" diff --git a/sdk/python/tests/unit/test_dockercontext.py b/sdk/python/tests/unit/test_dockercontext.py index 20c5a0f..c7b1244 100644 --- a/sdk/python/tests/unit/test_dockercontext.py +++ b/sdk/python/tests/unit/test_dockercontext.py @@ -2,6 +2,7 @@ from __future__ import annotations +import errno import io import os import tempfile @@ -170,18 +171,28 @@ def test_wildcards_directories_and_collisions(self) -> None: self.assertEqual(paths(manifest.select("*.py")), [("top.py", "top.py")]) self.assertEqual( paths(manifest.select("src/**/*.py")), - [("src/a.py", "a.py"), ("src/lib/b.py", "b.py")], + [("src/lib/b.py", "b.py")], ) + wildcard_directories = manifest.select("modules/**") self.assertEqual( - paths(manifest.select("modules/*")), - [("modules/one/x.py", "one/x.py"), ("modules/two/y.txt", "two/y.txt")], - ) - self.assertEqual( - paths(manifest.select("modules/**")), [ - ("modules/one/x.py", "modules/one/x.py"), - ("modules/two/y.txt", "modules/two/y.txt"), + (item.source_path, item.relative_target, item.kind) + for item in wildcard_directories.entries ], + [ + ("modules/one", "", "directory"), + ("modules/one/x.py", "x.py", "file"), + ("modules/two", "", "directory"), + ("modules/two/y.txt", "y.txt", "file"), + ], + ) + self.assertEqual( + paths(wildcard_directories), + [("modules/one/x.py", "x.py"), ("modules/two/y.txt", "y.txt")], + ) + self.assertEqual( + paths(manifest.select("modules/*")), + paths(wildcard_directories), ) with self.assertRaisesRegex(DockerContextError, "colliding"): manifest.select("*/*.py") @@ -283,6 +294,8 @@ def test_local_and_memory_contexts_match(self) -> None: path = Path(directory, name) path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(content) + os.chmod(path, 0o644) + os.chmod(Path(directory, "src"), 0o755) actual = _ContextManifest.from_context( LocalDockerContext("FROM scratch", context_dir=directory) ).select(".") @@ -314,6 +327,46 @@ def test_local_walk_determinism_and_symlink_protection(self) -> None: finally: outside.unlink(missing_ok=True) + def test_local_walk_fails_closed_when_descending(self) -> None: + with tempfile.TemporaryDirectory() as directory: + blocked = Path(directory, "blocked") + blocked.mkdir() + (blocked / "required.txt").write_text("required", encoding="utf-8") + local = LocalDockerContext("FROM scratch", context_dir=directory) + real_scandir = os.scandir + + def deny_blocked(path: str) -> object: + if os.path.abspath(os.fspath(path)) == str(blocked): + raise PermissionError( + errno.EACCES, "Permission denied", str(blocked) + ) + return real_scandir(path) + + with ( + patch( + "akernel_sdk._dockercontext.os.scandir", + side_effect=deny_blocked, + ), + self.assertRaisesRegex(DockerContextError, "blocked") as raised, + ): + list(local.walk()) + self.assertIsInstance(raised.exception.__cause__, PermissionError) + + with ( + patch( + "akernel_sdk._dockercontext.os.scandir", + side_effect=deny_blocked, + ), + self.assertRaisesRegex( + DockerContextError, "failed to walk Docker context" + ) as raised, + ): + _ContextManifest.from_context(local) + walk_error = raised.exception.__cause__ + self.assertIsInstance(walk_error, DockerContextError) + self.assertIn("blocked", str(walk_error)) + self.assertIsInstance(walk_error.__cause__, PermissionError) + def test_local_open_rejects_symlink_replacement_race(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) diff --git a/sdk/python/tests/unit/test_dockerfile.py b/sdk/python/tests/unit/test_dockerfile.py index 9e34d88..5dde7eb 100644 --- a/sdk/python/tests/unit/test_dockerfile.py +++ b/sdk/python/tests/unit/test_dockerfile.py @@ -16,12 +16,17 @@ from __future__ import annotations +import errno import io +import os import tarfile +import tempfile import unittest from collections.abc import Iterator from contextlib import contextmanager +from pathlib import Path from typing import BinaryIO +from unittest.mock import patch from akernel_sdk._dockercontext import ( DockerContext, @@ -650,7 +655,7 @@ def test_literal_directory_dot_and_wildcard_targets(self) -> None: ], ), ("COPY *.py /four/\n", ["/four/root.py"]), - ("COPY src/**/*.py /five/\n", ["/five/a.py", "/five/b.py"]), + ("COPY src/**/*.py /five/\n", ["/five/b.py"]), ) for instruction, expected in cases: with self.subTest(instruction=instruction): @@ -658,6 +663,156 @@ def test_literal_directory_dot_and_wildcard_targets(self) -> None: copies = [op[2] for op in sandbox.files.ops if op[0] == "cp"] self.assertEqual(copies, expected) + def test_wildcard_directory_copy_targets_match_buildkit(self) -> None: + for destination in ("/subdest/", "/subdest"): + with self.subTest(destination=destination): + sandbox, _ = self._apply_memory( + f"FROM ubuntu\nCOPY sub/* {destination}\n", + {"sub/dir1/dir2/foo": b"foo"}, + ) + self.assertEqual( + [ + operation[2] + for operation in sandbox.files.ops + if operation[0] == "cp" + ], + ["/subdest/dir2/foo"], + ) + made_dirs = [ + operation[1] + for operation in sandbox.files.ops + if operation[0] == "mkdir" + ] + self.assertEqual(made_dirs, ["/subdest", "/subdest/dir2"]) + self.assertNotIn("/subdest/dir1", made_dirs) + + mixed, _ = self._apply_memory( + "FROM ubuntu\nCOPY sub/* /subdest/\n", + {"sub/dir1/dir2/foo": b"foo", "sub/file": b"file"}, + ) + self.assertEqual( + [operation[2] for operation in mixed.files.ops if operation[0] == "cp"], + ["/subdest/dir2/foo", "/subdest/file"], + ) + + modules, _ = self._apply_memory( + "FROM ubuntu\nCOPY modules/** /dest/\n", + {"modules/one/x.py": b"x", "modules/two/y.txt": b"y"}, + ) + self.assertEqual( + [ + operation[2] + for operation in modules.files.ops + if operation[0] == "cp" + ], + ["/dest/x.py", "/dest/y.txt"], + ) + module_directories = [ + operation[1] + for operation in modules.files.ops + if operation[0] == "mkdir" + ] + self.assertEqual(module_directories, ["/dest"]) + + multiple, _ = self._apply_memory( + "FROM ubuntu\nCOPY * /target/\n", + {"one/a": b"a", "two/b": b"b"}, + ) + self.assertEqual( + [ + operation[2] + for operation in multiple.files.ops + if operation[0] == "cp" + ], + ["/target/a", "/target/b"], + ) + + empty = _MemoryDockerContext( + "FROM ubuntu\nCOPY --chown=app:app empty* /target/\n", + {}, + entries=(DockerContextEntry("empty", "directory", 0o700),), + ) + empty_sandbox = _MockSandbox() + apply_dockerfile( + empty_sandbox, + parse_dockerfile(empty), + empty, + auto_start_cmd=False, + ) + self.assertEqual(empty_sandbox.files.ops, [("mkdir", "/target")]) + self.assertEqual( + [op[1] for op in empty_sandbox.commands.ops if "chown" in op[1]], + ["chown app:app /target"], + ) + + root_empty = _MemoryDockerContext( + "FROM ubuntu\nCOPY --chown=app:app empty* /\n", + {}, + entries=(DockerContextEntry("empty", "directory", 0o700),), + ) + root_empty_sandbox = _MockSandbox() + apply_dockerfile( + root_empty_sandbox, + parse_dockerfile(root_empty), + root_empty, + auto_start_cmd=False, + ) + self.assertEqual(root_empty_sandbox.files.ops, []) + self.assertEqual(root_empty_sandbox.files.exists_calls, []) + self.assertEqual(root_empty_sandbox.commands.ops, []) + + collision = _MemoryDockerContext( + "FROM ubuntu\nCOPY * /target/\n", + {"left/same": b"left", "right/same": b"right"}, + ) + collision_sandbox = _MockSandbox() + with self.assertRaisesRegex(DockerfileBuildError, "colliding"): + apply_dockerfile( + collision_sandbox, + parse_dockerfile(collision), + collision, + auto_start_cmd=False, + ) + self.assertEqual(collision_sandbox.files.ops, []) + self.assertEqual(collision_sandbox.commands.ops, []) + + def test_local_traversal_failure_prevents_copy_side_effects(self) -> None: + with tempfile.TemporaryDirectory() as directory: + blocked = Path(directory, "blocked") + blocked.mkdir() + (blocked / "required.txt").write_text("required", encoding="utf-8") + context = LocalDockerContext( + "FROM ubuntu\nCOPY blocked /app\n", context_dir=directory + ) + sandbox = _MockSandbox() + real_scandir = os.scandir + + def deny_blocked(path: str | int) -> object: + if ( + isinstance(path, (str, bytes, os.PathLike)) + and os.path.abspath(os.fspath(path)) == str(blocked) + ): + raise PermissionError( + errno.EACCES, "Permission denied", str(blocked) + ) + return real_scandir(path) + + with ( + patch( + "akernel_sdk._dockercontext.os.scandir", + side_effect=deny_blocked, + ), + self.assertRaisesRegex(DockerfileBuildError, "blocked"), + ): + apply_dockerfile( + sandbox, + parse_dockerfile(context), + context, + auto_start_cmd=False, + ) + self.assertEqual(sandbox.files.ops, []) + self.assertEqual(sandbox.commands.ops, []) + def test_selection_and_target_errors_have_no_sandbox_operations(self) -> None: cases = ( ("COPY no-match /dest/\n", {"ok": b""}), @@ -889,7 +1044,7 @@ def test_directory_copy_chown_skips_existing_destination(self) -> None: self.assertNotIn("chown app:app /app", chown) self.assertFalse(any("-R" in command for command in chown)) - def test_directory_copy_chown_includes_only_new_parent_directories(self) -> None: + def test_directory_copy_chown_includes_new_destination_marker(self) -> None: sandbox, _ = self._apply_memory( "FROM ubuntu\nCOPY --chown=app:app src /app\n", {"src/a": b"a", "src/sub/b": b"b"}, From f357c5d87cb357f33ab641651894f1f5cb140624 Mon Sep 17 00:00:00 2001 From: Peng-YM Date: Fri, 14 Aug 2026 13:08:37 +0800 Subject: [PATCH 04/12] fix(sdk): validate Docker source patterns Match Dockerfile source patterns with Go filepath-compatible character classes and reject malformed patterns before context files are read. Track top-level wildcard expansions so multi-source copies require a directory destination. Add regressions for class negation, malformed ranges, Unicode matching, ignored sources, and wildcard expansion counts. Signed-off-by: Peng-YM --- sdk/python/README.md | 9 +- sdk/python/akernel_sdk/_dockercontext.py | 166 +++++++++++++++++-- sdk/python/akernel_sdk/_dockerfile_runner.py | 11 ++ sdk/python/tests/unit/test_dockercontext.py | 67 ++++++++ sdk/python/tests/unit/test_dockerfile.py | 76 +++++++++ 5 files changed, 312 insertions(+), 17 deletions(-) diff --git a/sdk/python/README.md b/sdk/python/README.md index eefce9e..c47e10f 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -173,9 +173,12 @@ restored. For a literal directory source, the source root itself is only a content container: the destination is created when needed, but does not inherit that source-root mode. A wildcard that matches a directory likewise copies its contents, rather than the matched directory name. Wildcard source patterns -follow Docker filepath-style segment matching, so `**` has the same one-segment -behavior as `*`. Paths and target collisions fail closed. `--chown` affects only -files and directories +follow Go `filepath.Match`-style segment matching, so `**` has the same +one-segment behavior as `*`; `[^a]` negates a character class while `[!a]` +matches `!` or `a`. Backslash escapes are outside this strict subset, and +malformed classes fail closed. When one wildcard expands to multiple top-level +sources after `.dockerignore` filtering, the destination must end in `/`. Paths +and target collisions fail closed. `--chown` affects only files and directories created by the current instruction. Local tar `ADD` accepts only regular files and directories with safe paths, preserves tar member metadata, and always extracts as the builder/root identity before applying `--chown`. Remote `ADD` diff --git a/sdk/python/akernel_sdk/_dockercontext.py b/sdk/python/akernel_sdk/_dockercontext.py index 58dfed6..7053764 100644 --- a/sdk/python/akernel_sdk/_dockercontext.py +++ b/sdk/python/akernel_sdk/_dockercontext.py @@ -23,7 +23,6 @@ from __future__ import annotations -import fnmatch import os import posixpath import stat @@ -47,6 +46,20 @@ class DockerContextError(ValueError): """Raised when a Docker build context cannot be safely selected.""" +@dataclass(frozen=True) +class _CharacterClass: + """One prevalidated Go filepath-style character class.""" + + negated: bool + ranges: tuple[tuple[str, str], ...] + + def matches(self, character: str) -> bool: + """Return whether a Unicode codepoint belongs to this class.""" + + matched = any(low <= character <= high for low, high in self.ranges) + return matched != self.negated + + @dataclass(frozen=True) class DockerContextEntry: """One file or directory exposed by a :class:`DockerContext`. @@ -87,6 +100,7 @@ class _ContextSelection: source: str kind: Literal["literal_file", "literal_directory", "dot", "wildcard"] entries: tuple[_SelectedContextEntry, ...] + top_level_source_count: int @property def has_directories(self) -> bool: @@ -140,6 +154,7 @@ def select(self, source: str) -> _ContextSelection: _selected_entry(entry, entry.path) for entry in self._visible_entries ), + top_level_source_count=1, ) if _has_wildcard(normalized): return self._select_wildcard(normalized) @@ -158,6 +173,7 @@ def _select_literal(self, source: str) -> _ContextSelection: source=source, kind="literal_file", entries=(_selected_entry(entry, posixpath.basename(source)),), + top_level_source_count=1, ) entries = tuple( @@ -174,18 +190,24 @@ def _select_literal(self, source: str) -> _ContextSelection: source=source, kind="literal_directory", entries=_validate_selection_entries(entries), + top_level_source_count=1, ) def _select_wildcard(self, source: str) -> _ContextSelection: + pattern = _compile_source_pattern(source) matched_entries = [ - entry for entry in self._raw_entries if _glob_matches(source, entry.path) + entry for entry in self._raw_entries if _glob_matches(pattern, entry.path) ] if not matched_entries: raise DockerContextError(f"context source has no match: {source!r}") top_directories: list[str] = [] for entry in sorted( - (entry for entry in matched_entries if entry.kind == "directory"), + ( + entry + for entry in self._visible_entries + if entry.kind == "directory" and _glob_matches(pattern, entry.path) + ), key=lambda item: (item.path.count("/"), item.path), ): if not any( @@ -205,14 +227,17 @@ def _select_wildcard(self, source: str) -> _ContextSelection: ) entries.append(_selected_entry(entry, target)) - for entry in self._visible_entries: - if not _glob_matches(source, entry.path): - continue - if any( - entry.path == directory or entry.path.startswith(f"{directory}/") + top_level_files = [ + entry + for entry in self._visible_entries + if entry.kind == "file" + and _glob_matches(pattern, entry.path) + and not any( + entry.path.startswith(f"{directory}/") for directory in top_directories - ): - continue + ) + ] + for entry in top_level_files: entries.append(_selected_entry(entry, posixpath.basename(entry.path))) if not entries: @@ -221,6 +246,7 @@ def _select_wildcard(self, source: str) -> _ContextSelection: source=source, kind="wildcard", entries=_validate_selection_entries(entries), + top_level_source_count=len(top_directories) + len(top_level_files), ) @@ -412,19 +438,131 @@ def _has_wildcard(path: str) -> bool: return any(character in path for character in "*?[") -def _glob_matches(pattern: str, path: str) -> bool: - """Match Docker source patterns one POSIX path segment at a time.""" +def _compile_source_pattern( + pattern: str, +) -> tuple[tuple[str | _CharacterClass, ...], ...]: + """Compile the strict no-escape subset of Go filepath-style source globs. + + Source normalization rejects backslashes, so escaped pattern syntax is not + supported. + """ + + return tuple( + _compile_glob_segment(segment, pattern) for segment in pattern.split("/") + ) + + +def _compile_glob_segment( + segment: str, source: str +) -> tuple[str | _CharacterClass, ...]: + """Compile one source pattern segment before context entries are inspected.""" + + tokens: list[str | _CharacterClass] = [] + index = 0 + while index < len(segment): + character = segment[index] + if character == "[": + character_class, index = _parse_character_class(segment, index, source) + tokens.append(character_class) + continue + tokens.append(character) + index += 1 + return tuple(tokens) + + +def _parse_character_class( + segment: str, index: int, source: str +) -> tuple[_CharacterClass, int]: + """Parse one non-empty Go filepath-style character class without escapes.""" + + index += 1 + negated = index < len(segment) and segment[index] == "^" + if negated: + index += 1 + + ranges: list[tuple[str, str]] = [] + while True: + if index >= len(segment): + raise DockerContextError(f"malformed context source pattern: {source!r}") + if segment[index] == "]": + if not ranges: + raise DockerContextError( + f"malformed context source pattern: {source!r}" + ) + return _CharacterClass(negated, tuple(ranges)), index + 1 + + low = segment[index] + if low == "-": + raise DockerContextError(f"malformed context source pattern: {source!r}") + index += 1 + high = low + if index < len(segment) and segment[index] == "-": + index += 1 + if index >= len(segment) or segment[index] in "-]": + raise DockerContextError( + f"malformed context source pattern: {source!r}" + ) + high = segment[index] + index += 1 + ranges.append((low, high)) + + +def _glob_matches( + pattern_segments: tuple[tuple[str | _CharacterClass, ...], ...], path: str +) -> bool: + """Match a compiled source pattern one POSIX path segment at a time.""" - pattern_segments = pattern.split("/") path_segments = path.split("/") return len(pattern_segments) == len(path_segments) and all( - fnmatch.fnmatchcase(path_segment, pattern_segment) + _glob_segment_matches(pattern_segment, path_segment) for pattern_segment, path_segment in zip( pattern_segments, path_segments, strict=True ) ) +def _glob_segment_matches( + pattern: tuple[str | _CharacterClass, ...], path: str +) -> bool: + """Match one source segment; stars, questions, and classes never cross '/'.""" + + pattern_index = 0 + path_index = 0 + star_index: int | None = None + star_path_index = 0 + while path_index < len(path): + if pattern_index < len(pattern): + token = pattern[pattern_index] + if token == "*": + star_index = pattern_index + star_path_index = path_index + pattern_index += 1 + continue + if token == "?": + pattern_index += 1 + path_index += 1 + continue + if _glob_token_matches(token, path[path_index]): + pattern_index += 1 + path_index += 1 + continue + if star_index is None: + return False + star_path_index += 1 + path_index = star_path_index + pattern_index = star_index + 1 + + return all(token == "*" for token in pattern[pattern_index:]) + + +def _glob_token_matches(token: str | _CharacterClass, character: str) -> bool: + """Return whether one literal or character class token matches a codepoint.""" + + if isinstance(token, _CharacterClass): + return token.matches(character) + return token == character + + def _validate_selection_entries( entries: tuple[_SelectedContextEntry, ...] | list[_SelectedContextEntry], ) -> tuple[_SelectedContextEntry, ...]: diff --git a/sdk/python/akernel_sdk/_dockerfile_runner.py b/sdk/python/akernel_sdk/_dockerfile_runner.py index 787dd1c..a7250f2 100644 --- a/sdk/python/akernel_sdk/_dockerfile_runner.py +++ b/sdk/python/akernel_sdk/_dockerfile_runner.py @@ -292,6 +292,17 @@ def prepare_copy( plans: list[_PlannedCopy] = [] for source, selection in selections: + if ( + selection.kind == "wildcard" + and selection.top_level_source_count > 1 + and not ins.dest.endswith("/") + ): + raise DockerfileBuildError( + f"{instruction} source {source!r} expands to multiple sources; " + "destination must end in '/'", + index=index, + instruction=instruction, + ) directory_target = ( selection.kind in ("literal_directory", "dot") or selection.has_directories diff --git a/sdk/python/tests/unit/test_dockercontext.py b/sdk/python/tests/unit/test_dockercontext.py index c7b1244..b8242ae 100644 --- a/sdk/python/tests/unit/test_dockercontext.py +++ b/sdk/python/tests/unit/test_dockercontext.py @@ -197,6 +197,73 @@ def test_wildcards_directories_and_collisions(self) -> None: with self.assertRaisesRegex(DockerContextError, "colliding"): manifest.select("*/*.py") + def test_wildcard_character_classes_follow_go_filepath_semantics(self) -> None: + manifest = _ContextManifest.from_context( + MemoryDockerContext( + { + "!.txt": b"", + "a.txt": b"", + "b.txt": b"", + "c.txt": b"", + "é.txt": b"", + "nested/x.txt": b"", + "nested/deep/y.txt": b"", + } + ) + ) + self.assertEqual( + paths(manifest.select("[!a].txt")), + [("!.txt", "!.txt"), ("a.txt", "a.txt")], + ) + self.assertEqual( + paths(manifest.select("[^a].txt")), + [ + ("!.txt", "!.txt"), + ("b.txt", "b.txt"), + ("c.txt", "c.txt"), + ("é.txt", "é.txt"), + ], + ) + self.assertEqual( + paths(manifest.select("[a-c].txt")), + [("a.txt", "a.txt"), ("b.txt", "b.txt"), ("c.txt", "c.txt")], + ) + self.assertEqual( + paths(manifest.select("?.txt")), + [ + ("!.txt", "!.txt"), + ("a.txt", "a.txt"), + ("b.txt", "b.txt"), + ("c.txt", "c.txt"), + ("é.txt", "é.txt"), + ], + ) + self.assertEqual( + paths(manifest.select("**.txt")), paths(manifest.select("*.txt")) + ) + self.assertEqual( + paths(manifest.select("*/?.txt")), [("nested/x.txt", "x.txt")] + ) + + def test_malformed_wildcard_patterns_fail_closed_before_file_reads(self) -> None: + for files in ({}, {"safe.txt": b""}): + for pattern in ( + "file[.txt", + "[]", + "[^]", + "[a-]", + "[-a]", + "[a-b-c]", + ): + with self.subTest(files=files, pattern=pattern): + context = MemoryDockerContext(files) + manifest = _ContextManifest.from_context(context) + with self.assertRaisesRegex( + DockerContextError, "malformed context source pattern" + ): + manifest.select(pattern) + self.assertEqual(context.open_paths, []) + def test_wildcard_no_match_and_only_ignored(self) -> None: manifest = _ContextManifest.from_context( MemoryDockerContext( diff --git a/sdk/python/tests/unit/test_dockerfile.py b/sdk/python/tests/unit/test_dockerfile.py index 5dde7eb..00e17d8 100644 --- a/sdk/python/tests/unit/test_dockerfile.py +++ b/sdk/python/tests/unit/test_dockerfile.py @@ -776,6 +776,81 @@ def test_wildcard_directory_copy_targets_match_buildkit(self) -> None: self.assertEqual(collision_sandbox.files.ops, []) self.assertEqual(collision_sandbox.commands.ops, []) + def test_wildcard_multiple_sources_require_directory_destination(self) -> None: + rejected_cases = ( + ("two directories", {"one/a": b"a", "two/b": b"b"}), + ("two files", {"one": b"one", "two": b"two"}), + ("directory and file", {"dir/a": b"a", "file": b"file"}), + ) + for name, files in rejected_cases: + with self.subTest(name=name): + context = _MemoryDockerContext("FROM ubuntu\nCOPY * /target\n", files) + sandbox = _MockSandbox() + with self.assertRaisesRegex(DockerfileBuildError, "multiple sources"): + apply_dockerfile( + sandbox, + parse_dockerfile(context), + context, + auto_start_cmd=False, + ) + self.assertEqual(sandbox.files.ops, []) + self.assertEqual(sandbox.commands.ops, []) + self.assertEqual(context.open_paths, []) + + valid_cases = ( + ( + "two directories", + {"one/a": b"a", "two/b": b"b"}, + ["/target/a", "/target/b"], + ), + ( + "two files", + {"one": b"one", "two": b"two"}, + ["/target/one", "/target/two"], + ), + ( + "directory and file", + {"dir/a": b"a", "file": b"file"}, + ["/target/a", "/target/file"], + ), + ) + for name, files, targets in valid_cases: + with self.subTest(name=name): + sandbox, _ = self._apply_memory( + "FROM ubuntu\nCOPY * /target/\n", files + ) + self.assertEqual( + [ + operation[2] + for operation in sandbox.files.ops + if operation[0] == "cp" + ], + targets, + ) + + single_directory, _ = self._apply_memory( + "FROM ubuntu\nCOPY * /target\n", {"one/a": b"a"} + ) + self.assertEqual( + [ + operation[2] + for operation in single_directory.files.ops + if operation[0] == "cp" + ], + ["/target/a"], + ) + single_file, _ = self._apply_memory( + "FROM ubuntu\nCOPY * /target\n", {"one": b"one"} + ) + self.assertEqual( + [ + operation[2] + for operation in single_file.files.ops + if operation[0] == "cp" + ], + ["/target"], + ) + def test_local_traversal_failure_prevents_copy_side_effects(self) -> None: with tempfile.TemporaryDirectory() as directory: blocked = Path(directory, "blocked") @@ -823,6 +898,7 @@ def test_selection_and_target_errors_have_no_sandbox_operations(self) -> None: ("COPY /absolute /dest/\n", {"absolute": b""}), ("COPY ../parent /dest/\n", {"parent": b""}), ("COPY *.py /dest\n", {"a.py": b"", "b.py": b""}), + ("COPY file[.txt /dest/\n", {}), ("COPY one/a two/a /dest/\n", {"one/a": b"", "two/a": b""}), ("COPY . /dest/\n", {}), ( From 178394d38b67a8eeb8857184145687e2944c6ed8 Mon Sep 17 00:00:00 2001 From: Peng-YM Date: Tue, 18 Aug 2026 12:38:27 +0800 Subject: [PATCH 05/12] ci: test Dockerfile launch in standalone Include the maintained Dockerfile direct-launch example in the all-in-one standalone job introduced on main. This makes the rebased pull request exercise parsing, context transfer, in-sandbox build instructions, startup command handling, ownership, archive extraction, and fail-closed prechecks in the same CI environment used for the other SDK examples. Signed-off-by: Peng-YM --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5cc7701..f6b8cd9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -151,6 +151,7 @@ jobs: basic_usage.py command_stdin.py custom_image.py + dockerfile_launch.py named_sandbox.py network_policy.py port_forwarding.py From 0cffaa2871ad70fa17252abaf6a69d81ae2fcdfa Mon Sep 17 00:00:00 2001 From: Peng-YM Date: Tue, 18 Aug 2026 18:05:31 +0800 Subject: [PATCH 06/12] fix(sdk): align Docker ignore semantics Replace GitIgnoreSpec with a fail-closed matcher that follows Moby ignore-file preprocessing, ordered parent matching, embedded double-star behavior, path cleaning, and supported character classes. Reject RE2-only escapes instead of treating them as literals so excluded secrets cannot become visible. Keep directory ancestors as virtual sources when later negations re-include descendants, while retaining one top-level wildcard count. Reuse parent match results with a depth-bounded directory stack to avoid repeated matching and unbounded entry-by-pattern storage. Remove the pathspec runtime dependency, document the supported boundary and licensing, and add regressions for Moby parity, virtual literal and wildcard directories, unsupported escapes, prefix-sibling ordering, and bounded parent-result reuse. Signed-off-by: Peng-YM --- sdk/python/README.md | 44 +- sdk/python/akernel_sdk/_dockercontext.py | 423 ++++++++++++++++++-- sdk/python/pyproject.toml | 1 - sdk/python/tests/unit/test_dockercontext.py | 146 +++++++ sdk/python/tests/unit/test_dockerfile.py | 22 + 5 files changed, 580 insertions(+), 56 deletions(-) diff --git a/sdk/python/README.md b/sdk/python/README.md index c47e10f..0ac8e09 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -165,21 +165,30 @@ or `directory`, and a permission-bit `mode` from `0o000` through `0o777`; custom contexts must expose files and every ancestor or empty directory. `LocalDockerContext` uses a local directory and rejects symbolic links. Before any sandbox operation, direct launch walks and validates the manifest, applies -root `.dockerignore` rules to files and directories, excludes `Dockerfile` and -`.dockerignore` from selection, plans every `COPY`/`ADD`, and materializes all -selected files. Selected directory entries create empty and nested directories, -while each copied child file or directory has its own non-recursive mode -restored. For a literal directory source, the source root itself is only a -content container: the destination is created when needed, but does not inherit -that source-root mode. A wildcard that matches a directory likewise copies its -contents, rather than the matched directory name. Wildcard source patterns -follow Go `filepath.Match`-style segment matching, so `**` has the same -one-segment behavior as `*`; `[^a]` negates a character class while `[!a]` -matches `!` or `a`. Backslash escapes are outside this strict subset, and -malformed classes fail closed. When one wildcard expands to multiple top-level -sources after `.dockerignore` filtering, the destination must end in `/`. Paths -and target collisions fail closed. `--chown` affects only files and directories -created by the current instruction. Local tar `ADD` accepts only regular files +Moby-compatible root `.dockerignore` preprocessing and ordered matching to files +and directories, excludes `Dockerfile` and `.dockerignore` from selection, +plans every `COPY`/`ADD`, and materializes all selected files. Docker ignore +patterns are cleaned like `filepath.Clean`, comments require `#` in column one, +embedded `**` can span directories, and later `!` patterns can re-include +descendants. Alphanumeric backslash escapes and nested POSIX character +classes that Moby routes through its regular-expression engine are rejected +rather than treated as literals. When a descendant is re-included beneath an +ignored directory, its +required directory ancestors remain selectable as virtual source directories; +fully ignored directories remain unavailable. Selected directory entries create +empty and nested directories, while each copied child file or directory has its +own non-recursive mode restored. For a literal directory source, the source root +itself is only a content container: the destination is created when needed, but +does not inherit that source-root mode. A wildcard that matches a directory +likewise copies its contents, rather than the matched directory name. Dockerfile +`COPY`/`ADD` source patterns follow Go `filepath.Match`-style segment +matching, so `**` has the same one-segment behavior as `*`; `[^a]` +negates a character class while `[!a]` matches `!` or `a`. Backslash +escapes are outside this strict source-pattern subset, and malformed classes fail +closed. When one wildcard expands to multiple top-level sources after +`.dockerignore` filtering, the destination must end in `/`. Paths and target +collisions fail closed. `--chown` affects only files and directories created by +the current instruction. Local tar `ADD` accepts only regular files and directories with safe paths, preserves tar member metadata, and always extracts as the builder/root identity before applying `--chown`. Remote `ADD` URLs are rejected; the SDK never fetches them. Secure local context opening @@ -200,8 +209,9 @@ the desired command explicitly with `sandbox.commands.run(..., background=True)` image configuration does not auto-start a process in this SDK path. Parsing uses [`dockerfile-parse`](https://github.com/containerbuildsystem/dockerfile-parse) -(BSD-3-Clause), and context matching uses [`pathspec`](https://github.com/cpburnz/python-path-specification) -(MIT). The value post-processing approach was informed by the +(BSD-3-Clause). `.dockerignore` behavior follows +[`moby/patternmatcher`](https://github.com/moby/patternmatcher) (Apache-2.0). +The value post-processing approach was informed by the [E2B Python SDK](https://github.com/e2b-dev/E2B) (MIT). ### Experimental GPU and writable storage diff --git a/sdk/python/akernel_sdk/_dockercontext.py b/sdk/python/akernel_sdk/_dockercontext.py index 7053764..d6f5048 100644 --- a/sdk/python/akernel_sdk/_dockercontext.py +++ b/sdk/python/akernel_sdk/_dockercontext.py @@ -33,8 +33,6 @@ from pathlib import Path from typing import BinaryIO, Literal -from pathspec import GitIgnoreSpec - _SECURE_OPEN_SUPPORTED = ( os.open in getattr(os, "supports_dir_fd", ()) and hasattr(os, "O_DIRECTORY") @@ -130,12 +128,9 @@ def from_context(cls, context: DockerContext) -> _ContextManifest: raw_files = tuple( entry.path for entry in raw_entries if entry.kind == "file" ) - ignore_spec = _read_ignore_spec(context, raw_files) - visible_entries = tuple( - entry - for entry in raw_entries - if not _is_reserved_root_path(entry.path) - and not ignore_spec.match_file(entry.path) + ignore_matcher = _read_ignore_matcher(context, raw_files) + visible_entries = _visible_entries_with_ancestors( + raw_entries, ignore_matcher ) return cls(raw_entries, visible_entries) @@ -343,68 +338,420 @@ def _validate_walk_entries(entries: tuple[object, ...]) -> None: ) parent = posixpath.dirname(parent) + def _is_reserved_root_path(path: str) -> bool: """Return whether a root metadata file is never selectable.""" return path.lower() == "dockerfile" or path == ".dockerignore" -def _read_ignore_spec( +# The matcher below adapts moby/patternmatcher commit 5a6d8429a19b +# (Apache-2.0) to keep Docker context filtering backend-neutral. +@dataclass(frozen=True) +class _DockerIgnoreToken: + """One token in a Moby-compatible ignore pattern.""" + + kind: Literal[ + "literal", + "star", + "question", + "globstar", + "globstar_directory", + "class", + "anchor", + ] + value: str | _CharacterClass | None = None + + +@dataclass(frozen=True) +class _DockerIgnorePattern: + """One precompiled Moby patternmatcher-compatible ignore pattern.""" + + value: str + exclusion: bool + match_type: Literal["exact", "prefix", "suffix", "tokens"] + tokens: tuple[_DockerIgnoreToken, ...] = () + + def matches(self, path: str) -> bool: + """Match one context path using Moby's optimized pattern forms.""" + + if self.match_type == "exact": + return path == self.value + if self.match_type == "prefix": + return path.startswith(self.value[:-2]) + if self.match_type == "suffix": + suffix = self.value[2:] + return path.endswith(suffix) or ( + suffix.startswith("/") and path == suffix[1:] + ) + return _ignore_tokens_match(self.tokens, path) + + +@dataclass(frozen=True) +class _DockerIgnoreMatcher: + """Ordered Docker ignore patterns with parent-directory matching.""" + + patterns: tuple[_DockerIgnorePattern, ...] + + @classmethod + def from_lines(cls, lines: list[str]) -> _DockerIgnoreMatcher: + return cls(tuple(_compile_ignore_pattern(line) for line in lines)) + + def is_ignored(self, path: str) -> bool: + """Return Moby MatchesOrParentMatches result for a context path.""" + + matched = False + segments = path.split("/") + parents = tuple( + "/".join(segments[:index]) for index in range(1, len(segments)) + ) + for pattern in self.patterns: + if pattern.exclusion != matched: + continue + pattern_matched = pattern.matches(path) or any( + pattern.matches(parent) for parent in parents + ) + if pattern_matched: + matched = not pattern.exclusion + return matched + + def match_with_parent_results( + self, path: str, parent_results: tuple[bool, ...] + ) -> tuple[bool, tuple[bool, ...]]: + """Match a path while reusing per-pattern results from its parent.""" + + if parent_results and len(parent_results) != len(self.patterns): + raise DockerContextError("invalid .dockerignore parent match results") + + matched = False + results = [False] * len(self.patterns) + for index, pattern in enumerate(self.patterns): + pattern_matched = ( + parent_results[index] if parent_results else False + ) + if not pattern_matched: + if pattern.exclusion != matched: + continue + pattern_matched = pattern.matches(path) + results[index] = pattern_matched + if pattern_matched: + matched = not pattern.exclusion + return matched, tuple(results) + + +def _visible_entries_with_ancestors( + raw_entries: tuple[DockerContextEntry, ...], + ignore_matcher: _DockerIgnoreMatcher, +) -> tuple[DockerContextEntry, ...]: + """Keep visible entries plus directory ancestors needed to reach them.""" + + raw_by_path = {entry.path: entry for entry in raw_entries} + active_directories: list[tuple[str, tuple[bool, ...]]] = [] + visible_paths: set[str] = set() + tree_entries = sorted( + raw_entries, key=lambda entry: tuple(entry.path.split("/")) + ) + for entry in tree_entries: + while active_directories and not entry.path.startswith( + f"{active_directories[-1][0]}/" + ): + active_directories.pop() + + parent = posixpath.dirname(entry.path) + if parent: + if not active_directories or active_directories[-1][0] != parent: + raise DockerContextError( + f"context walk omitted directory entry: {parent!r}" + ) + parent_results = active_directories[-1][1] + else: + parent_results = () + + ignored, match_results = ignore_matcher.match_with_parent_results( + entry.path, parent_results + ) + if entry.kind == "directory": + active_directories.append((entry.path, match_results)) + if not _is_reserved_root_path(entry.path) and not ignored: + visible_paths.add(entry.path) + for path in tuple(visible_paths): + parent = posixpath.dirname(path) + while parent not in ("", "."): + ancestor = raw_by_path[parent] + if ancestor.kind != "directory": + raise DockerContextError( + f"context walk yielded a file-as-ancestor path: {parent!r}" + ) + visible_paths.add(parent) + parent = posixpath.dirname(parent) + return tuple(entry for entry in raw_entries if entry.path in visible_paths) + + +def _read_ignore_matcher( context: DockerContext, raw_files: tuple[str, ...] -) -> GitIgnoreSpec: +) -> _DockerIgnoreMatcher: """Read and compile the root .dockerignore when it was walked.""" if ".dockerignore" not in raw_files: - return GitIgnoreSpec.from_lines(()) + return _DockerIgnoreMatcher(()) try: with context.open(".dockerignore") as stream: content = stream.read() lines = _prepare_ignore_lines(content.decode("utf-8-sig")) - return GitIgnoreSpec.from_lines(lines) + return _DockerIgnoreMatcher.from_lines(lines) except DockerContextError: raise except Exception as error: raise DockerContextError("failed to read .dockerignore") from error +_MOBY_TRIM_SPACE = ( + " \t\n\v\f\r\u0085\u00a0\u1680" + "\u2000\u2001\u2002\u2003\u2004\u2005" + "\u2006\u2007\u2008\u2009\u200a\u2028" + "\u2029\u202f\u205f\u3000" +) + + +def _trim_moby_space(value: str) -> str: + """Trim exactly the Unicode White_Space set used by Go.""" + + return value.strip(_MOBY_TRIM_SPACE) + + def _prepare_ignore_lines(content: str) -> list[str]: - """Normalize common Docker .dockerignore syntax for GitIgnoreSpec.""" + """Apply Moby ignorefile preprocessing to .dockerignore content.""" lines: list[str] = [] - for raw_line in content.splitlines(): - line = raw_line.strip() - if not line or line == ".": + for line_index, raw_line in enumerate(content.split("\n")): + if line_index == 0: + raw_line = raw_line.removeprefix("\ufeff") + if raw_line.endswith("\r"): + raw_line = raw_line[:-1] + if raw_line.startswith("#"): continue - if line.startswith("#"): + + pattern = _trim_moby_space(raw_line) + if not pattern: continue - negation = "" - pattern = line - if pattern.startswith("!"): - negation = "!" - pattern = pattern[1:] + invert = pattern.startswith("!") + if invert: + pattern = _trim_moby_space(pattern[1:]) + if pattern: + pattern = _clean_posix_pattern(pattern) + if len(pattern) > 1 and pattern.startswith("/"): + pattern = pattern[1:] + lines.append(("!" if invert else "") + pattern) + return lines - while pattern.startswith("/"): - pattern = pattern[1:] - while pattern.endswith("/") and not _is_escaped(pattern, len(pattern) - 1): - pattern = pattern[:-1] - if not pattern or pattern == ".": - continue - lines.append(f"{negation}{pattern}") - return lines +def _clean_posix_pattern(pattern: str) -> str: + """Return the Unix filepath.Clean equivalent used by Moby.""" + + cleaned = posixpath.normpath(pattern) + if cleaned.startswith("//"): + cleaned = "/" + cleaned.lstrip("/") + return cleaned + + +def _compile_ignore_pattern(line: str) -> _DockerIgnorePattern: + """Compile one preprocessed pattern with Moby patternmatcher semantics.""" + cleaned = _clean_posix_pattern(_trim_moby_space(line)) + exclusion = cleaned.startswith("!") + value = cleaned[1:] if exclusion else cleaned + if not value: + raise DockerContextError("invalid .dockerignore pattern: '!'") + if "\0" in value: + raise DockerContextError("invalid .dockerignore pattern containing NUL") -def _is_escaped(value: str, index: int) -> bool: - """Return whether a character is preceded by an odd number of backslashes.""" + tokens: list[_DockerIgnoreToken] = [] + match_type: Literal["exact", "prefix", "suffix", "tokens"] = "exact" + index = 0 + token_index = 0 + while index < len(value): + character = value[index] + if character == "*": + if index + 1 < len(value) and value[index + 1] == "*": + index += 2 + if index < len(value) and value[index] == "/": + index += 1 + if index == len(value): + if match_type == "exact": + match_type = "prefix" + else: + tokens.append(_DockerIgnoreToken("globstar")) + match_type = "tokens" + else: + tokens.append(_DockerIgnoreToken("globstar_directory")) + match_type = "tokens" + if token_index == 0: + match_type = "suffix" + else: + tokens.append(_DockerIgnoreToken("star")) + match_type = "tokens" + index += 1 + elif character == "?": + tokens.append(_DockerIgnoreToken("question")) + match_type = "tokens" + index += 1 + elif character == "[": + character_class, index = _compile_ignore_character_class( + value, index, line + ) + tokens.append(_DockerIgnoreToken("class", character_class)) + match_type = "tokens" + elif character == "\\": + index += 1 + if index >= len(value): + raise DockerContextError( + f"invalid .dockerignore pattern: {line!r}" + ) + # Moby passes alphanumeric escapes through to RE2. Reject + # them rather than silently downgrading their matching semantics. + if ( + value[index] == "/" + or value[index].isalnum() + or value[index] == "_" + ): + raise DockerContextError( + f"unsupported .dockerignore escape: {line!r}" + ) + tokens.append(_DockerIgnoreToken("literal", value[index])) + match_type = "tokens" + index += 1 + else: + kind: Literal["literal", "anchor"] = ( + "anchor" if character == "^" else "literal" + ) + tokens.append(_DockerIgnoreToken(kind, character)) + index += 1 + token_index += 1 + + return _DockerIgnorePattern(value, exclusion, match_type, tuple(tokens)) - count = 0 - index -= 1 - while index >= 0 and value[index] == "\\": - count += 1 - index -= 1 - return count % 2 == 1 + +def _compile_ignore_character_class( + pattern: str, index: int, original: str +) -> tuple[_CharacterClass, int]: + """Compile one strict Go filepath-compatible ignore character class.""" + + index += 1 + negated = index < len(pattern) and pattern[index] == "^" + if negated: + index += 1 + + ranges: list[tuple[str, str]] = [] + while True: + if index >= len(pattern) or pattern[index] == "]": + if index < len(pattern) and ranges: + index += 1 + break + raise DockerContextError( + f"invalid .dockerignore pattern: {original!r}" + ) + + low, index, low_escaped = _read_ignore_class_character( + pattern, index, original + ) + if low == "-" and not low_escaped: + raise DockerContextError( + f"invalid .dockerignore pattern: {original!r}" + ) + high = low + if index < len(pattern) and pattern[index] == "-": + index += 1 + if index >= len(pattern) or pattern[index] == "]": + raise DockerContextError( + f"invalid .dockerignore pattern: {original!r}" + ) + high, index, _ = _read_ignore_class_character( + pattern, index, original + ) + if ord(high) < ord(low): + raise DockerContextError( + f"invalid .dockerignore pattern: {original!r}" + ) + ranges.append((low, high)) + + return _CharacterClass(negated, tuple(ranges)), index + + +def _read_ignore_class_character( + pattern: str, index: int, original: str +) -> tuple[str, int, bool]: + """Read one literal or escaped character inside an ignore class.""" + + escaped = pattern[index] == "\\" + if escaped: + index += 1 + if index >= len(pattern): + raise DockerContextError( + f"invalid .dockerignore pattern: {original!r}" + ) + character = pattern[index] + if ( + character == "[" + or (escaped and (character.isalnum() or character in "_/")) + ): + raise DockerContextError( + f"unsupported .dockerignore character class: {original!r}" + ) + return character, index + 1, escaped + + +def _ignore_tokens_match( + tokens: tuple[_DockerIgnoreToken, ...], path: str +) -> bool: + """Match compiled tokens in O(pattern length * path length) time.""" + + previous = [False] * (len(path) + 1) + previous[0] = True + for token in tokens: + current = [False] * (len(path) + 1) + if token.kind == "star": + current[0] = previous[0] + for path_index in range(1, len(path) + 1): + current[path_index] = previous[path_index] or ( + path[path_index - 1] != "/" and current[path_index - 1] + ) + elif token.kind == "globstar": + current[0] = previous[0] + for path_index in range(1, len(path) + 1): + current[path_index] = ( + previous[path_index] or current[path_index - 1] + ) + elif token.kind == "globstar_directory": + previous_prefix = False + for path_index in range(len(path) + 1): + current[path_index] = previous[path_index] + if path_index > 0: + previous_prefix = ( + previous_prefix or previous[path_index - 1] + ) + if path[path_index - 1] == "/" and previous_prefix: + current[path_index] = True + elif token.kind == "anchor": + current[0] = previous[0] + elif token.kind in ("question", "literal", "class"): + for path_index in range(1, len(path) + 1): + character = path[path_index - 1] + token_matches = False + if token.kind == "question": + token_matches = character != "/" + elif token.kind == "literal": + token_matches = character == token.value + elif isinstance(token.value, _CharacterClass): + token_matches = token.value.matches(character) + current[path_index] = ( + previous[path_index - 1] and token_matches + ) + else: + raise DockerContextError("invalid compiled .dockerignore token") + previous = current + return previous[len(path)] def _normalize_source(source: str) -> str: diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index 683ca14..bd1d209 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -31,7 +31,6 @@ dependencies = [ "openyuanrong-sandbox==0.9.7", "websockets>=10.0", "dockerfile-parse>=2.0.1", - "pathspec>=0.12,<2", ] [project.optional-dependencies] diff --git a/sdk/python/tests/unit/test_dockercontext.py b/sdk/python/tests/unit/test_dockercontext.py index b8242ae..a27b655 100644 --- a/sdk/python/tests/unit/test_dockercontext.py +++ b/sdk/python/tests/unit/test_dockercontext.py @@ -13,6 +13,7 @@ from typing import BinaryIO from unittest.mock import patch +import akernel_sdk._dockercontext as dockercontext_module from akernel_sdk._dockercontext import ( DockerContext, DockerContextEntry, @@ -136,6 +137,151 @@ def test_dockerignore_semantics_and_reserved_files(self) -> None: manifest.select(source) self.assertEqual(context.open_paths, [".dockerignore"]) + def test_dockerignore_moby_preprocessing_and_embedded_double_star(self) -> None: + context = MemoryDockerContext( + { + ".dockerignore": ( + b"# column-one comment\n" + b" #secret\n" + b"a**/*.txt\n" + b"foo/../cleaned-secret\n" + b"[^/]*.pem\n" + b"foo[/]bar\n" + ), + "#secret": b"hidden", + "a/file.txt": b"hidden", + "a/dir/dir/secret.txt": b"hidden", + "a/keep.bin": b"visible", + "cleaned-secret": b"hidden", + "foo/bar": b"hidden", + "nested/root.pem": b"visible", + "root.pem": b"hidden", + "safe": b"visible", + } + ) + manifest = _ContextManifest.from_context(context) + self.assertEqual( + paths(manifest.select(".")), + [ + ("a/keep.bin", "a/keep.bin"), + ("nested/root.pem", "nested/root.pem"), + ("safe", "safe"), + ], + ) + for source in ( + "#secret", + "a/file.txt", + "a/dir/dir/secret.txt", + "cleaned-secret", + "foo/bar", + "root.pem", + ): + with self.subTest(source=source): + with self.assertRaisesRegex(DockerContextError, "ignored"): + manifest.select(source) + self.assertEqual(context.open_paths, [".dockerignore"]) + + def test_reincluded_descendants_make_virtual_source_directories(self) -> None: + context = MemoryDockerContext( + { + ".dockerignore": ( + b"docs\n" + b"!docs/README.md\n" + b"!docs/nested/guide.md\n" + ), + "docs/README.md": b"visible", + "docs/nested/guide.md": b"visible", + "docs/private.txt": b"hidden", + } + ) + manifest = _ContextManifest.from_context(context) + expected = [ + ("docs", "", "directory"), + ("docs/README.md", "README.md", "file"), + ("docs/nested", "nested", "directory"), + ("docs/nested/guide.md", "nested/guide.md", "file"), + ] + literal = manifest.select("docs") + self.assertEqual( + [ + (entry.source_path, entry.relative_target, entry.kind) + for entry in literal.entries + ], + expected, + ) + wildcard = manifest.select("*") + self.assertEqual( + [ + (entry.source_path, entry.relative_target, entry.kind) + for entry in wildcard.entries + ], + expected, + ) + self.assertEqual(wildcard.top_level_source_count, 1) + with self.assertRaisesRegex(DockerContextError, "ignored"): + manifest.select("docs/private.txt") + self.assertEqual(context.open_paths, [".dockerignore"]) + + def test_malformed_dockerignore_patterns_fail_closed(self) -> None: + for pattern in (b"!\n", b"[\n", b"trailing\\\n", b"[z-a]\n"): + with self.subTest(pattern=pattern): + context = MemoryDockerContext( + {".dockerignore": pattern, "safe": b"visible"} + ) + with self.assertRaisesRegex( + DockerContextError, "invalid .dockerignore pattern" + ): + _ContextManifest.from_context(context) + self.assertEqual(context.open_paths, [".dockerignore"]) + + + def test_unsupported_dockerignore_regex_escapes_fail_closed(self) -> None: + cases = ( + (br"\d*.pem" + NL, "unsupported .dockerignore escape"), + (br"\s*" + NL, "unsupported .dockerignore escape"), + (br"\x41*" + NL, "unsupported .dockerignore escape"), + (br"\foo" + NL, "unsupported .dockerignore escape"), + (br"[\d]*.pem" + NL, "unsupported .dockerignore character class"), + (b"[[:digit:]]*.pem" + NL, "unsupported .dockerignore character class"), + ) + for pattern, message in cases: + with self.subTest(pattern=pattern): + context = MemoryDockerContext( + {".dockerignore": pattern, "1secret.pem": b"hidden"} + ) + with self.assertRaisesRegex(DockerContextError, message): + _ContextManifest.from_context(context) + self.assertEqual(context.open_paths, [".dockerignore"]) + + def test_parent_match_stack_handles_path_prefix_siblings(self) -> None: + context = MemoryDockerContext( + { + ".dockerignore": b"never-match*\n", + "a/file": b"nested", + "a-b": b"sibling", + } + ) + manifest = _ContextManifest.from_context(context) + self.assertEqual( + paths(manifest.select(".")), + [("a-b", "a-b"), ("a/file", "a/file")], + ) + + def test_manifest_reuses_parent_ignore_results(self) -> None: + nested = "/".join(f"level-{index}" for index in range(40)) + path = f"{nested}/visible.txt" + context = MemoryDockerContext( + {".dockerignore": b"never-match*\n", path: b"visible"} + ) + with patch.object( + dockercontext_module, + "_ignore_tokens_match", + wraps=dockercontext_module._ignore_tokens_match, + ) as token_match: + manifest = _ContextManifest.from_context(context) + self.assertEqual(paths(manifest.select(path)), [(path, "visible.txt")]) + self.assertLessEqual(token_match.call_count, len(context.paths)) + def test_escaped_ignore_patterns_and_invalid_utf8(self) -> None: slash = chr(92).encode() escaped = MemoryDockerContext( diff --git a/sdk/python/tests/unit/test_dockerfile.py b/sdk/python/tests/unit/test_dockerfile.py index 00e17d8..0d4e326 100644 --- a/sdk/python/tests/unit/test_dockerfile.py +++ b/sdk/python/tests/unit/test_dockerfile.py @@ -609,6 +609,28 @@ def test_dot_honors_dockerignore_and_skips_reserved_files(self) -> None: self.assertEqual(copies, ["/app/allowed", "/app/sub/x"]) self.assertEqual(context.open_paths, [".dockerignore", "allowed", "sub/x"]) + def test_reincluded_dockerignore_descendant_copies_from_virtual_dir(self) -> None: + files = { + ".dockerignore": b"docs\n!docs/README.md\n", + "docs/README.md": b"visible", + "docs/private.txt": b"hidden", + } + for source in ("docs", "*"): + with self.subTest(source=source): + sandbox, context = self._apply_memory( + f"FROM ubuntu\nCOPY {source} /out/\n", files + ) + copies = [ + operation[2] + for operation in sandbox.files.ops + if operation[0] == "cp" + ] + self.assertEqual(copies, ["/out/README.md"]) + self.assertEqual( + context.open_paths, + [".dockerignore", "docs/README.md"], + ) + def test_local_and_memory_contexts_have_same_copy_targets(self) -> None: import os import tempfile From 5607365128facb5a20ca8664decb42e50c59b977 Mon Sep 17 00:00:00 2001 From: Peng-YM Date: Tue, 18 Aug 2026 18:05:31 +0800 Subject: [PATCH 07/12] test(sdk): expand Dockerfile standalone coverage Extend the maintained standalone example from five to nine sections. Exercise ignored-directory descendant re-inclusion through COPY dot, literal-directory, and wildcard sources, and verify excluded siblings remain absent. Add fresh-sandbox startup regressions for ENTRYPOINT without CMD, shell-form CMD, shell-form ENTRYPOINT overriding CMD, and auto_start_cmd=False. Each section asserts its result before the context manager destroys the sandbox, and the complete example remains within the CI timeout. Signed-off-by: Peng-YM --- sdk/python/examples/dockerfile_launch.py | 131 +++++++++++++++++++++-- 1 file changed, 123 insertions(+), 8 deletions(-) diff --git a/sdk/python/examples/dockerfile_launch.py b/sdk/python/examples/dockerfile_launch.py index 2d51b9a..eedcbf6 100644 --- a/sdk/python/examples/dockerfile_launch.py +++ b/sdk/python/examples/dockerfile_launch.py @@ -20,11 +20,15 @@ or build cache. Sections: - 1. Core path — filtered COPY ., wildcard directory COPY, modes, empty dirs + 1. Core path, ignore filtering, wildcard COPY, modes, and empty directories 2. Root cwd plus ENTRYPOINT + CMD exec-form combination - 3. COPY --chown ownership - 4. Builder/root ADD after USER - 5. Fail-closed pre-check without a sandbox + 3. Exec-form ENTRYPOINT without CMD + 4. Shell-form CMD + 5. Shell-form ENTRYPOINT ignoring CMD + 6. Disabled automatic startup dispatch + 7. COPY --chown ownership + 8. Builder/root ADD after USER + 9. Fail-closed pre-check without a sandbox """ import tarfile @@ -47,7 +51,10 @@ def section_core_path() -> None: print("\n=== Section 1: filtered core path ===") with tempfile.TemporaryDirectory() as directory: context_dir = Path(directory) - (context_dir / ".dockerignore").write_text("secret.txt\n", encoding="utf-8") + (context_dir / ".dockerignore").write_text( + "secret.txt\ndocs\n!docs/README.md\n", + encoding="utf-8", + ) (context_dir / "secret.txt").write_text("do not upload\n", encoding="utf-8") (context_dir / "app.py").write_text( "import os\n" @@ -57,6 +64,10 @@ def section_core_path() -> None: encoding="utf-8", ) (context_dir / "greeting.txt").write_text("hello\n", encoding="utf-8") + docs = context_dir / "docs" + docs.mkdir() + (docs / "README.md").write_text("visible\n", encoding="utf-8") + (docs / "private.txt").write_text("hidden\n", encoding="utf-8") executable = context_dir / "entrypoint.sh" executable.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") executable.chmod(0o755) @@ -79,6 +90,8 @@ def section_core_path() -> None: COPY empty/ /srv/core/literal-empty/ COPY . /srv/core/ COPY wild/* /srv/wild/ +COPY docs /srv/reincluded-literal/ +COPY doc* /srv/reincluded-wildcard/ CMD ["python3", "/srv/core/app.py"] """ context = LocalDockerContext(dockerfile, context_dir=context_dir) @@ -110,6 +123,15 @@ def section_core_path() -> None: "test -f /srv/wild/dir2/foo && test ! -e /srv/wild/dir1" ) assert wildcard.exit_code == 0, wildcard.stderr + reincluded = sandbox.commands.run( + "test -f /srv/core/docs/README.md " + "&& test ! -e /srv/core/docs/private.txt " + "&& test -f /srv/reincluded-literal/README.md " + "&& test ! -e /srv/reincluded-literal/private.txt " + "&& test -f /srv/reincluded-wildcard/README.md " + "&& test ! -e /srv/reincluded-wildcard/private.txt" + ) + assert reincluded.exit_code == 0, reincluded.stderr print( f" sandbox: {sandbox.id}; marker: {marker.stdout.strip()}; " f"modes: {modes.stdout.splitlines()}" @@ -147,9 +169,98 @@ def section_entrypoint_cmd_merge() -> None: ) +def section_entrypoint_only() -> None: + """Dispatch an exec-form ENTRYPOINT when CMD is absent.""" + print("\n=== Section 3: ENTRYPOINT only ===") + with tempfile.TemporaryDirectory() as directory: + context = LocalDockerContext( + "FROM ubuntu:22.04\n" + 'ENTRYPOINT ["/bin/sh", "-c", ' + '"printf entrypoint-only > /tmp/entrypoint-only.out"]\n', + context_dir=directory, + ) + _precheck(context) + + with Sandbox(context=context) as sandbox: + startup = sandbox.startup_command + assert startup is not None + result = startup.wait(timeout=60) + assert result.exit_code == 0, result.stderr + marker = sandbox.commands.run("cat /tmp/entrypoint-only.out") + assert marker.exit_code == 0, marker.stderr + assert marker.stdout == "entrypoint-only", marker.stdout + print(f" sandbox: {sandbox.id}; output: {marker.stdout}") + + +def section_shell_cmd() -> None: + """Dispatch a shell-form CMD with the declared WORKDIR.""" + print("\n=== Section 4: shell-form CMD ===") + with tempfile.TemporaryDirectory() as directory: + context = LocalDockerContext( + "FROM ubuntu:22.04\nWORKDIR /tmp\nCMD pwd > /tmp/shell-cmd.out\n", + context_dir=directory, + ) + _precheck(context) + + with Sandbox(context=context) as sandbox: + startup = sandbox.startup_command + assert startup is not None + result = startup.wait(timeout=60) + assert result.exit_code == 0, result.stderr + marker = sandbox.commands.run("cat /tmp/shell-cmd.out") + assert marker.exit_code == 0, marker.stderr + assert marker.stdout.strip() == "/tmp", marker.stdout + print(f" sandbox: {sandbox.id}; cwd: {marker.stdout.strip()}") + + +def section_shell_entrypoint_ignores_cmd() -> None: + """Ensure shell-form ENTRYPOINT replaces rather than appends CMD.""" + print("\n=== Section 5: shell ENTRYPOINT ignores CMD ===") + with tempfile.TemporaryDirectory() as directory: + context = LocalDockerContext( + "FROM ubuntu:22.04\n" + "ENTRYPOINT printf shell-entrypoint > /tmp/shell-entrypoint.out\n" + 'CMD ["/bin/sh", "-c", ' + '"printf unexpected > /tmp/cmd-should-not-run.out"]\n', + context_dir=directory, + ) + _precheck(context) + + with Sandbox(context=context) as sandbox: + startup = sandbox.startup_command + assert startup is not None + result = startup.wait(timeout=60) + assert result.exit_code == 0, result.stderr + marker = sandbox.commands.run( + "cat /tmp/shell-entrypoint.out && test ! -e /tmp/cmd-should-not-run.out" + ) + assert marker.exit_code == 0, marker.stderr + assert marker.stdout == "shell-entrypoint", marker.stdout + print(f" sandbox: {sandbox.id}; output: {marker.stdout}") + + +def section_auto_start_disabled() -> None: + """Keep CMD undispatched when auto_start_cmd is disabled.""" + print("\n=== Section 6: auto startup disabled ===") + with tempfile.TemporaryDirectory() as directory: + context = LocalDockerContext( + "FROM ubuntu:22.04\n" + 'CMD ["/bin/sh", "-c", ' + '"printf unexpected > /tmp/disabled-start.out"]\n', + context_dir=directory, + ) + _precheck(context) + + with Sandbox(context=context, auto_start_cmd=False) as sandbox: + assert sandbox.startup_command is None + absent = sandbox.commands.run("test ! -e /tmp/disabled-start.out") + assert absent.exit_code == 0, absent.stderr + print(f" sandbox: {sandbox.id}; startup dispatch: disabled") + + def section_copy_chown() -> None: """Verify COPY --chown without dispatching a startup command.""" - print("\n=== Section 3: COPY --chown ===") + print("\n=== Section 7: COPY --chown ===") with tempfile.TemporaryDirectory() as directory: context_dir = Path(directory) (context_dir / "payload.txt").write_text("owned\n", encoding="utf-8") @@ -171,7 +282,7 @@ def section_copy_chown() -> None: def section_add_tar() -> None: """Verify literal local ADD tar extraction without a startup command.""" - print("\n=== Section 4: ADD local tar ===") + print("\n=== Section 8: ADD local tar ===") with tempfile.TemporaryDirectory() as directory: context_dir = Path(directory) payload = context_dir / "payload" @@ -203,7 +314,7 @@ def section_add_tar() -> None: def section_fail_closed_precheck() -> None: """Reject remote ADD and unsupported USER forms before sandbox creation.""" - print("\n=== Section 5: fail-closed precheck ===") + print("\n=== Section 9: fail-closed precheck ===") cases = ( ( "FROM ubuntu:22.04\nADD https://example.test/app.tar /opt/app/\n", @@ -224,6 +335,10 @@ def section_fail_closed_precheck() -> None: def main() -> None: section_core_path() section_entrypoint_cmd_merge() + section_entrypoint_only() + section_shell_cmd() + section_shell_entrypoint_ignores_cmd() + section_auto_start_disabled() section_copy_chown() section_add_tar() section_fail_closed_precheck() From 90bbf46aca14cb465e9780d081d88da4589f8876 Mon Sep 17 00:00:00 2001 From: Peng-YM Date: Wed, 19 Aug 2026 18:11:10 +0800 Subject: [PATCH 08/12] fix(sdk): align Dockerfile ignore context Honor an existing adjacent .dockerignore before falling back to the root ignore file, including an empty companion. Keep Dockerfile and ignore control files visible to COPY and ADD whenever the active matcher permits them, while preserving inline and external input boundaries and secure no-follow reads.\n\nUpdate the context contract, unit coverage, documentation, and standalone example to verify both behaviors. Signed-off-by: Peng-YM --- AGENTS.md | 11 +- sdk/python/README.md | 19 +- sdk/python/akernel_sdk/_dockercontext.py | 191 ++++++++++++--- sdk/python/examples/dockerfile_launch.py | 59 +++-- sdk/python/tests/unit/test_dockercontext.py | 259 +++++++++++++++++++- sdk/python/tests/unit/test_dockerfile.py | 33 ++- 6 files changed, 489 insertions(+), 83 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1ff6fc7..f7d3c83 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -351,10 +351,13 @@ The SDK also supports a strict Dockerfile direct-launch subset through `DockerContext` and `Sandbox(context=...)`. `FROM` supplies only the root filesystem; inherited OCI image configuration is not applied. `DockerContext.walk()` uses public structured entries for files and directories, including permission -modes and empty directories. Context transfer must remain backend-neutral, honor -`.dockerignore`, and fail closed for syntax outside the documented subset. Keep -the contract, security boundaries, public types, unit tests, SDK README, and -`examples/dockerfile_launch.py` in sync. +modes and empty directories. Context transfer must remain backend-neutral and +fail closed for syntax outside the documented subset. A path-form Dockerfile uses an existing adjacent +`.dockerignore` ahead of the root `.dockerignore`, including an empty +companion; only its absence permits root fallback. Dockerfiles and active or root +ignore files remain ordinary context entries and are selectable unless the active +matcher excludes them. Keep the contract, security boundaries, public types, unit +tests, SDK README, and `examples/dockerfile_launch.py` in sync. When changing a public SDK method, update its type annotations and docstring, add or update unit coverage, and keep the SDK README and maintained examples in diff --git a/sdk/python/README.md b/sdk/python/README.md index 0ac8e09..45a2266 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -163,11 +163,22 @@ A `DockerContext` exposes Dockerfile text plus structured `DockerContextEntry` values from `walk()`. Each entry has a relative POSIX `path`, `kind` of `file` or `directory`, and a permission-bit `mode` from `0o000` through `0o777`; custom contexts must expose files and every ancestor or empty directory. +Custom contexts may implement `dockerfile_ignore()`: returning a +`(diagnostic_name, bytes)` tuple supplies the active matcher, while only +returning `None` falls back to the manifest-root `.dockerignore`; empty `bytes` +still denote a present, higher-priority matcher. Dockerfiles and ignore files +that belong to the filesystem context must still be enumerated by `walk()`. `LocalDockerContext` uses a local directory and rejects symbolic links. Before -any sandbox operation, direct launch walks and validates the manifest, applies -Moby-compatible root `.dockerignore` preprocessing and ordered matching to files -and directories, excludes `Dockerfile` and `.dockerignore` from selection, -plans every `COPY`/`ADD`, and materializes all selected files. Docker ignore +any sandbox operation, direct launch walks and validates the manifest, then applies +Moby-compatible ordered ignore matching to files and directories. For a path-form +Dockerfile, its adjacent `.dockerignore` is the active ignore file when +it exists, including when it is empty. +An inline Dockerfile creates no virtual context file, while a Dockerfile outside the +context and its companion remain outside the manifest even when that companion supplies +the active matcher. Dockerfiles, the root `.dockerignore`, and Dockerfile-specific +ignore files are ordinary context files: `COPY`/`ADD` can select them unless the +active ignore file explicitly excludes them. Direct launch then plans every +`COPY`/`ADD` and materializes all selected files. Docker ignore patterns are cleaned like `filepath.Clean`, comments require `#` in column one, embedded `**` can span directories, and later `!` patterns can re-include descendants. Alphanumeric backslash escapes and nested POSIX character diff --git a/sdk/python/akernel_sdk/_dockercontext.py b/sdk/python/akernel_sdk/_dockercontext.py index d6f5048..451fe1b 100644 --- a/sdk/python/akernel_sdk/_dockercontext.py +++ b/sdk/python/akernel_sdk/_dockercontext.py @@ -270,6 +270,16 @@ class DockerContext(ABC): def dockerfile_text(self) -> str: """Return the Dockerfile content.""" + def dockerfile_ignore(self) -> tuple[str, bytes] | None: + """Return the selected Dockerfile-specific ignore file, if any. + + The returned name is used for diagnostics and the bytes are compiled as + the active ignore matcher. Returning None lets the caller fall back to + the root .dockerignore. + """ + + return None + @abstractmethod @contextmanager def open(self, path: str) -> Iterator[BinaryIO]: @@ -283,9 +293,11 @@ def open(self, path: str) -> Iterator[BinaryIO]: def walk(self) -> Iterator[DockerContextEntry]: """Enumerate every context file and directory as structured entries. - Entries use relative POSIX paths. The Dockerfile itself is not included. - File entries must be readable through :meth:`open`; directory entries - make empty directories and their permission modes representable. + Entries use relative POSIX paths. Control files that belong to the + filesystem context, including Dockerfiles and ignore files, must also + be enumerated; their COPY visibility is determined by the active + ignore matcher. File entries must be readable through :meth:`open`; directory + entries make empty directories and their permission modes representable. """ @@ -339,12 +351,6 @@ def _validate_walk_entries(entries: tuple[object, ...]) -> None: parent = posixpath.dirname(parent) -def _is_reserved_root_path(path: str) -> bool: - """Return whether a root metadata file is never selectable.""" - - return path.lower() == "dockerfile" or path == ".dockerignore" - - # The matcher below adapts moby/patternmatcher commit 5a6d8429a19b # (Apache-2.0) to keep Docker context filtering backend-neutral. @dataclass(frozen=True) @@ -472,7 +478,7 @@ def _visible_entries_with_ancestors( ) if entry.kind == "directory": active_directories.append((entry.path, match_results)) - if not _is_reserved_root_path(entry.path) and not ignored: + if not ignored: visible_paths.add(entry.path) for path in tuple(visible_paths): parent = posixpath.dirname(path) @@ -490,20 +496,53 @@ def _visible_entries_with_ancestors( def _read_ignore_matcher( context: DockerContext, raw_files: tuple[str, ...] ) -> _DockerIgnoreMatcher: - """Read and compile the root .dockerignore when it was walked.""" + """Read and compile the active Dockerfile or root ignore file.""" - if ".dockerignore" not in raw_files: - return _DockerIgnoreMatcher(()) + try: + selected = context.dockerfile_ignore() + except DockerContextError: + raise + except Exception as error: + raise DockerContextError( + "failed to obtain Dockerfile-specific ignore file" + ) from error + + if selected is None: + if ".dockerignore" not in raw_files: + return _DockerIgnoreMatcher(()) + name = ".dockerignore" + try: + with context.open(name) as stream: + content = stream.read() + except DockerContextError: + raise + except Exception as error: + raise DockerContextError(f"failed to read {name}") from error + else: + if ( + type(selected) is not tuple + or len(selected) != 2 + or type(selected[0]) is not str + or not selected[0] + or type(selected[1]) is not bytes + ): + raise DockerContextError( + "dockerfile_ignore() must return None or a " + "(non-empty str, bytes) tuple" + ) + name, content = selected try: - with context.open(".dockerignore") as stream: - content = stream.read() lines = _prepare_ignore_lines(content.decode("utf-8-sig")) return _DockerIgnoreMatcher.from_lines(lines) - except DockerContextError: - raise + except DockerContextError as error: + raise DockerContextError( + f"invalid active ignore file {name!r}: {error}" + ) from error except Exception as error: - raise DockerContextError("failed to read .dockerignore") from error + raise DockerContextError( + f"failed to read active ignore file {name!r}" + ) from error _MOBY_TRIM_SPACE = ( @@ -929,6 +968,65 @@ def _validate_selection_entries( return ordered +@contextmanager +def _open_absolute_regular_file(path: str) -> Iterator[BinaryIO]: + """Open an absolute regular file without following symbolic links.""" + + if not _SECURE_OPEN_SUPPORTED: + raise DockerContextError( + "secure Dockerfile-specific ignore file opening is not supported " + "on this platform" + ) + absolute_path = os.path.abspath(path) + segments = tuple(segment for segment in absolute_path.split(os.sep) if segment) + if not segments: + raise DockerContextError("Dockerfile-specific ignore path must name a file") + + common_flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) + directory_flags = common_flags | os.O_DIRECTORY | os.O_NOFOLLOW + file_flags = common_flags | os.O_NOFOLLOW | getattr(os, "O_NONBLOCK", 0) + directory_fds: list[int] = [] + final_fd: int | None = None + handle: BinaryIO | None = None + try: + root_fd = os.open(os.sep, directory_flags) + directory_fds.append(root_fd) + parent_fd = root_fd + for segment in segments[:-1]: + parent_fd = os.open(segment, directory_flags, dir_fd=parent_fd) + directory_fds.append(parent_fd) + final_fd = os.open(segments[-1], file_flags, dir_fd=parent_fd) + if not stat.S_ISREG(os.fstat(final_fd).st_mode): + raise DockerContextError( + "Dockerfile-specific ignore path is not a regular file: " + f"{absolute_path!r}" + ) + handle = os.fdopen(final_fd, "rb") + final_fd = None + except DockerContextError: + raise + except (OSError, TypeError, ValueError) as error: + raise DockerContextError( + "cannot securely open Dockerfile-specific ignore file: " + f"{absolute_path!r}" + ) from error + finally: + if handle is None: + if final_fd is not None: + os.close(final_fd) + for descriptor in reversed(directory_fds): + os.close(descriptor) + + try: + yield handle + finally: + try: + handle.close() + finally: + for descriptor in reversed(directory_fds): + os.close(descriptor) + + class LocalDockerContext(DockerContext): """Local filesystem backed DockerContext. @@ -963,6 +1061,51 @@ def __init__( def dockerfile_text(self) -> str: return self._dockerfile_text + def dockerfile_ignore(self) -> tuple[str, bytes] | None: + """Return the adjacent Dockerfile-specific ignore file, if present.""" + + if not self._dockerfile_path: + return None + companion_path = f"{self._dockerfile_path}.dockerignore" + try: + info = os.lstat(companion_path) + except FileNotFoundError: + return None + except OSError as error: + raise DockerContextError( + "cannot inspect Dockerfile-specific ignore file: " + f"{companion_path!r}" + ) from error + if not stat.S_ISREG(info.st_mode): + raise DockerContextError( + "Dockerfile-specific ignore path is not a regular file: " + f"{companion_path!r}" + ) + + try: + relative_path = os.path.relpath(companion_path, self._context_dir) + except ValueError: + relative_path = os.pardir + inside_context = relative_path != os.pardir and not relative_path.startswith( + f"{os.pardir}{os.sep}" + ) + name = _to_posix(relative_path) if inside_context else companion_path + try: + if inside_context: + with self.open(name) as stream: + content = stream.read() + else: + with _open_absolute_regular_file(companion_path) as stream: + content = stream.read() + except DockerContextError: + raise + except Exception as error: + raise DockerContextError( + "failed to read Dockerfile-specific ignore file: " + f"{name!r}" + ) from error + return name, content + @contextmanager def open(self, path: str) -> Iterator[BinaryIO]: """Open a regular context file without following symbolic links. @@ -1040,10 +1183,6 @@ def walk(self) -> Iterator[DockerContextEntry]: if not os.path.isdir(self._context_dir): return - dockerfile_abs = ( - os.path.normpath(self._dockerfile_path) if self._dockerfile_path else None - ) - def traversal_error(error: OSError, path: str) -> DockerContextError: candidate = error.filename or path try: @@ -1092,14 +1231,6 @@ def onerror(error: OSError) -> None: raise DockerContextError( f"context path is not a regular file: {name!r}" ) - if dockerfile_abs and os.path.normpath(full) == dockerfile_abs: - continue - if ( - not self._dockerfile_path - and root == os.path.normpath(self._context_dir) - and name.lower() == "dockerfile" - ): - continue rel = _to_posix(os.path.relpath(full, self._context_dir)) entries.append( DockerContextEntry(rel, "file", stat.S_IMODE(info.st_mode)) diff --git a/sdk/python/examples/dockerfile_launch.py b/sdk/python/examples/dockerfile_launch.py index eedcbf6..f1ffa3a 100644 --- a/sdk/python/examples/dockerfile_launch.py +++ b/sdk/python/examples/dockerfile_launch.py @@ -47,15 +47,19 @@ def _precheck(context: LocalDockerContext) -> None: def section_core_path() -> None: - """Copy a filtered context and wildcard directory, then run a finite CMD.""" - print("\n=== Section 1: filtered core path ===") + """Copy a companion-filtered context and wildcard directory.""" + print("\n=== Section 1: companion-filtered core path ===") with tempfile.TemporaryDirectory() as directory: context_dir = Path(directory) - (context_dir / ".dockerignore").write_text( - "secret.txt\ndocs\n!docs/README.md\n", - encoding="utf-8", - ) + build = context_dir / "build" + build.mkdir() + (context_dir / ".dockerignore").write_text("greeting.txt\n", encoding="utf-8") (context_dir / "secret.txt").write_text("do not upload\n", encoding="utf-8") + (context_dir / "greeting.txt").write_text("hello\n", encoding="utf-8") + docs = context_dir / "docs" + docs.mkdir() + (docs / "README.md").write_text("visible\n", encoding="utf-8") + (docs / "private.txt").write_text("hidden\n", encoding="utf-8") (context_dir / "app.py").write_text( "import os\n" "open('/tmp/app.started', 'w').write(\n" @@ -63,11 +67,6 @@ def section_core_path() -> None: ")\n", encoding="utf-8", ) - (context_dir / "greeting.txt").write_text("hello\n", encoding="utf-8") - docs = context_dir / "docs" - docs.mkdir() - (docs / "README.md").write_text("visible\n", encoding="utf-8") - (docs / "private.txt").write_text("hidden\n", encoding="utf-8") executable = context_dir / "entrypoint.sh" executable.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") executable.chmod(0o755) @@ -80,20 +79,30 @@ def section_core_path() -> None: wildcard_file.write_text("wildcard\n", encoding="utf-8") empty.chmod(0o711) nested_empty.chmod(0o750) - dockerfile = """\ -FROM ubuntu:22.04 + dockerfile = build / "custom.Dockerfile" + dockerfile.write_text( + """FROM ubuntu:22.04 RUN apt-get update && apt-get install -y --no-install-recommends python3 RUN useradd -m app ENV WHOAMI=app WORKDIR /srv USER app +COPY greeting.txt /srv/control/greeting.txt +COPY build/custom.Dockerfile /srv/control/ +COPY .dockerignore /srv/control/ +COPY build/*.dockerignore /srv/control/ COPY empty/ /srv/core/literal-empty/ COPY . /srv/core/ COPY wild/* /srv/wild/ COPY docs /srv/reincluded-literal/ COPY doc* /srv/reincluded-wildcard/ CMD ["python3", "/srv/core/app.py"] -""" +""", + encoding="utf-8", + ) + (build / "custom.Dockerfile.dockerignore").write_text( + "secret.txt\ndocs\n!docs/README.md\n", encoding="utf-8" + ) context = LocalDockerContext(dockerfile, context_dir=context_dir) _precheck(context) @@ -102,23 +111,27 @@ def section_core_path() -> None: assert startup is not None result = startup.wait(timeout=60) assert result.exit_code == 0, result.stderr - marker = sandbox.commands.run("cat /tmp/app.started") assert marker.exit_code == 0, marker.stderr assert marker.stdout.strip() == "whoami=app cwd=/srv", marker.stdout - absent = sandbox.commands.run("test ! -e /srv/core/secret.txt") - assert absent.exit_code == 0, absent.stderr + controls = sandbox.commands.run( + "test -f /srv/control/greeting.txt " + "&& test -f /srv/control/custom.Dockerfile " + "&& test -f /srv/control/.dockerignore " + "&& test -f /srv/control/custom.Dockerfile.dockerignore " + "&& test -f /srv/core/greeting.txt " + "&& test -f /srv/core/build/custom.Dockerfile " + "&& test -f /srv/core/.dockerignore " + "&& test -f /srv/core/build/custom.Dockerfile.dockerignore " + "&& test ! -e /srv/core/secret.txt" + ) + assert controls.exit_code == 0, controls.stderr modes = sandbox.commands.run( "stat -c '%a' /srv/core/entrypoint.sh /srv/core/empty " "/srv/core/tree/nested-empty /srv/core/literal-empty" ) assert modes.exit_code == 0, modes.stderr - assert modes.stdout.splitlines() == [ - "755", - "711", - "750", - "755", - ], modes.stdout + assert modes.stdout.splitlines() == ["755", "711", "750", "755"], modes.stdout wildcard = sandbox.commands.run( "test -f /srv/wild/dir2/foo && test ! -e /srv/wild/dir1" ) diff --git a/sdk/python/tests/unit/test_dockercontext.py b/sdk/python/tests/unit/test_dockercontext.py index a27b655..926d635 100644 --- a/sdk/python/tests/unit/test_dockercontext.py +++ b/sdk/python/tests/unit/test_dockercontext.py @@ -29,14 +29,23 @@ class MemoryDockerContext(DockerContext): """A context fixture that records manifest file opens.""" - def __init__(self, files: dict[str, bytes], paths: list[object] | None = None): + def __init__( + self, + files: dict[str, bytes], + paths: list[object] | None = None, + dockerfile_ignore: tuple[str, bytes] | None = None, + ): self.files = files self.paths = _entries(files) if paths is None else paths + self._dockerfile_ignore = dockerfile_ignore self.open_paths: list[str] = [] def dockerfile_text(self) -> str: return "FROM scratch" + def dockerfile_ignore(self) -> tuple[str, bytes] | None: + return self._dockerfile_ignore + @contextmanager def open(self, path: str) -> Iterator[BinaryIO]: self.open_paths.append(path) @@ -88,7 +97,219 @@ def test_deterministic_literal_file_directory_and_dot(self) -> None: ) self.assertEqual(context.open_paths, []) - def test_dockerignore_semantics_and_reserved_files(self) -> None: + def test_dockerfile_specific_ignore_precedence_and_hook_failures(self) -> None: + files = { + ".dockerignore": b"root-only\n", + "docker/custom.Dockerfile": b"FROM scratch\n", + "docker/custom.Dockerfile.dockerignore": b"companion-only\n", + "companion-only": b"", + "root-only": b"", + } + cases = ( + (("docker/custom.Dockerfile.dockerignore", b"companion-only\n"), + [".dockerignore", "docker/custom.Dockerfile", + "docker/custom.Dockerfile.dockerignore", "root-only"], []), + (("docker/custom.Dockerfile.dockerignore", b""), sorted(files), []), + (None, [".dockerignore", "companion-only", "docker/custom.Dockerfile", + "docker/custom.Dockerfile.dockerignore"], [".dockerignore"]), + ) + for selected, expected, opened in cases: + with self.subTest(selected=selected): + context = MemoryDockerContext(files, dockerfile_ignore=selected) + manifest = _ContextManifest.from_context(context) + self.assertEqual( + [path for path, _ in paths(manifest.select("."))], + expected, + ) + self.assertEqual(context.open_paths, opened) + + bad_pattern = MemoryDockerContext( + {"safe": b""}, dockerfile_ignore=("custom.Dockerfile.dockerignore", b"[\n") + ) + with self.assertRaisesRegex( + DockerContextError, "custom.Dockerfile.dockerignore" + ): + _ContextManifest.from_context(bad_pattern) + + class RaisingContext(MemoryDockerContext): + def __init__(self, error: Exception) -> None: + super().__init__({"safe": b""}) + self.error = error + + def dockerfile_ignore(self) -> tuple[str, bytes] | None: + raise self.error + + with self.assertRaisesRegex(DockerContextError, "failed to obtain"): + _ContextManifest.from_context(RaisingContext(OSError("unreadable"))) + with self.assertRaisesRegex(DockerContextError, "sentinel"): + _ContextManifest.from_context( + RaisingContext(DockerContextError("sentinel")) + ) + + for value in ([], ("name",), ("name", b"", b"x"), ("", b""), + (1, b""), ("name", "text"), ["name", b""]): + with self.subTest(value=repr(value)): + context = MemoryDockerContext({"safe": b""}) + context._dockerfile_ignore = value # type: ignore[assignment] + with self.assertRaisesRegex(DockerContextError, "dockerfile_ignore"): + _ContextManifest.from_context(context) + + def test_default_dockerfile_ignore_method_remains_root_compatible(self) -> None: + class DefaultIgnoreContext(DockerContext): + def dockerfile_text(self) -> str: + return "FROM scratch" + + @contextmanager + def open(self, path: str) -> Iterator[BinaryIO]: + yield io.BytesIO({".dockerignore": b"hidden\n", "hidden": b""}[path]) + + def walk(self) -> Iterator[DockerContextEntry]: + yield DockerContextEntry(".dockerignore", "file", 0o644) + yield DockerContextEntry("hidden", "file", 0o644) + + manifest = _ContextManifest.from_context(DefaultIgnoreContext()) + with self.assertRaisesRegex(DockerContextError, "ignored"): + manifest.select("hidden") + + def test_local_companion_visibility_and_explicit_control_exclusion(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + dockerfile = root / "docker" / "custom.Dockerfile" + dockerfile.parent.mkdir() + dockerfile.write_text("FROM scratch\n", encoding="utf-8") + (root / ".dockerignore").write_text("root-only\n", encoding="utf-8") + companion = dockerfile.with_name("custom.Dockerfile.dockerignore") + companion.write_text("companion-only\n", encoding="utf-8") + for name in ("root-only", "companion-only", "visible"): + (root / name).write_text(name, encoding="utf-8") + + context = LocalDockerContext(dockerfile, context_dir=root) + manifest = _ContextManifest.from_context(context) + self.assertEqual( + [path for path, _ in paths(manifest.select("."))], + [".dockerignore", "docker/custom.Dockerfile", + "docker/custom.Dockerfile.dockerignore", "root-only", "visible"], + ) + for name in (".dockerignore", "docker/custom.Dockerfile", + "docker/custom.Dockerfile.dockerignore"): + self.assertEqual( + paths(manifest.select(name)), + [(name, Path(name).name)], + ) + + companion.write_text( + ".dockerignore\ndocker/custom.Dockerfile\n" + "docker/custom.Dockerfile.dockerignore\n", encoding="utf-8" + ) + filtered = _ContextManifest.from_context( + LocalDockerContext(dockerfile, context_dir=root) + ) + for name in (".dockerignore", "docker/custom.Dockerfile", + "docker/custom.Dockerfile.dockerignore"): + with self.assertRaisesRegex(DockerContextError, "ignored"): + filtered.select(name) + + def test_inline_and_external_dockerfiles_are_not_synthesized(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "Dockerfile").write_text("FROM scratch\n", encoding="utf-8") + self.assertEqual( + list(LocalDockerContext("FROM scratch\n", context_dir=root).walk()), + [DockerContextEntry("Dockerfile", "file", 0o644)], + ) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + context_dir = root / "context" + context_dir.mkdir() + dockerfile = root / "outside.Dockerfile" + dockerfile.write_text("FROM scratch\n", encoding="utf-8") + dockerfile.with_name("outside.Dockerfile.dockerignore").write_text( + "hidden\n", encoding="utf-8" + ) + (context_dir / ".dockerignore").write_text("visible\n", encoding="utf-8") + (context_dir / "hidden").write_text("", encoding="utf-8") + (context_dir / "visible").write_text("", encoding="utf-8") + manifest = _ContextManifest.from_context( + LocalDockerContext(dockerfile, context_dir=context_dir) + ) + self.assertEqual( + [entry.path for entry in manifest._raw_entries], + [".dockerignore", "hidden", "visible"], + ) + self.assertEqual( + [path for path, _ in paths(manifest.select("."))], + [".dockerignore", "visible"], + ) + + def test_context_companion_non_regular_files_fail_closed(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + dockerfile = root / "Dockerfile" + dockerfile.write_text("FROM scratch\n", encoding="utf-8") + companion = root / "Dockerfile.dockerignore" + local = LocalDockerContext(dockerfile, context_dir=root) + target = root / "target" + target.write_text("", encoding="utf-8") + companion.symlink_to(target) + with self.assertRaisesRegex(DockerContextError, "regular file"): + local.dockerfile_ignore() + companion.unlink() + companion.mkdir() + with self.assertRaisesRegex(DockerContextError, "regular file"): + local.dockerfile_ignore() + companion.rmdir() + if not hasattr(os, "mkfifo"): + self.skipTest("mkfifo is unavailable on this platform") + try: + os.mkfifo(companion) + except (NotImplementedError, OSError) as error: + self.skipTest(f"mkfifo is unavailable: {error}") + try: + with self.assertRaisesRegex(DockerContextError, "regular file"): + local.dockerfile_ignore() + finally: + companion.unlink() + + def test_context_companion_lstat_symlink_race_fails_closed(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + dockerfile = root / "Dockerfile" + dockerfile.write_text("FROM scratch\n", encoding="utf-8") + companion = root / "Dockerfile.dockerignore" + companion.write_bytes(b"safe\n") + target = root / "target" + target.write_bytes(b"target\n") + local = LocalDockerContext(dockerfile, context_dir=root) + real_lstat = os.lstat + replaced = False + + def racing_lstat( + path: str | bytes, + *, + dir_fd: int | None = None, + ) -> os.stat_result: + nonlocal replaced + info = real_lstat(path, dir_fd=dir_fd) + if ( + os.path.abspath(os.fsdecode(path)) == str(companion) + and not replaced + ): + replaced = True + companion.unlink() + companion.symlink_to(target) + return info + + with ( + patch( + "akernel_sdk._dockercontext.os.lstat", + side_effect=racing_lstat, + ), + self.assertRaises(DockerContextError), + ): + local.dockerfile_ignore() + self.assertTrue(replaced) + + def test_dockerignore_semantics_and_control_file_visibility(self) -> None: ignore = ( bytes([0xEF, 0xBB, 0xBF]) + b"# comment" @@ -129,9 +350,14 @@ def test_dockerignore_semantics_and_reserved_files(self) -> None: self.assertEqual(context.open_paths, [".dockerignore"]) self.assertEqual( paths(manifest.select(".")), - [("keep.txt", "keep.txt"), ("src/ok.py", "src/ok.py")], + [ + (".dockerignore", ".dockerignore"), + ("Dockerfile", "Dockerfile"), + ("keep.txt", "keep.txt"), + ("src/ok.py", "src/ok.py"), + ], ) - for source in (".dockerignore", "Dockerfile", ".env", ".git", "drop.txt"): + for source in (".env", ".git", "drop.txt"): with self.subTest(source=source): with self.assertRaisesRegex(DockerContextError, "ignored"): manifest.select(source) @@ -163,6 +389,7 @@ def test_dockerignore_moby_preprocessing_and_embedded_double_star(self) -> None: self.assertEqual( paths(manifest.select(".")), [ + (".dockerignore", ".dockerignore"), ("a/keep.bin", "a/keep.bin"), ("nested/root.pem", "nested/root.pem"), ("safe", "safe"), @@ -215,9 +442,9 @@ def test_reincluded_descendants_make_virtual_source_directories(self) -> None: (entry.source_path, entry.relative_target, entry.kind) for entry in wildcard.entries ], - expected, + [(".dockerignore", ".dockerignore", "file"), *expected], ) - self.assertEqual(wildcard.top_level_source_count, 1) + self.assertEqual(wildcard.top_level_source_count, 2) with self.assertRaisesRegex(DockerContextError, "ignored"): manifest.select("docs/private.txt") self.assertEqual(context.open_paths, [".dockerignore"]) @@ -264,7 +491,11 @@ def test_parent_match_stack_handles_path_prefix_siblings(self) -> None: manifest = _ContextManifest.from_context(context) self.assertEqual( paths(manifest.select(".")), - [("a-b", "a-b"), ("a/file", "a/file")], + [ + (".dockerignore", ".dockerignore"), + ("a-b", "a-b"), + ("a/file", "a/file"), + ], ) def test_manifest_reuses_parent_ignore_results(self) -> None: @@ -294,7 +525,10 @@ def test_escaped_ignore_patterns_and_invalid_utf8(self) -> None: ) self.assertEqual( paths(_ContextManifest.from_context(escaped).select(".")), - [("visible", "visible")], + [ + (".dockerignore", ".dockerignore"), + ("visible", "visible"), + ], ) invalid = MemoryDockerContext({".dockerignore": bytes([0xFF]), "ok": b""}) with self.assertRaisesRegex(DockerContextError, "dockerignore"): @@ -424,8 +658,7 @@ def test_wildcard_no_match_and_only_ignored(self) -> None: def test_dot_distinguishes_empty_and_fully_filtered_contexts(self) -> None: cases = ( ({}, "no match"), - ({"Dockerfile": b""}, "ignored"), - ({".dockerignore": b"*.txt" + NL, "hidden.txt": b""}, "ignored"), + ({".dockerignore": b"*" + NL, "hidden.txt": b""}, "ignored"), ) for files, message in cases: with self.subTest(files=files): @@ -654,7 +887,11 @@ def test_dockerignore_filters_empty_directories(self) -> None: (entry.source_path, entry.relative_target) for entry in manifest.select(".").entries ], - [("visible", "visible"), ("visible/nested", "visible/nested")], + [ + (".dockerignore", ".dockerignore"), + ("visible", "visible"), + ("visible/nested", "visible/nested"), + ], ) with self.assertRaisesRegex(DockerContextError, "ignored"): manifest.select("ignored") diff --git a/sdk/python/tests/unit/test_dockerfile.py b/sdk/python/tests/unit/test_dockerfile.py index 0d4e326..114230e 100644 --- a/sdk/python/tests/unit/test_dockerfile.py +++ b/sdk/python/tests/unit/test_dockerfile.py @@ -593,7 +593,7 @@ def _apply_memory( ) return sandbox, context - def test_dot_honors_dockerignore_and_skips_reserved_files(self) -> None: + def test_dot_honors_dockerignore_and_copies_visible_control_files(self) -> None: files = { ".dockerignore": b".env\n.git/**\n", "Dockerfile": b"FROM scratch\n", @@ -606,8 +606,14 @@ def test_dot_honors_dockerignore_and_skips_reserved_files(self) -> None: copies = [ operation[2] for operation in sandbox.files.ops if operation[0] == "cp" ] - self.assertEqual(copies, ["/app/allowed", "/app/sub/x"]) - self.assertEqual(context.open_paths, [".dockerignore", "allowed", "sub/x"]) + self.assertEqual( + copies, + ["/app/.dockerignore", "/app/Dockerfile", "/app/allowed", "/app/sub/x"], + ) + self.assertEqual( + context.open_paths, + [".dockerignore", ".dockerignore", "Dockerfile", "allowed", "sub/x"], + ) def test_reincluded_dockerignore_descendant_copies_from_virtual_dir(self) -> None: files = { @@ -625,11 +631,13 @@ def test_reincluded_dockerignore_descendant_copies_from_virtual_dir(self) -> Non for operation in sandbox.files.ops if operation[0] == "cp" ] - self.assertEqual(copies, ["/out/README.md"]) - self.assertEqual( - context.open_paths, - [".dockerignore", "docs/README.md"], - ) + expected = ["/out/README.md"] + opened = [".dockerignore", "docs/README.md"] + if source == "*": + expected.insert(0, "/out/.dockerignore") + opened.insert(1, ".dockerignore") + self.assertEqual(copies, expected) + self.assertEqual(context.open_paths, opened) def test_local_and_memory_contexts_have_same_copy_targets(self) -> None: import os @@ -925,7 +933,7 @@ def test_selection_and_target_errors_have_no_sandbox_operations(self) -> None: ("COPY . /dest/\n", {}), ( "COPY . /dest/\n", - {".dockerignore": b"*.txt\n", "hidden.txt": b""}, + {".dockerignore": b"*\n", "hidden.txt": b""}, ), ) for instruction, files in cases: @@ -1438,7 +1446,7 @@ def test_file_path( self.assertEqual(ctx.dockerfile_text(), "FROM node:20\n") self.assertEqual(ctx.context_dir, os.path.abspath(d)) - def test_walk_excludes_dockerfile(self): + def test_walk_includes_control_files(self): import os import tempfile @@ -1451,7 +1459,10 @@ def test_walk_excludes_dockerfile(self): ctx = LocalDockerContext("FROM ubuntu\n", context_dir=d) self.assertEqual( list(ctx.walk()), - [DockerContextEntry("app.py", "file", 0o640)], + [ + DockerContextEntry("Dockerfile", "file", 0o644), + DockerContextEntry("app.py", "file", 0o640), + ], ) def test_open_path_escape_rejected(self): From 7506ecd0c3701494ea7e23c09f1a3ad7952d2ac5 Mon Sep 17 00:00:00 2001 From: Peng-YM Date: Wed, 19 Aug 2026 20:45:04 +0800 Subject: [PATCH 09/12] test(sdk): make context modes umask-independent Set the Dockerfile fixture modes explicitly so manifest-mode expectations do not depend on the process umask used by the test runner. Signed-off-by: Peng-YM --- sdk/python/tests/unit/test_dockercontext.py | 1 + sdk/python/tests/unit/test_dockerfile.py | 1 + 2 files changed, 2 insertions(+) diff --git a/sdk/python/tests/unit/test_dockercontext.py b/sdk/python/tests/unit/test_dockercontext.py index 926d635..112ff39 100644 --- a/sdk/python/tests/unit/test_dockercontext.py +++ b/sdk/python/tests/unit/test_dockercontext.py @@ -213,6 +213,7 @@ def test_inline_and_external_dockerfiles_are_not_synthesized(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) (root / "Dockerfile").write_text("FROM scratch\n", encoding="utf-8") + (root / "Dockerfile").chmod(0o644) self.assertEqual( list(LocalDockerContext("FROM scratch\n", context_dir=root).walk()), [DockerContextEntry("Dockerfile", "file", 0o644)], diff --git a/sdk/python/tests/unit/test_dockerfile.py b/sdk/python/tests/unit/test_dockerfile.py index 114230e..bce0edc 100644 --- a/sdk/python/tests/unit/test_dockerfile.py +++ b/sdk/python/tests/unit/test_dockerfile.py @@ -1453,6 +1453,7 @@ def test_walk_includes_control_files(self): with tempfile.TemporaryDirectory() as d: with open(os.path.join(d, "Dockerfile"), "w") as _f: _f.write("FROM ubuntu\n") + os.chmod(os.path.join(d, "Dockerfile"), 0o644) with open(os.path.join(d, "app.py"), "w") as _f: _f.write("print(1)") os.chmod(os.path.join(d, "app.py"), 0o640) From 34b3215b9f4bc0567d957112da8e6137ea9df4b7 Mon Sep 17 00:00:00 2001 From: Peng-YM Date: Thu, 20 Aug 2026 11:32:42 +0800 Subject: [PATCH 10/12] refactor(sdk): encapsulate Dockerfile launch options Encapsulate experimental Dockerfile launch settings in one DockerfileLaunch value, remove the meaningless top-level combination, and split the detailed Experimental guide from the SDK README. Signed-off-by: Peng-YM --- AGENTS.md | 25 ++-- sdk/python/README.md | 148 ++++---------------- sdk/python/akernel_sdk/__init__.py | 2 + sdk/python/akernel_sdk/_dockerfile.py | 38 ++++- sdk/python/akernel_sdk/sandbox.py | 63 ++++----- sdk/python/docs/launch-from-dockerfile.md | 161 ++++++++++++++++++++++ sdk/python/examples/dockerfile_launch.py | 18 +-- sdk/python/tests/unit/test_dockerfile.py | 38 +++++ sdk/python/tests/unit/test_sandbox.py | 63 +++++++-- sdk/python/tests/unit/test_types.py | 6 + 10 files changed, 376 insertions(+), 186 deletions(-) create mode 100644 sdk/python/docs/launch-from-dockerfile.md diff --git a/AGENTS.md b/AGENTS.md index f7d3c83..92f757a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -347,17 +347,20 @@ Keep public `Sandbox`, `Commands`, `Filesystem`, and value types independent of both native packages; all native conversions belong under `akernel_sdk._backends`. -The SDK also supports a strict Dockerfile direct-launch subset through -`DockerContext` and `Sandbox(context=...)`. `FROM` supplies only the root -filesystem; inherited OCI image configuration is not applied. `DockerContext.walk()` -uses public structured entries for files and directories, including permission -modes and empty directories. Context transfer must remain backend-neutral and -fail closed for syntax outside the documented subset. A path-form Dockerfile uses an existing adjacent -`.dockerignore` ahead of the root `.dockerignore`, including an empty -companion; only its absence permits root fallback. Dockerfiles and active or root -ignore files remain ordinary context entries and are selectable unless the active -matcher excludes them. Keep the contract, security boundaries, public types, unit -tests, SDK README, and `examples/dockerfile_launch.py` in sync. +The SDK also supports an experimental, strict Dockerfile direct-launch subset +through `DockerContext` and +`Sandbox(dockerfile=DockerfileLaunch(context=..., auto_start_cmd=..., run_timeout=...))`. +Read [`sdk/python/docs/launch-from-dockerfile.md`](./sdk/python/docs/launch-from-dockerfile.md) +before changing this path. `FROM` supplies only the root filesystem; inherited +OCI configuration is not applied. Runtime availability and compatibility remain +backend-owned. `DockerContext.walk()` exposes public structured file and +directory entries, including modes and empty directories; context transfer must +remain backend-neutral, reject unsafe manifests and unsupported syntax +fail-closed, and preserve documented Dockerfile-specific ignore-file +precedence. Dockerfiles and active or root ignore files remain ordinary context +entries unless the active matcher excludes them. Keep the public types, unit +tests, SDK README, Dockerfile launch guide, and +`examples/dockerfile_launch.py` in sync. When changing a public SDK method, update its type annotations and docstring, add or update unit coverage, and keep the SDK README and maintained examples in diff --git a/sdk/python/README.md b/sdk/python/README.md index 45a2266..8a64b7d 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -24,6 +24,7 @@ It supports two backends: - [Port forwarding](#port-forwarding) - [Reverse tunnels](#reverse-tunnels) - [Rootfs and mounts](#rootfs-and-mounts) + - [Experimental: launch from a Dockerfile](#experimental-launch-from-a-dockerfile) - [Resources and lifecycle](#resources-and-lifecycle) - [CLI](#cli) - [Examples and tests](#examples-and-tests) @@ -101,130 +102,10 @@ Sandbox( xpu: str | None = None, storage_mb: int | None = None, network_policy: NetworkPolicy | None = None, - context: DockerContext | None = None, - auto_start_cmd: bool = True, - build_run_timeout: int = 600, + dockerfile: DockerfileLaunch | None = None, ) ``` -### Launch a sandbox from a Dockerfile - -`image`, `rootfs`, and `context` are mutually exclusive. `Sandbox(context=...)` -implements a deliberately narrow, direct-launch subset of Dockerfile syntax -through the public `Sandbox`, `Commands`, and `Filesystem` facades. It needs no -BuildKit, Docker daemon, or registry push. - -**`FROM` is rootfs-only.** The selected image supplies the sandbox root -filesystem only. Its OCI `ENV`, `USER`, `WORKDIR`, `CMD`, and `ENTRYPOINT` -configuration is not inherited. Declare every required runtime setting in the -Dockerfile passed to `DockerContext`. - -Pre-check the Dockerfile before creating a sandbox. A check returning `False` -means direct launch will fail closed; pre-build externally instead. This example -creates one context instance, reports diagnostic warnings, launches it, and -inspects the optional startup handle: - -```python -from akernel_sdk import LocalDockerContext, Sandbox, check_direct_launch - -context = LocalDockerContext("Dockerfile", context_dir=".") -check = check_direct_launch(context) -for warning in check.warnings: - print(f"Dockerfile warning: {warning}") - -if check.direct_launchable: - with Sandbox(context=context, build_run_timeout=300) as sandbox: - startup = sandbox.startup_command - if startup is not None: - result = startup.wait(timeout=60) - print(result.exit_code, result.stderr) - # Perform the application's own health check here when it is long-lived. -else: - image = build_image_externally("Dockerfile") - with Sandbox(image=image) as sandbox: - # An image launch does not auto-start its configured CMD or ENTRYPOINT. - sandbox.commands.run("/srv/app --serve", background=True) -``` - -`check_direct_launch()` is diagnostic and returns reason codes such as -`multi_stage`, `remote_add`, `no_from`, `unsupported_instruction`, and -`unsupported_syntax`. `Sandbox(context=...)` parses strictly, while -`parse_dockerfile(strict=False)` is diagnostic only. `apply_dockerfile()` also -rejects a parsed result containing unsupported syntax. - -| Supported direct-launch subset | Rejected and requires an external build | -| --- | --- | -| Exactly one literal `FROM`, optionally `AS alias`; shell-form `RUN` | Multiple stages, `COPY --from`, `FROM` flags or variables, and exec-form `RUN` | -| Shell-form local `COPY` and `ADD` paths: files, directories, `.`, and wildcards; literal `--chown`; literal local tar extraction for `ADD` | JSON-form `COPY`/`ADD`, remote `ADD` URLs, `--chmod`, `--link`, unknown flags, or build-time variable expansion | -| Literal `ENV`, absolute `WORKDIR`, named `USER` values such as `app` or `root`, and `EXPOSE` metadata | Any `ARG`, relative `WORKDIR`, `USER user:group` or numeric UID/GID values, and `VOLUME`, `LABEL`, `HEALTHCHECK`, `SHELL`, `STOPSIGNAL`, `ONBUILD`, `MAINTAINER`, or unknown instructions | -| Exec- or shell-form `CMD` and `ENTRYPOINT`, normalized and combined | — | - -A `DockerContext` exposes Dockerfile text plus structured `DockerContextEntry` -values from `walk()`. Each entry has a relative POSIX `path`, `kind` of `file` -or `directory`, and a permission-bit `mode` from `0o000` through `0o777`; -custom contexts must expose files and every ancestor or empty directory. -Custom contexts may implement `dockerfile_ignore()`: returning a -`(diagnostic_name, bytes)` tuple supplies the active matcher, while only -returning `None` falls back to the manifest-root `.dockerignore`; empty `bytes` -still denote a present, higher-priority matcher. Dockerfiles and ignore files -that belong to the filesystem context must still be enumerated by `walk()`. -`LocalDockerContext` uses a local directory and rejects symbolic links. Before -any sandbox operation, direct launch walks and validates the manifest, then applies -Moby-compatible ordered ignore matching to files and directories. For a path-form -Dockerfile, its adjacent `.dockerignore` is the active ignore file when -it exists, including when it is empty. -An inline Dockerfile creates no virtual context file, while a Dockerfile outside the -context and its companion remain outside the manifest even when that companion supplies -the active matcher. Dockerfiles, the root `.dockerignore`, and Dockerfile-specific -ignore files are ordinary context files: `COPY`/`ADD` can select them unless the -active ignore file explicitly excludes them. Direct launch then plans every -`COPY`/`ADD` and materializes all selected files. Docker ignore -patterns are cleaned like `filepath.Clean`, comments require `#` in column one, -embedded `**` can span directories, and later `!` patterns can re-include -descendants. Alphanumeric backslash escapes and nested POSIX character -classes that Moby routes through its regular-expression engine are rejected -rather than treated as literals. When a descendant is re-included beneath an -ignored directory, its -required directory ancestors remain selectable as virtual source directories; -fully ignored directories remain unavailable. Selected directory entries create -empty and nested directories, while each copied child file or directory has its -own non-recursive mode restored. For a literal directory source, the source root -itself is only a content container: the destination is created when needed, but -does not inherit that source-root mode. A wildcard that matches a directory -likewise copies its contents, rather than the matched directory name. Dockerfile -`COPY`/`ADD` source patterns follow Go `filepath.Match`-style segment -matching, so `**` has the same one-segment behavior as `*`; `[^a]` -negates a character class while `[!a]` matches `!` or `a`. Backslash -escapes are outside this strict source-pattern subset, and malformed classes fail -closed. When one wildcard expands to multiple top-level sources after -`.dockerignore` filtering, the destination must end in `/`. Paths and target -collisions fail closed. `--chown` affects only files and directories created by -the current instruction. Local tar `ADD` accepts only regular files -and directories with safe paths, preserves tar member metadata, and always -extracts as the builder/root identity before applying `--chown`. Remote `ADD` -URLs are rejected; the SDK never fetches them. Secure local context opening -requires platform support for directory-relative file descriptors and no-follow -flags; unsupported platforms fail closed. - -Each launch executes `RUN`, `COPY`, and `ADD` again in a new sandbox. There is -no snapshot or build cache. After instructions finish, the SDK polls **sandbox -readiness** and dispatches the resolved `CMD`/`ENTRYPOINT` in the background. -`Sandbox.startup_command` is a `CommandHandle | None`; use `wait()` for finite -commands or `kill()` when needed. Construction confirms dispatch, not that an -application remains running or passes a health check. Callers own -application-specific readiness checks. - -For Dockerfiles outside this subset, or for build-once reuse, pre-build the -image with your chosen build system and use `Sandbox(image=...)`. Then launch -the desired command explicitly with `sandbox.commands.run(..., background=True)`; -image configuration does not auto-start a process in this SDK path. - -Parsing uses [`dockerfile-parse`](https://github.com/containerbuildsystem/dockerfile-parse) -(BSD-3-Clause). `.dockerignore` behavior follows -[`moby/patternmatcher`](https://github.com/moby/patternmatcher) (Apache-2.0). -The value post-processing approach was informed by the -[E2B Python SDK](https://github.com/e2b-dev/E2B) (MIT). - ### Experimental GPU and writable storage Request a whole NVIDIA GPU by type, exact product model, and count: @@ -531,6 +412,27 @@ OCI images can also be mounted read-only: mount = Mount(target="/opt/tools", image_url="ubuntu:24.04") ``` +## Experimental: launch from a Dockerfile +> **Warning:** This first direct-launch release supports a strict, changing +> Dockerfile subset. It is not a general-purpose Docker build. + +`FROM` supplies only the root filesystem; inherited OCI configuration is not +applied. Precheck the context, then pass its launch configuration to `Sandbox`: + +```python +from akernel_sdk import DockerfileLaunch, LocalDockerContext, Sandbox, check_direct_launch +context = LocalDockerContext("Dockerfile", context_dir=".") +if check_direct_launch(context).direct_launchable: + with Sandbox(dockerfile=DockerfileLaunch(context, run_timeout=300)) as sandbox: + pass +``` + +`RUN`, `COPY`, and `ADD` run on every launch without a snapshot or cache; +unsupported Dockerfiles must be built externally. Read the complete contract, +security boundaries, and supported syntax in +[the Dockerfile launch guide](./docs/launch-from-dockerfile.md). See the +[runnable example](./examples/dockerfile_launch.py). + ## Resources and lifecycle `resources()` returns stable `NodeInfo` values rather than backend objects: @@ -623,3 +525,7 @@ not part of the default test suite. | `Mount` | `target`, one source, and `type` | | `HttpReverseTunnel` | `target`, `reverse_port`, `listen_port`, `connect_timeout` | | `NetworkPolicy` | `block_network`, `dns_blacklist` | +| `DockerfileLaunch` | `context`, `auto_start_cmd`, `run_timeout` | +| `DockerContext` | Abstract Dockerfile and build-context source | +| `DockerContextEntry` | `path`, `kind`, `mode` | +| `LocalDockerContext` | Local Dockerfile and context implementation | diff --git a/sdk/python/akernel_sdk/__init__.py b/sdk/python/akernel_sdk/__init__.py index ea69876..3cc46a0 100644 --- a/sdk/python/akernel_sdk/__init__.py +++ b/sdk/python/akernel_sdk/__init__.py @@ -57,6 +57,7 @@ "UnsupportedBackendFeatureError", "BackendOperationError", "DockerContext", + "DockerfileLaunch", "DockerContextEntry", "LocalDockerContext", "parse_dockerfile", @@ -78,6 +79,7 @@ "PtyError": (".pty", "PtyError"), "resources": ("._resources", "resources"), "DockerContext": ("._dockercontext", "DockerContext"), + "DockerfileLaunch": ("._dockerfile", "DockerfileLaunch"), "DockerContextEntry": ("._dockercontext", "DockerContextEntry"), "LocalDockerContext": ("._dockercontext", "LocalDockerContext"), "parse_dockerfile": ("._dockerfile", "parse_dockerfile"), diff --git a/sdk/python/akernel_sdk/_dockerfile.py b/sdk/python/akernel_sdk/_dockerfile.py index 74b852b..74d5736 100644 --- a/sdk/python/akernel_sdk/_dockerfile.py +++ b/sdk/python/akernel_sdk/_dockerfile.py @@ -31,6 +31,35 @@ from ._dockercontext import DockerContext +@dataclass(frozen=True) +class DockerfileLaunch: + """Experimental immutable configuration for a Dockerfile sandbox launch. + + Args: + context: Dockerfile and build context to apply in the sandbox. + auto_start_cmd: Dispatch the Dockerfile CMD/ENTRYPOINT after applying + build-time instructions. Defaults to ``True``. + run_timeout: Positive per-``RUN`` timeout in seconds. Defaults to + ``600``. + """ + + context: DockerContext + auto_start_cmd: bool = True + run_timeout: int = 600 + + def __post_init__(self) -> None: + if not isinstance(self.context, DockerContext): + raise TypeError("context must be a DockerContext") + if not isinstance(self.auto_start_cmd, bool): + raise TypeError("auto_start_cmd must be a boolean") + if isinstance(self.run_timeout, bool) or not isinstance( + self.run_timeout, int + ): + raise TypeError("run_timeout must be an integer") + if self.run_timeout <= 0: + raise ValueError("run_timeout must be greater than zero") + + @dataclass(frozen=True) class FromInstruction: image: str @@ -404,7 +433,9 @@ def parse_dockerfile( def check_direct_launch( context: DockerContext, *, strict: bool = False ) -> DockerfileCheckResult: - """Check whether ``Sandbox(context=...)`` can execute a Dockerfile safely.""" + """Check whether ``Sandbox(dockerfile=DockerfileLaunch(...))`` can + execute a Dockerfile safely. + """ try: parsed = parse_dockerfile(context, strict=strict) except DockerfileParseError as exc: @@ -426,8 +457,9 @@ def check_direct_launch( warnings = list(parsed.warnings) if has_build and not parsed.unsupported: warnings.append( - "Dockerfile contains RUN/COPY/ADD; Sandbox(context=...) re-runs " - "them on every launch (no snapshot). For build-once reuse, " + "Dockerfile contains RUN/COPY/ADD; " + "Sandbox(dockerfile=DockerfileLaunch(...)) re-runs them on every " + "launch (no snapshot). For build-once reuse, " "pre-build the image and launch with Sandbox(image=...)." ) if not parsed.unsupported: diff --git a/sdk/python/akernel_sdk/sandbox.py b/sdk/python/akernel_sdk/sandbox.py index 8bf2ad2..5504b97 100644 --- a/sdk/python/akernel_sdk/sandbox.py +++ b/sdk/python/akernel_sdk/sandbox.py @@ -22,17 +22,20 @@ import urllib.request from collections.abc import Mapping, Sequence from types import MappingProxyType +from typing import TYPE_CHECKING from ._addresses import Endpoint, api_endpoint_from_env, gateway_endpoint_from_env from ._backends.base import BackendSession, SandboxSpec from ._backends.registry import load_backend -from ._dockercontext import DockerContext from ._sandbox_resources import normalize_xpu, validate_storage_mb from .commands import CommandHandle, Commands from .filesystem import Filesystem from .pty import Pty from .types import HttpReverseTunnel, Mount, NetworkPolicy, S3Config, SandboxInfo +if TYPE_CHECKING: + from ._dockerfile import DockerfileLaunch + _traefik_internal_ip_cache: str | None = None logger = logging.getLogger(__name__) @@ -134,9 +137,7 @@ def __init__( xpu: str | None = None, storage_mb: int | None = None, network_policy: NetworkPolicy | None = None, - context: DockerContext | None = None, - auto_start_cmd: bool = True, - build_run_timeout: int = 600, + dockerfile: DockerfileLaunch | None = None, ) -> None: """Create and wait for a sandbox to become ready. @@ -168,19 +169,12 @@ def __init__( are validated against the selected runtime by the backend. network_policy: Optional creation-time network policy. Omitting it leaves sandbox networking unrestricted. - context: A :class:`DockerContext` carrying a Dockerfile and build - context. ``FROM`` supplies only the root filesystem; its OCI - ENV, USER, WORKDIR, CMD and ENTRYPOINT configuration is not - inherited. The sandbox applies only state explicitly declared - in this Dockerfile, then executes build-time instructions - in-sandbox. Mutually exclusive with ``image`` and ``rootfs``. - auto_start_cmd: When ``context`` is given, wait for sandbox - readiness and dispatch CMD/ENTRYPOINT in the background after - build-time instructions complete. Construction confirms only - successful dispatch, not that the process remains running or - healthy. Default ``True``. - build_run_timeout: Per-``RUN`` timeout in seconds when ``context`` - is given. Default ``600``. + dockerfile: Experimental Dockerfile launch configuration. ``FROM`` + supplies only the root filesystem; its OCI ENV, USER, WORKDIR, + CMD and ENTRYPOINT configuration is not inherited. The sandbox + applies only state explicitly declared in this Dockerfile, then + executes build-time instructions in-sandbox. Mutually exclusive + with ``image`` and ``rootfs``. Raises: TypeError: An argument has an invalid type. @@ -192,11 +186,14 @@ def __init__( raise ValueError("image must be a non-empty string") if rootfs is not None and not isinstance(rootfs, S3Config): raise TypeError("rootfs must be an S3Config") - if context is not None and not isinstance(context, DockerContext): - raise TypeError("context must be a DockerContext") - if sum(value is not None for value in (image, rootfs, context)) > 1: + if dockerfile is not None: + from ._dockerfile import DockerfileLaunch + + if not isinstance(dockerfile, DockerfileLaunch): + raise TypeError("dockerfile must be a DockerfileLaunch") + if sum(value is not None for value in (image, rootfs, dockerfile)) > 1: raise ValueError( - "image, rootfs and context are mutually exclusive: at most one " + "image, rootfs and dockerfile are mutually exclusive: at most one " "may be given" ) if not isinstance(runtime, str): @@ -258,10 +255,10 @@ def __init__( ) parsed_dockerfile = None - if context is not None: + if dockerfile is not None: from ._dockerfile import parse_dockerfile - parsed_dockerfile = parse_dockerfile(context, strict=True) + parsed_dockerfile = parse_dockerfile(dockerfile.context, strict=True) image = parsed_dockerfile.base_image if not isinstance(image, str) or not image.strip(): raise ValueError("Dockerfile base image must be a non-empty string") @@ -312,15 +309,15 @@ def __init__( self._files = Filesystem(self._session.files) self._commands = Commands(self._session.commands) self._pty = Pty(self._id) - if context is not None and parsed_dockerfile is not None: + if dockerfile is not None and parsed_dockerfile is not None: from ._dockerfile_runner import apply_dockerfile apply_result = apply_dockerfile( self, parsed_dockerfile, - context, - auto_start_cmd=auto_start_cmd, - run_timeout=build_run_timeout, + dockerfile.context, + auto_start_cmd=dockerfile.auto_start_cmd, + run_timeout=dockerfile.run_timeout, ) self._startup_command = apply_result.startup_command except Exception: @@ -355,13 +352,13 @@ def commands(self) -> Commands: @property def startup_command(self) -> CommandHandle | None: - """Background CMD/ENTRYPOINT handle for a context launch, if dispatched. + """Background CMD/ENTRYPOINT handle for a Dockerfile launch, if dispatched. - The handle is available only when context and auto_start_cmd are used - with a startup command. It is None for normal image/rootfs launches, - auto_start_cmd=False, or Dockerfiles without CMD or ENTRYPOINT. - Sandbox construction does not guarantee that the process remains - running or healthy after dispatch. + The handle is available only when ``DockerfileLaunch.auto_start_cmd`` + is true and the Dockerfile declares a startup command. It is None for + normal image/rootfs launches, disabled startup dispatch, or Dockerfiles + without CMD or ENTRYPOINT. Sandbox construction does not guarantee that + the process remains running or healthy after dispatch. """ return self._startup_command diff --git a/sdk/python/docs/launch-from-dockerfile.md b/sdk/python/docs/launch-from-dockerfile.md new file mode 100644 index 0000000..dcbe1ad --- /dev/null +++ b/sdk/python/docs/launch-from-dockerfile.md @@ -0,0 +1,161 @@ +# Experimental: Launch a sandbox from a Dockerfile + +> **Warning:** This is the first AKernel release of direct Dockerfile launch. It +> implements a deliberately strict, narrow subset whose API and behavior can +> change. It is not a general-purpose Docker build facility and does not replace +> Docker, BuildKit, a registry, or an external image build pipeline. + +## Scope and root filesystem semantics + +Direct launch applies a supported Dockerfile to a fresh AKernel sandbox through +the public `Sandbox`, `Commands`, and `Filesystem` facades. It needs no +BuildKit, Docker daemon, or registry push. `FROM` is **rootfs-only**: its image +supplies only the sandbox root filesystem. OCI `ENV`, `USER`, `WORKDIR`, +`CMD`, and `ENTRYPOINT` configuration is not inherited, so declare every +required runtime setting in the Dockerfile passed through `DockerContext`. + +## Quick start + +Create one context, inspect its diagnostic result, and launch only when it is +direct-launchable: + +```python +from akernel_sdk import ( + DockerfileLaunch, LocalDockerContext, Sandbox, check_direct_launch, +) + +context = LocalDockerContext("Dockerfile", context_dir=".") +check = check_direct_launch(context) +for warning in check.warnings: + print(f"Dockerfile warning: {warning}") +if not check.direct_launchable: + raise RuntimeError(check.reasons) + +with Sandbox(dockerfile=DockerfileLaunch(context, run_timeout=300)) as sandbox: + startup = sandbox.startup_command + if startup is not None: + result = startup.wait(timeout=60) + print(result.exit_code, result.stderr) + # Perform the application's own health check when it is long-lived. +``` + +See the maintained end-to-end +[`examples/dockerfile_launch.py`](../examples/dockerfile_launch.py). + +## Precheck and DockerfileLaunch configuration + +`check_direct_launch(context)` is diagnostic and returns reason codes including +`multi_stage`, `remote_add`, `no_from`, `unsupported_instruction`, and +`unsupported_syntax`. A false `direct_launchable` result means direct launch +will fail closed; build externally instead. `Sandbox(dockerfile=...)` parses +strictly, `parse_dockerfile(strict=False)` is diagnostic only, and +`apply_dockerfile()` also rejects parsed unsupported syntax. + +Configure the immutable `DockerfileLaunch` value: + +| Field | Meaning | +| --- | --- | +| `context: DockerContext` | Dockerfile text and build context to apply. | +| `auto_start_cmd: bool = True` | Dispatch the resolved `CMD`/`ENTRYPOINT` in the background after build-time instructions. | +| `run_timeout: int = 600` | Positive timeout in seconds for each `RUN` instruction. | + +`dockerfile`, `image`, and `rootfs` are mutually exclusive constructor sources. + +## Supported Dockerfile subset + +| Supported direct-launch subset | Rejected; use an external build | +| --- | --- | +| Exactly one literal `FROM`, optionally `AS alias`; shell-form `RUN` | Multiple stages, `COPY --from`, `FROM` flags or variables, and exec-form `RUN` | +| Shell-form local `COPY` and `ADD` paths: files, directories, `.`, and wildcards; literal `--chown`; literal local tar extraction for `ADD` | JSON-form `COPY`/`ADD`, remote `ADD` URLs, `--chmod`, `--link`, unknown flags, or build-time variable expansion | +| Literal `ENV`, absolute `WORKDIR`, named `USER` values such as `app` or `root`, and `EXPOSE` metadata | Any `ARG`, relative `WORKDIR`, `USER user:group` or numeric UID/GID values, and `VOLUME`, `LABEL`, `HEALTHCHECK`, `SHELL`, `STOPSIGNAL`, `ONBUILD`, `MAINTAINER`, or unknown instructions | +| Exec- or shell-form `CMD` and `ENTRYPOINT`, normalized and combined | — | + +## DockerContext extension contract + +A `DockerContext` exposes Dockerfile text and structured +`DockerContextEntry` values from `walk()`. Each entry has a relative POSIX +`path`, `kind` of `file` or `directory`, and permission-bit `mode` from +`0o000` through `0o777`. Custom contexts must expose readable files, every +ancestor, and empty directories. Directory entries make empty directories and +their modes representable. + +Custom contexts can implement `dockerfile_ignore()`. A +`(diagnostic_name, bytes)` tuple supplies the active matcher. Only `None` +falls back to the manifest-root `.dockerignore`; empty `bytes` still denote a +present higher-priority matcher. Dockerfiles and ignore files that belong to the +filesystem context must be enumerated by `walk()`. They remain ordinary context +entries that `COPY`/`ADD` can select unless the active matcher excludes them. + +`LocalDockerContext` reads a local directory and rejects symbolic links. A +path-form Dockerfile uses adjacent `.dockerignore` when it exists, +including when empty; only its absence permits root `.dockerignore` fallback. +An inline Dockerfile creates no virtual context file. A Dockerfile outside the +context and its companion remain outside the manifest even if that companion +supplies the active matcher. + +## Ignore filtering and COPY/ADD selection + +Before sandbox operations, direct launch validates the manifest, then applies +Moby-compatible ordered ignore matching to files and directories. + +- Ignore patterns are cleaned like `filepath.Clean` and comments require `#` + in column one. Embedded `**` in `.dockerignore` can span directories; later + `!` patterns can re-include descendants. +- A re-included descendant keeps required ignored directory ancestors as virtual + selectable source directories. A fully ignored directory remains unavailable. + Alphanumeric backslash escapes and nested POSIX character classes that Moby + routes through its regular-expression engine are rejected, not literal. +- Direct launch plans every `COPY`/`ADD` before materializing files. Selected + directory entries create empty and nested directories, and every copied child + file or directory restores its own mode non-recursively. +- A literal directory source is a content container: its root mode is not + inherited, though the destination is created as needed. A wildcard that + matches a directory likewise copies its contents, not the matched name. + +Dockerfile `COPY`/`ADD` source patterns use Go `filepath.Match`-style +**segment** matching. Unlike `.dockerignore`, source `**` has the same +one-segment behavior as `*`. `[^a]` negates a character class, while `[!a]` +matches either `!` or `a`. Backslash escapes are outside this strict source +subset and malformed classes fail closed. When one wildcard expands to multiple +top-level sources after `.dockerignore` filtering, the destination must end in +`/`. Unsafe paths and destination collisions fail closed. + +`--chown` affects only files and directories created by the current instruction. +A local tar `ADD` accepts only regular files and directories with safe paths, +preserves tar member metadata, and always extracts as the builder/root identity +before applying `--chown`. Remote `ADD` URLs are rejected; the SDK never +fetches them. + +## Fail-closed security boundary + +The manifest is validated before selectable files are opened. Secure local +opening requires platform support for directory-relative file descriptors and +no-follow flags; unsupported platforms fail closed rather than weakening the +local-context boundary. Unsupported Dockerfile syntax, malformed patterns, +unsafe paths, missing sources, and target collisions also fail closed. + +## Startup and lifecycle + +Each launch executes `RUN`, `COPY`, and `ADD` again in a new sandbox. There +is no snapshot or build cache. After build-time instructions finish, the SDK +polls **sandbox readiness** and, when enabled, dispatches the resolved +`CMD`/`ENTRYPOINT` in the background. `Sandbox.startup_command` is a +`CommandHandle | None`; use `wait()` for finite commands or `kill()` when +needed. Construction confirms dispatch, not that the application remains +running or passes a health check. Callers own application readiness checks. + +## External-build fallback + +For Dockerfiles outside this subset, or build-once reuse, build with the chosen +external build system and use `Sandbox(image=...)`. Then explicitly launch the +desired command with `sandbox.commands.run(..., background=True)`; image +configuration does not auto-start `CMD` or `ENTRYPOINT` in this SDK path. + +## Parser, matching, and license references + +Line-level parsing uses +[`dockerfile-parse`](https://github.com/containerbuildsystem/dockerfile-parse) +(BSD-3-Clause). `.dockerignore` behavior follows +[`moby/patternmatcher`](https://github.com/moby/patternmatcher) (Apache-2.0). +The value post-processing approach was informed by the +[`E2B Python SDK`](https://github.com/e2b-dev/E2B/tree/main/packages/python-sdk) ([MIT license](https://github.com/e2b-dev/E2B/blob/main/packages/python-sdk/LICENSE)). diff --git a/sdk/python/examples/dockerfile_launch.py b/sdk/python/examples/dockerfile_launch.py index f1ffa3a..0f559e1 100644 --- a/sdk/python/examples/dockerfile_launch.py +++ b/sdk/python/examples/dockerfile_launch.py @@ -35,7 +35,7 @@ import tempfile from pathlib import Path -from akernel_sdk import LocalDockerContext, Sandbox, check_direct_launch +from akernel_sdk import DockerfileLaunch, LocalDockerContext, Sandbox, check_direct_launch def _precheck(context: LocalDockerContext) -> None: @@ -106,7 +106,7 @@ def section_core_path() -> None: context = LocalDockerContext(dockerfile, context_dir=context_dir) _precheck(context) - with Sandbox(context=context, build_run_timeout=300) as sandbox: + with Sandbox(dockerfile=DockerfileLaunch(context, run_timeout=300)) as sandbox: startup = sandbox.startup_command assert startup is not None result = startup.wait(timeout=60) @@ -165,7 +165,7 @@ def section_entrypoint_cmd_merge() -> None: context = LocalDockerContext(dockerfile, context_dir=context_dir) _precheck(context) - with Sandbox(context=context) as sandbox: + with Sandbox(dockerfile=DockerfileLaunch(context)) as sandbox: startup = sandbox.startup_command assert startup is not None result = startup.wait(timeout=60) @@ -194,7 +194,7 @@ def section_entrypoint_only() -> None: ) _precheck(context) - with Sandbox(context=context) as sandbox: + with Sandbox(dockerfile=DockerfileLaunch(context)) as sandbox: startup = sandbox.startup_command assert startup is not None result = startup.wait(timeout=60) @@ -215,7 +215,7 @@ def section_shell_cmd() -> None: ) _precheck(context) - with Sandbox(context=context) as sandbox: + with Sandbox(dockerfile=DockerfileLaunch(context)) as sandbox: startup = sandbox.startup_command assert startup is not None result = startup.wait(timeout=60) @@ -239,7 +239,7 @@ def section_shell_entrypoint_ignores_cmd() -> None: ) _precheck(context) - with Sandbox(context=context) as sandbox: + with Sandbox(dockerfile=DockerfileLaunch(context)) as sandbox: startup = sandbox.startup_command assert startup is not None result = startup.wait(timeout=60) @@ -264,7 +264,7 @@ def section_auto_start_disabled() -> None: ) _precheck(context) - with Sandbox(context=context, auto_start_cmd=False) as sandbox: + with Sandbox(dockerfile=DockerfileLaunch(context, auto_start_cmd=False)) as sandbox: assert sandbox.startup_command is None absent = sandbox.commands.run("test ! -e /tmp/disabled-start.out") assert absent.exit_code == 0, absent.stderr @@ -285,7 +285,7 @@ def section_copy_chown() -> None: context = LocalDockerContext(dockerfile, context_dir=context_dir) _precheck(context) - with Sandbox(context=context) as sandbox: + with Sandbox(dockerfile=DockerfileLaunch(context)) as sandbox: assert sandbox.startup_command is None owner = sandbox.commands.run("stat -c '%U:%G' /data/payload.txt") assert owner.exit_code == 0, owner.stderr @@ -317,7 +317,7 @@ def section_add_tar() -> None: ) _precheck(context) - with Sandbox(context=context) as sandbox: + with Sandbox(dockerfile=DockerfileLaunch(context)) as sandbox: assert sandbox.startup_command is None listing = sandbox.commands.run("find /opt/app -type f -printf '%P\n'") assert listing.exit_code == 0, listing.stderr diff --git a/sdk/python/tests/unit/test_dockerfile.py b/sdk/python/tests/unit/test_dockerfile.py index bce0edc..cd0c754 100644 --- a/sdk/python/tests/unit/test_dockerfile.py +++ b/sdk/python/tests/unit/test_dockerfile.py @@ -39,6 +39,7 @@ CmdInstruction, CopyInstruction, DockerfileBuildError, + DockerfileLaunch, DockerfileParseError, RunInstruction, UserInstruction, @@ -190,6 +191,43 @@ def walk(self) -> Iterator[DockerContextEntry]: ) +class TestDockerfileLaunch(unittest.TestCase): + def test_defaults_and_custom_values(self): + context = LocalDockerContext("FROM ubuntu\n") + defaults = DockerfileLaunch(context) + self.assertIs(defaults.context, context) + self.assertTrue(defaults.auto_start_cmd) + self.assertEqual(defaults.run_timeout, 600) + + custom = DockerfileLaunch( + context, + auto_start_cmd=False, + run_timeout=300, + ) + self.assertFalse(custom.auto_start_cmd) + self.assertEqual(custom.run_timeout, 300) + + def test_is_frozen(self): + launch = DockerfileLaunch(LocalDockerContext("FROM ubuntu\n")) + with self.assertRaisesRegex(AttributeError, "cannot assign"): + launch.run_timeout = 300 # type: ignore[misc] + + def test_rejects_invalid_values(self): + context = LocalDockerContext("FROM ubuntu\n") + with self.assertRaisesRegex(TypeError, "context"): + DockerfileLaunch(object()) # type: ignore[arg-type] + with self.assertRaisesRegex(TypeError, "auto_start_cmd"): + DockerfileLaunch(context, auto_start_cmd=1) # type: ignore[arg-type] + for timeout in (True, 1.5): + with self.subTest(timeout=timeout): + with self.assertRaisesRegex(TypeError, "run_timeout"): + DockerfileLaunch(context, run_timeout=timeout) # type: ignore[arg-type] + for timeout in (0, -1): + with self.subTest(timeout=timeout): + with self.assertRaisesRegex(ValueError, "run_timeout"): + DockerfileLaunch(context, run_timeout=timeout) + + class TestParseDockerfile(unittest.TestCase): def _parse(self, content, strict=False): return parse_dockerfile(LocalDockerContext(content), strict=strict) diff --git a/sdk/python/tests/unit/test_sandbox.py b/sdk/python/tests/unit/test_sandbox.py index 8a4ee1e..065c13c 100644 --- a/sdk/python/tests/unit/test_sandbox.py +++ b/sdk/python/tests/unit/test_sandbox.py @@ -12,11 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +import inspect import os import unittest from unittest.mock import MagicMock, patch -from akernel_sdk import HttpReverseTunnel, NetworkPolicy, S3Config, Sandbox +from akernel_sdk import ( + DockerfileLaunch, + HttpReverseTunnel, + NetworkPolicy, + S3Config, + Sandbox, +) from akernel_sdk import sandbox as sandbox_module from akernel_sdk._dockercontext import LocalDockerContext from akernel_sdk._dockerfile import DockerfileBuildError, DockerfileParseError @@ -324,6 +331,30 @@ def test_partial_facade_cleanup_preserves_initialization_error(self): self.session.terminate.assert_called_once_with() self.session.close.assert_called_once_with() + def test_dockerfile_signature_and_mutual_exclusion(self): + parameters = inspect.signature(Sandbox).parameters + self.assertIn("dockerfile", parameters) + self.assertNotIn("context", parameters) + self.assertNotIn("auto_start_cmd", parameters) + self.assertNotIn("build_run_timeout", parameters) + + dockerfile = DockerfileLaunch(LocalDockerContext("FROM ubuntu\n")) + for kwargs in ( + {"image": "ubuntu", "dockerfile": dockerfile}, + { + "rootfs": S3Config("https://s3.example.com", "bucket", "rootfs"), + "dockerfile": dockerfile, + }, + ): + with self.subTest(kwargs=kwargs): + with self.assertRaisesRegex(ValueError, "mutually exclusive"): + Sandbox(**kwargs) + with self.assertRaisesRegex(TypeError, "dockerfile"): + Sandbox(dockerfile=object()) # type: ignore[arg-type] + with self.assertRaises(TypeError): + Sandbox(context=LocalDockerContext("FROM ubuntu\n")) # type: ignore[call-arg] + self.backend.create.assert_not_called() + def test_dockerfile_context_is_strict_before_backend_creation(self): unsupported = ( "ADD https://example.test/a /opt/", @@ -345,11 +376,16 @@ def test_dockerfile_context_is_strict_before_backend_creation(self): else f"FROM ubuntu\n{instruction}\n" ) with self.assertRaises(DockerfileParseError): - Sandbox(context=LocalDockerContext(dockerfile)) + Sandbox(dockerfile=DockerfileLaunch(LocalDockerContext(dockerfile))) self.backend.create.assert_not_called() def test_dockerfile_context_uses_base_image_and_applies_after_facades(self): context = LocalDockerContext("FROM ubuntu:24.04\nRUN true\n") + dockerfile = DockerfileLaunch( + context, + auto_start_cmd=False, + run_timeout=300, + ) apply_result = DockerfileApplyResult( start_cmd=None, startup_command=None, @@ -360,7 +396,7 @@ def test_dockerfile_context_uses_base_image_and_applies_after_facades(self): "akernel_sdk._dockerfile_runner.apply_dockerfile", return_value=apply_result, ) as apply: - sandbox = Sandbox(context=context) + sandbox = Sandbox(dockerfile=dockerfile) spec = self.backend.create.call_args.args[0] self.assertEqual(spec.image, "ubuntu:24.04") @@ -369,8 +405,8 @@ def test_dockerfile_context_uses_base_image_and_applies_after_facades(self): self.assertIsNotNone(sandbox._commands) self.assertIsNotNone(sandbox._pty) self.assertIs(apply.call_args.args[2], context) - self.assertTrue(apply.call_args.kwargs["auto_start_cmd"]) - self.assertEqual(apply.call_args.kwargs["run_timeout"], 600) + self.assertFalse(apply.call_args.kwargs["auto_start_cmd"]) + self.assertEqual(apply.call_args.kwargs["run_timeout"], 300) self.assertIsNone(sandbox.startup_command) sandbox.kill() @@ -387,7 +423,7 @@ def test_dockerfile_context_exposes_startup_command_handle(self): "akernel_sdk._dockerfile_runner.apply_dockerfile", return_value=apply_result, ): - sandbox = Sandbox(context=context) + sandbox = Sandbox(dockerfile=DockerfileLaunch(context)) self.assertIs(sandbox.startup_command, startup_handle) sandbox.kill() @@ -406,7 +442,12 @@ def test_dockerfile_context_without_dispatched_command_has_no_startup_handle(sel "akernel_sdk._dockerfile_runner.apply_dockerfile", return_value=apply_result, ): - sandbox = Sandbox(context=context, auto_start_cmd=auto_start_cmd) + sandbox = Sandbox( + dockerfile=DockerfileLaunch( + context, + auto_start_cmd=auto_start_cmd, + ) + ) self.assertIsNone(sandbox.startup_command) sandbox.kill() @@ -421,7 +462,9 @@ def test_dockerfile_startup_dispatch_failure_terminates_and_closes_session(self) ): with self.assertRaises(DockerfileBuildError) as raised: Sandbox( - context=LocalDockerContext('FROM ubuntu\nCMD ["server"]\n'), + dockerfile=DockerfileLaunch( + LocalDockerContext('FROM ubuntu\nCMD ["server"]\n') + ), detached=True, ) @@ -437,7 +480,9 @@ def test_dockerfile_build_failure_terminates_and_closes_detached_session(self): ): with self.assertRaises(DockerfileBuildError) as raised: Sandbox( - context=LocalDockerContext("FROM ubuntu\nRUN false\n"), + dockerfile=DockerfileLaunch( + LocalDockerContext("FROM ubuntu\nRUN false\n") + ), detached=True, ) diff --git a/sdk/python/tests/unit/test_types.py b/sdk/python/tests/unit/test_types.py index 0c9eefc..0cad809 100644 --- a/sdk/python/tests/unit/test_types.py +++ b/sdk/python/tests/unit/test_types.py @@ -36,6 +36,11 @@ def test_lightweight_imports_do_not_load_yuanrong(self): assert 'yr' not in sys.modules assert 'yr_sandbox' not in sys.modules assert 'YR_HTTP_CONNECTION_NUM' not in os.environ +assert 'akernel_sdk._dockerfile' not in sys.modules +from akernel_sdk import Sandbox +assert 'akernel_sdk._dockerfile' not in sys.modules +from akernel_sdk import DockerfileLaunch +assert 'akernel_sdk._dockerfile' in sys.modules """ result = subprocess.run( [sys.executable, "-c", code], @@ -71,6 +76,7 @@ def test_public_exports_are_minimal(self): "UnsupportedBackendFeatureError", "BackendOperationError", "DockerContext", + "DockerfileLaunch", "DockerContextEntry", "LocalDockerContext", "parse_dockerfile", From a18dee52f15da63407dc60d7fd8c4089d7c8a101 Mon Sep 17 00:00:00 2001 From: Peng-YM Date: Thu, 20 Aug 2026 12:04:50 +0800 Subject: [PATCH 11/12] docs(sdk): mark Dockerfile launch as supported Remove the Experimental status from Dockerfile direct launch now that the capability is supported and directly usable. The API will not be deprecated; the documented strict subset may grow only through backward-compatible additions, while unsupported inputs retain the existing fail-closed boundary. Signed-off-by: Peng-YM --- AGENTS.md | 9 ++++++--- sdk/python/README.md | 11 +++++++---- sdk/python/akernel_sdk/_dockerfile.py | 5 ++++- sdk/python/akernel_sdk/sandbox.py | 15 +++++++++------ sdk/python/docs/launch-from-dockerfile.md | 14 ++++++++------ 5 files changed, 34 insertions(+), 20 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 92f757a..d76f2de 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -347,10 +347,13 @@ Keep public `Sandbox`, `Commands`, `Filesystem`, and value types independent of both native packages; all native conversions belong under `akernel_sdk._backends`. -The SDK also supports an experimental, strict Dockerfile direct-launch subset -through `DockerContext` and +Dockerfile direct launch is a supported AKernel SDK capability through +`DockerContext` and `Sandbox(dockerfile=DockerfileLaunch(context=..., auto_start_cmd=..., run_timeout=...))`. -Read [`sdk/python/docs/launch-from-dockerfile.md`](./sdk/python/docs/launch-from-dockerfile.md) +This API will not be deprecated. Future releases may expand the documented +strict subset only through backward-compatible additions; unsupported inputs +continue to fail closed. Read +[`sdk/python/docs/launch-from-dockerfile.md`](./sdk/python/docs/launch-from-dockerfile.md) before changing this path. `FROM` supplies only the root filesystem; inherited OCI configuration is not applied. Runtime availability and compatibility remain backend-owned. `DockerContext.walk()` exposes public structured file and diff --git a/sdk/python/README.md b/sdk/python/README.md index 8a64b7d..e43e32e 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -24,7 +24,7 @@ It supports two backends: - [Port forwarding](#port-forwarding) - [Reverse tunnels](#reverse-tunnels) - [Rootfs and mounts](#rootfs-and-mounts) - - [Experimental: launch from a Dockerfile](#experimental-launch-from-a-dockerfile) + - [Launch from a Dockerfile](#launch-from-a-dockerfile) - [Resources and lifecycle](#resources-and-lifecycle) - [CLI](#cli) - [Examples and tests](#examples-and-tests) @@ -412,9 +412,12 @@ OCI images can also be mounted read-only: mount = Mount(target="/opt/tools", image_url="ubuntu:24.04") ``` -## Experimental: launch from a Dockerfile -> **Warning:** This first direct-launch release supports a strict, changing -> Dockerfile subset. It is not a general-purpose Docker build. +## Launch from a Dockerfile + +Dockerfile direct launch is a supported AKernel SDK capability. This API will +not be deprecated. Future releases may expand the documented strict subset +through backward-compatible additions; unsupported inputs continue to fail +closed. It is not a general-purpose Docker build. `FROM` supplies only the root filesystem; inherited OCI configuration is not applied. Precheck the context, then pass its launch configuration to `Sandbox`: diff --git a/sdk/python/akernel_sdk/_dockerfile.py b/sdk/python/akernel_sdk/_dockerfile.py index 74d5736..2c41a5b 100644 --- a/sdk/python/akernel_sdk/_dockerfile.py +++ b/sdk/python/akernel_sdk/_dockerfile.py @@ -33,7 +33,10 @@ @dataclass(frozen=True) class DockerfileLaunch: - """Experimental immutable configuration for a Dockerfile sandbox launch. + """Immutable supported configuration for Dockerfile direct launch. + + Dockerfile direct launch accepts the documented strict subset. Unsupported + inputs fail closed. Args: context: Dockerfile and build context to apply in the sandbox. diff --git a/sdk/python/akernel_sdk/sandbox.py b/sdk/python/akernel_sdk/sandbox.py index 5504b97..dc78fbc 100644 --- a/sdk/python/akernel_sdk/sandbox.py +++ b/sdk/python/akernel_sdk/sandbox.py @@ -169,12 +169,15 @@ def __init__( are validated against the selected runtime by the backend. network_policy: Optional creation-time network policy. Omitting it leaves sandbox networking unrestricted. - dockerfile: Experimental Dockerfile launch configuration. ``FROM`` - supplies only the root filesystem; its OCI ENV, USER, WORKDIR, - CMD and ENTRYPOINT configuration is not inherited. The sandbox - applies only state explicitly declared in this Dockerfile, then - executes build-time instructions in-sandbox. Mutually exclusive - with ``image`` and ``rootfs``. + dockerfile: Supported Dockerfile direct-launch configuration. This + API will not be deprecated; future releases may expand the + documented strict subset through backward-compatible additions, + and unsupported inputs fail closed. ``FROM`` supplies only the + root filesystem; its OCI ENV, USER, WORKDIR, CMD and ENTRYPOINT + configuration is not inherited. The sandbox applies only state + explicitly declared in this Dockerfile, then executes build-time + instructions in-sandbox. Mutually exclusive with ``image`` and + ``rootfs``. Raises: TypeError: An argument has an invalid type. diff --git a/sdk/python/docs/launch-from-dockerfile.md b/sdk/python/docs/launch-from-dockerfile.md index dcbe1ad..ebae60e 100644 --- a/sdk/python/docs/launch-from-dockerfile.md +++ b/sdk/python/docs/launch-from-dockerfile.md @@ -1,9 +1,11 @@ -# Experimental: Launch a sandbox from a Dockerfile - -> **Warning:** This is the first AKernel release of direct Dockerfile launch. It -> implements a deliberately strict, narrow subset whose API and behavior can -> change. It is not a general-purpose Docker build facility and does not replace -> Docker, BuildKit, a registry, or an external image build pipeline. +# Launch a sandbox from a Dockerfile + +Dockerfile direct launch is a supported, currently available AKernel SDK +capability. This API will not be deprecated and is not scheduled for removal. +Future releases may expand the documented strict subset through +backward-compatible additions; unsupported inputs continue to fail closed. It +is not a general-purpose Docker build facility and does not replace Docker, +BuildKit, a registry, or an external image build pipeline. ## Scope and root filesystem semantics From 0f5a969e531a67a257f1a1cb774467dc10086f15 Mon Sep 17 00:00:00 2001 From: Peng-YM Date: Thu, 20 Aug 2026 15:24:47 +0800 Subject: [PATCH 12/12] fix(sdk): resolve Dockerfile launch annotations Move DockerfileLaunch into a lightweight value module so public annotations resolve at runtime without loading the parser or a backend. Keep parser and backend imports lazy, and calibrate the supported-capability lifecycle wording to allow API evolution with migration guidance. Signed-off-by: Peng-YM --- AGENTS.md | 9 ++-- sdk/python/README.md | 9 ++-- sdk/python/akernel_sdk/__init__.py | 2 +- sdk/python/akernel_sdk/_dockerfile.py | 33 +----------- sdk/python/akernel_sdk/_dockerfile_launch.py | 53 ++++++++++++++++++++ sdk/python/akernel_sdk/_dockerfile_runner.py | 6 +-- sdk/python/akernel_sdk/sandbox.py | 19 +++---- sdk/python/docs/launch-from-dockerfile.md | 8 +-- sdk/python/tests/unit/test_types.py | 19 +++++-- 9 files changed, 96 insertions(+), 62 deletions(-) create mode 100644 sdk/python/akernel_sdk/_dockerfile_launch.py diff --git a/AGENTS.md b/AGENTS.md index d76f2de..55f8bed 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,6 +30,8 @@ tunnels. The project overview and deployment quick start are in - `sdk/python/` - AKernel Python SDK and CLI. - `sdk/python/akernel_sdk/` - SDK implementation for `Sandbox`, commands, filesystem, PTY support, instance plumbing, and CLI helpers. +- `sdk/python/akernel_sdk/_dockerfile_launch.py` - lightweight public + Dockerfile direct-launch configuration, independent of the parser and backend. - `sdk/python/examples/` - maintained AKernel SDK examples. - `sdk/python/tests/` - maintained AKernel SDK tests. - `src/yuanrong/` - pinned openYuanRong mirror checkout, including its @@ -350,9 +352,10 @@ of both native packages; all native conversions belong under Dockerfile direct launch is a supported AKernel SDK capability through `DockerContext` and `Sandbox(dockerfile=DockerfileLaunch(context=..., auto_start_cmd=..., run_timeout=...))`. -This API will not be deprecated. Future releases may expand the documented -strict subset only through backward-compatible additions; unsupported inputs -continue to fail closed. Read +The capability will remain available. Its documented strict subset evolves +incrementally with production experience, while unsupported inputs continue to +fail closed. The specific API surface may evolve; material changes require +documentation and migration guidance. Read [`sdk/python/docs/launch-from-dockerfile.md`](./sdk/python/docs/launch-from-dockerfile.md) before changing this path. `FROM` supplies only the root filesystem; inherited OCI configuration is not applied. Runtime availability and compatibility remain diff --git a/sdk/python/README.md b/sdk/python/README.md index e43e32e..7396350 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -414,10 +414,11 @@ mount = Mount(target="/opt/tools", image_url="ubuntu:24.04") ## Launch from a Dockerfile -Dockerfile direct launch is a supported AKernel SDK capability. This API will -not be deprecated. Future releases may expand the documented strict subset -through backward-compatible additions; unsupported inputs continue to fail -closed. It is not a general-purpose Docker build. +Dockerfile direct launch is a supported AKernel SDK capability and will remain +available. Its documented strict subset evolves incrementally with production +experience; unsupported inputs continue to fail closed. The specific API surface +may evolve, with documentation and migration guidance for material changes. It +is not a general-purpose Docker build. `FROM` supplies only the root filesystem; inherited OCI configuration is not applied. Precheck the context, then pass its launch configuration to `Sandbox`: diff --git a/sdk/python/akernel_sdk/__init__.py b/sdk/python/akernel_sdk/__init__.py index 3cc46a0..5e77dd9 100644 --- a/sdk/python/akernel_sdk/__init__.py +++ b/sdk/python/akernel_sdk/__init__.py @@ -79,7 +79,7 @@ "PtyError": (".pty", "PtyError"), "resources": ("._resources", "resources"), "DockerContext": ("._dockercontext", "DockerContext"), - "DockerfileLaunch": ("._dockerfile", "DockerfileLaunch"), + "DockerfileLaunch": ("._dockerfile_launch", "DockerfileLaunch"), "DockerContextEntry": ("._dockercontext", "DockerContextEntry"), "LocalDockerContext": ("._dockercontext", "LocalDockerContext"), "parse_dockerfile": ("._dockerfile", "parse_dockerfile"), diff --git a/sdk/python/akernel_sdk/_dockerfile.py b/sdk/python/akernel_sdk/_dockerfile.py index 2c41a5b..6404f1b 100644 --- a/sdk/python/akernel_sdk/_dockerfile.py +++ b/sdk/python/akernel_sdk/_dockerfile.py @@ -29,38 +29,7 @@ from dataclasses import dataclass, field from ._dockercontext import DockerContext - - -@dataclass(frozen=True) -class DockerfileLaunch: - """Immutable supported configuration for Dockerfile direct launch. - - Dockerfile direct launch accepts the documented strict subset. Unsupported - inputs fail closed. - - Args: - context: Dockerfile and build context to apply in the sandbox. - auto_start_cmd: Dispatch the Dockerfile CMD/ENTRYPOINT after applying - build-time instructions. Defaults to ``True``. - run_timeout: Positive per-``RUN`` timeout in seconds. Defaults to - ``600``. - """ - - context: DockerContext - auto_start_cmd: bool = True - run_timeout: int = 600 - - def __post_init__(self) -> None: - if not isinstance(self.context, DockerContext): - raise TypeError("context must be a DockerContext") - if not isinstance(self.auto_start_cmd, bool): - raise TypeError("auto_start_cmd must be a boolean") - if isinstance(self.run_timeout, bool) or not isinstance( - self.run_timeout, int - ): - raise TypeError("run_timeout must be an integer") - if self.run_timeout <= 0: - raise ValueError("run_timeout must be greater than zero") +from ._dockerfile_launch import DockerfileLaunch # noqa: F401 @dataclass(frozen=True) diff --git a/sdk/python/akernel_sdk/_dockerfile_launch.py b/sdk/python/akernel_sdk/_dockerfile_launch.py new file mode 100644 index 0000000..2b1b232 --- /dev/null +++ b/sdk/python/akernel_sdk/_dockerfile_launch.py @@ -0,0 +1,53 @@ +# Copyright (c) 2026 Ant Group Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Lightweight public configuration for Dockerfile direct launch.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from ._dockercontext import DockerContext + + +@dataclass(frozen=True) +class DockerfileLaunch: + """Immutable supported configuration for Dockerfile direct launch. + + Dockerfile direct launch accepts the documented strict subset. Unsupported + inputs fail closed. + + Args: + context: Dockerfile and build context to apply in the sandbox. + auto_start_cmd: Dispatch the Dockerfile CMD/ENTRYPOINT after applying + build-time instructions. Defaults to ``True``. + run_timeout: Positive per-``RUN`` timeout in seconds. Defaults to + ``600``. + """ + + context: DockerContext + auto_start_cmd: bool = True + run_timeout: int = 600 + + def __post_init__(self) -> None: + if not isinstance(self.context, DockerContext): + raise TypeError("context must be a DockerContext") + if not isinstance(self.auto_start_cmd, bool): + raise TypeError("auto_start_cmd must be a boolean") + if isinstance(self.run_timeout, bool) or not isinstance( + self.run_timeout, int + ): + raise TypeError("run_timeout must be an integer") + if self.run_timeout <= 0: + raise ValueError("run_timeout must be greater than zero") diff --git a/sdk/python/akernel_sdk/_dockerfile_runner.py b/sdk/python/akernel_sdk/_dockerfile_runner.py index a7250f2..addf25f 100644 --- a/sdk/python/akernel_sdk/_dockerfile_runner.py +++ b/sdk/python/akernel_sdk/_dockerfile_runner.py @@ -32,7 +32,7 @@ import time from collections.abc import Iterable from dataclasses import dataclass -from typing import TYPE_CHECKING, Literal +from typing import Literal from ._dockercontext import DockerContext, _ContextManifest from ._dockerfile import ( @@ -50,9 +50,7 @@ resolve_start_cmd, ) from .commands import CommandHandle - -if TYPE_CHECKING: - from .sandbox import Sandbox +from .sandbox import Sandbox @dataclass(frozen=True) diff --git a/sdk/python/akernel_sdk/sandbox.py b/sdk/python/akernel_sdk/sandbox.py index dc78fbc..4a9c147 100644 --- a/sdk/python/akernel_sdk/sandbox.py +++ b/sdk/python/akernel_sdk/sandbox.py @@ -22,20 +22,17 @@ import urllib.request from collections.abc import Mapping, Sequence from types import MappingProxyType -from typing import TYPE_CHECKING from ._addresses import Endpoint, api_endpoint_from_env, gateway_endpoint_from_env from ._backends.base import BackendSession, SandboxSpec from ._backends.registry import load_backend +from ._dockerfile_launch import DockerfileLaunch from ._sandbox_resources import normalize_xpu, validate_storage_mb from .commands import CommandHandle, Commands from .filesystem import Filesystem from .pty import Pty from .types import HttpReverseTunnel, Mount, NetworkPolicy, S3Config, SandboxInfo -if TYPE_CHECKING: - from ._dockerfile import DockerfileLaunch - _traefik_internal_ip_cache: str | None = None logger = logging.getLogger(__name__) @@ -169,11 +166,13 @@ def __init__( are validated against the selected runtime by the backend. network_policy: Optional creation-time network policy. Omitting it leaves sandbox networking unrestricted. - dockerfile: Supported Dockerfile direct-launch configuration. This - API will not be deprecated; future releases may expand the - documented strict subset through backward-compatible additions, - and unsupported inputs fail closed. ``FROM`` supplies only the - root filesystem; its OCI ENV, USER, WORKDIR, CMD and ENTRYPOINT + dockerfile: Supported Dockerfile direct-launch configuration. + Dockerfile direct launch remains available as a supported + capability. Its documented strict subset evolves incrementally + with production experience; unsupported inputs fail closed. The + specific API surface may evolve, with documentation and migration + guidance for material changes. ``FROM`` supplies only the root + filesystem; its OCI ENV, USER, WORKDIR, CMD and ENTRYPOINT configuration is not inherited. The sandbox applies only state explicitly declared in this Dockerfile, then executes build-time instructions in-sandbox. Mutually exclusive with ``image`` and @@ -190,8 +189,6 @@ def __init__( if rootfs is not None and not isinstance(rootfs, S3Config): raise TypeError("rootfs must be an S3Config") if dockerfile is not None: - from ._dockerfile import DockerfileLaunch - if not isinstance(dockerfile, DockerfileLaunch): raise TypeError("dockerfile must be a DockerfileLaunch") if sum(value is not None for value in (image, rootfs, dockerfile)) > 1: diff --git a/sdk/python/docs/launch-from-dockerfile.md b/sdk/python/docs/launch-from-dockerfile.md index ebae60e..f4428fa 100644 --- a/sdk/python/docs/launch-from-dockerfile.md +++ b/sdk/python/docs/launch-from-dockerfile.md @@ -1,9 +1,9 @@ # Launch a sandbox from a Dockerfile -Dockerfile direct launch is a supported, currently available AKernel SDK -capability. This API will not be deprecated and is not scheduled for removal. -Future releases may expand the documented strict subset through -backward-compatible additions; unsupported inputs continue to fail closed. It +Dockerfile direct launch is a supported AKernel SDK capability and will remain +available. Its documented strict subset evolves incrementally with production +experience; unsupported inputs continue to fail closed. The specific API surface +may evolve, with documentation and migration guidance for material changes. It is not a general-purpose Docker build facility and does not replace Docker, BuildKit, a registry, or an external image build pipeline. diff --git a/sdk/python/tests/unit/test_types.py b/sdk/python/tests/unit/test_types.py index 0cad809..e9766c4 100644 --- a/sdk/python/tests/unit/test_types.py +++ b/sdk/python/tests/unit/test_types.py @@ -30,6 +30,8 @@ def test_lightweight_imports_do_not_load_yuanrong(self): code = """ import os import sys +from typing import get_type_hints + os.environ.pop('YR_HTTP_CONNECTION_NUM', None) import akernel_sdk import akernel_sdk.cli @@ -37,10 +39,21 @@ def test_lightweight_imports_do_not_load_yuanrong(self): assert 'yr_sandbox' not in sys.modules assert 'YR_HTTP_CONNECTION_NUM' not in os.environ assert 'akernel_sdk._dockerfile' not in sys.modules -from akernel_sdk import Sandbox +from akernel_sdk import DockerfileLaunch, Sandbox +assert 'akernel_sdk._dockerfile_launch' in sys.modules +assert 'akernel_sdk._dockercontext' in sys.modules assert 'akernel_sdk._dockerfile' not in sys.modules -from akernel_sdk import DockerfileLaunch -assert 'akernel_sdk._dockerfile' in sys.modules +assert 'dockerfile_parse' not in sys.modules +assert 'yr' not in sys.modules +assert 'yr_sandbox' not in sys.modules +assert 'akernel_sdk._backends.openyuanrong_sandbox' not in sys.modules +assert 'akernel_sdk._backends.openyuanrong_sdk' not in sys.modules +assert 'akernel_sdk._backends.openyuanrong_sdk_impl' not in sys.modules +assert get_type_hints(Sandbox.__init__)['dockerfile'] == DockerfileLaunch | None +from akernel_sdk._dockerfile import DockerfileLaunch as CompatDockerfileLaunch +assert CompatDockerfileLaunch is DockerfileLaunch +from akernel_sdk._dockerfile_runner import apply_dockerfile +assert get_type_hints(apply_dockerfile)['sb'] is Sandbox """ result = subprocess.run( [sys.executable, "-c", code],