Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/helm-lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ jobs:

- id: changes
if: github.event_name == 'push'
uses: tj-actions/changed-files@aa08304bd477b800d468db44fe10f6c61f7f7b11 # v42.1.0
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
with:
base_sha: ${{ steps.merge-base.outputs.base_sha }}
skip_initial_fetch: ${{ steps.merge-base.outputs.base_sha != '' }}
Expand Down
3 changes: 3 additions & 0 deletions crates/openshell-core/src/sandbox_env.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@ pub const LOG_LEVEL: &str = "OPENSHELL_LOG_LEVEL";
/// Shell command to run inside the sandbox.
pub const SANDBOX_COMMAND: &str = "OPENSHELL_SANDBOX_COMMAND";

/// JSON-serialized workspace substrate configuration for supervisor setup.
pub const WORKSPACE_CONFIG: &str = "OPENSHELL_WORKSPACE_CONFIG";

/// Deployment-controlled telemetry toggle propagated to the sandbox supervisor.
pub const TELEMETRY_ENABLED: &str = "OPENSHELL_TELEMETRY_ENABLED";

Expand Down
228 changes: 198 additions & 30 deletions crates/openshell-driver-docker/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ use url::Url;
const WATCH_BUFFER: usize = 128;
const WATCH_POLL_INTERVAL: Duration = Duration::from_secs(2);
const WATCH_POLL_MAX_BACKOFF: Duration = Duration::from_secs(30);
const WORKSPACE_BACKING_MOUNT_PATH: &str = "/var/lib/openshell/workspace";

const SUPERVISOR_MOUNT_PATH: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_BINARY;
const TLS_CA_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CA_MOUNT_PATH;
Expand Down Expand Up @@ -284,6 +285,7 @@ struct DockerSandboxDriverConfig {
)]
cdi_devices: Option<Vec<String>>,
mounts: Vec<DockerDriverMountConfig>,
workspace: Option<DockerWorkspaceConfig>,
}

impl DockerSandboxDriverConfig {
Expand Down Expand Up @@ -345,10 +347,30 @@ enum DockerDriverMountConfig {
},
}

#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
enum DockerWorkspaceConfig {
ForkcellOverlay {
volume: String,
target: String,
#[serde(default = "default_workspace_backing_path", skip_deserializing)]
backing_path: String,
lower_subpath: String,
upper_subpath: String,
work_subpath: String,
merged_subpath: String,
checkpoint_id: String,
},
}

fn default_true() -> bool {
true
}

fn default_workspace_backing_path() -> String {
WORKSPACE_BACKING_MOUNT_PATH.to_string()
}

type WatchStream =
Pin<Box<dyn Stream<Item = Result<WatchSandboxesEvent, Status>> + Send + 'static>>;

Expand Down Expand Up @@ -549,27 +571,32 @@ impl DockerComputeDriver {
let config = docker_driver_config(template, self.config.enable_bind_mounts)?;
for mount in config.mounts {
if let DockerDriverMountConfig::Volume { source, .. } = mount {
match self.docker.inspect_volume(source.trim()).await {
Ok(volume) => {
if !self.config.enable_bind_mounts && docker_volume_is_bind_backed(&volume)
{
return Err(Status::failed_precondition(format!(
"docker volume '{}' is backed by a host bind mount and requires enable_bind_mounts = true in [openshell.drivers.docker]",
source.trim()
)));
}
}
Err(err) if is_not_found_error(&err) => {
return Err(Status::failed_precondition(format!(
"docker volume '{}' does not exist",
source.trim()
)));
}
Err(err) => {
return Err(internal_status("inspect docker volume", err));
}
self.validate_named_volume_available(source.trim()).await?;
}
}
if let Some(DockerWorkspaceConfig::ForkcellOverlay { volume, .. }) = config.workspace {
self.validate_named_volume_available(volume.trim()).await?;
}
Ok(())
}

async fn validate_named_volume_available(&self, source: &str) -> Result<(), Status> {
match self.docker.inspect_volume(source).await {
Ok(volume) => {
if !self.config.enable_bind_mounts && docker_volume_is_bind_backed(&volume) {
return Err(Status::failed_precondition(format!(
"docker volume '{source}' is backed by a host bind mount and requires enable_bind_mounts = true in [openshell.drivers.docker]"
)));
}
}
Err(err) if is_not_found_error(&err) => {
return Err(Status::failed_precondition(format!(
"docker volume '{source}' does not exist"
)));
}
Err(err) => {
return Err(internal_status("inspect docker volume", err));
}
}
Ok(())
}
Expand Down Expand Up @@ -1723,18 +1750,10 @@ fn docker_driver_config(
) -> Result<DockerSandboxDriverConfig, Status> {
let config =
DockerSandboxDriverConfig::from_template(template).map_err(Status::invalid_argument)?;
validate_docker_driver_mounts(&config.mounts, enable_bind_mounts)?;
validate_docker_driver_config(&config, enable_bind_mounts)?;
Ok(config)
}

fn docker_driver_mounts(
template: &DriverSandboxTemplate,
enable_bind_mounts: bool,
) -> Result<Vec<Mount>, Status> {
let config = docker_driver_config(template, enable_bind_mounts)?;
config.mounts.iter().map(docker_mount_from_config).collect()
}

fn docker_mount_from_config(config: &DockerDriverMountConfig) -> Result<Mount, Status> {
match config {
DockerDriverMountConfig::Bind {
Expand Down Expand Up @@ -1885,6 +1904,127 @@ fn validate_docker_driver_mounts(
Ok(())
}

fn validate_docker_driver_config(
config: &DockerSandboxDriverConfig,
enable_bind_mounts: bool,
) -> Result<(), Status> {
validate_docker_driver_mounts(&config.mounts, enable_bind_mounts)?;
if let Some(workspace) = &config.workspace {
validate_docker_workspace_config(workspace)?;
let workspace_target = docker_workspace_target(workspace)?;
let workspace_backing_target = docker_workspace_backing_target(workspace)?;
for mount in &config.mounts {
let mount_target =
driver_mounts::validate_container_mount_target(docker_driver_mount_target(mount))
.map_err(Status::failed_precondition)?;
if mount_target == workspace_target {
return Err(Status::failed_precondition(format!(
"docker driver_config workspace target '{workspace_target}' duplicates a mount target"
)));
}
if mount_target == workspace_backing_target {
return Err(Status::failed_precondition(format!(
"docker driver_config workspace backing target '{workspace_backing_target}' duplicates a mount target"
)));
}
}
}
Ok(())
}

fn validate_docker_workspace_config(config: &DockerWorkspaceConfig) -> Result<(), Status> {
match config {
DockerWorkspaceConfig::ForkcellOverlay {
volume,
target,
backing_path,
lower_subpath,
upper_subpath,
work_subpath,
merged_subpath,
checkpoint_id,
} => {
driver_mounts::validate_mount_source(volume, "workspace volume")
.map_err(Status::failed_precondition)?;
let target = driver_mounts::validate_container_mount_target(target)
.map_err(Status::failed_precondition)?;
if target != "/sandbox/work" {
return Err(Status::failed_precondition(format!(
"forkcell_overlay workspace target must be '/sandbox/work', got '{target}'"
)));
}
let backing_path = driver_mounts::validate_container_mount_target(backing_path)
.map_err(Status::failed_precondition)?;
if backing_path == target {
return Err(Status::failed_precondition(
"forkcell_overlay workspace backing_path must differ from target",
));
}
for (field, subpath) in [
("workspace lower_subpath", lower_subpath),
("workspace upper_subpath", upper_subpath),
("workspace work_subpath", work_subpath),
("workspace merged_subpath", merged_subpath),
] {
driver_mounts::validate_mount_subpath(subpath)
.map_err(|err| Status::failed_precondition(format!("{field}: {err}")))?;
}
driver_mounts::validate_mount_source(checkpoint_id, "workspace checkpoint_id")
.map_err(Status::failed_precondition)?;
}
}
Ok(())
}

fn docker_workspace_target(config: &DockerWorkspaceConfig) -> Result<String, Status> {
match config {
DockerWorkspaceConfig::ForkcellOverlay { target, .. } => {
driver_mounts::validate_container_mount_target(target)
.map_err(Status::failed_precondition)
}
}
}

fn docker_workspace_backing_target(config: &DockerWorkspaceConfig) -> Result<String, Status> {
match config {
DockerWorkspaceConfig::ForkcellOverlay { backing_path, .. } => {
driver_mounts::validate_container_mount_target(backing_path)
.map_err(Status::failed_precondition)
}
}
}

fn docker_workspace_mount_from_config(config: &DockerWorkspaceConfig) -> Result<Mount, Status> {
match config {
DockerWorkspaceConfig::ForkcellOverlay {
volume,
backing_path,
..
} => Ok(Mount {
typ: Some(MountTypeEnum::VOLUME),
source: Some(
driver_mounts::validate_mount_source(volume, "workspace volume")
.map_err(Status::failed_precondition)?,
),
target: Some(
driver_mounts::validate_container_mount_target(backing_path)
.map_err(Status::failed_precondition)?,
),
read_only: Some(false),
..Default::default()
}),
}
}

fn docker_driver_mount_target(config: &DockerDriverMountConfig) -> &str {
match config {
DockerDriverMountConfig::Bind { target, .. }
| DockerDriverMountConfig::Volume { target, .. }
| DockerDriverMountConfig::Tmpfs { target, .. }
| DockerDriverMountConfig::Image { target, .. } => target,
}
}

fn validate_optional_positive_integral_i64(
value: Option<f64>,
field: &str,
Expand Down Expand Up @@ -2087,6 +2227,14 @@ fn cleanup_sandbox_token_file_by_id(sandbox_id: &str, config: &DockerDriverRunti
}

fn build_environment(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig) -> Vec<String> {
build_environment_with_driver_config(sandbox, config, None)
}

fn build_environment_with_driver_config(
sandbox: &DriverSandbox,
config: &DockerDriverRuntimeConfig,
driver_config: Option<&DockerSandboxDriverConfig>,
) -> Vec<String> {
let mut environment = HashMap::from([
("HOME".to_string(), "/root".to_string()),
("PATH".to_string(), SUPERVISOR_PATH.to_string()),
Expand Down Expand Up @@ -2134,6 +2282,14 @@ fn build_environment(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig
openshell_core::sandbox_env::SANDBOX_COMMAND.to_string(),
SANDBOX_COMMAND.to_string(),
);
if let Some(workspace) = driver_config.and_then(|config| config.workspace.as_ref())
&& let Ok(json) = serde_json::to_string(workspace)
{
environment.insert(
openshell_core::sandbox_env::WORKSPACE_CONFIG.to_string(),
json,
);
}
environment.insert(
openshell_core::sandbox_env::TELEMETRY_ENABLED.to_string(),
openshell_core::telemetry::enabled_env_value().to_string(),
Expand Down Expand Up @@ -2260,7 +2416,15 @@ fn build_container_create_body_with_default(
.as_ref()
.ok_or_else(|| Status::invalid_argument("sandbox.spec.template is required"))?;
let resource_limits = docker_resource_limits(template)?;
let user_mounts = docker_driver_mounts(template, config.enable_bind_mounts)?;
let driver_config = docker_driver_config(template, config.enable_bind_mounts)?;
let mut user_mounts = driver_config
.mounts
.iter()
.map(docker_mount_from_config)
.collect::<Result<Vec<_>, _>>()?;
if let Some(workspace) = &driver_config.workspace {
user_mounts.push(docker_workspace_mount_from_config(workspace)?);
}
let device_requests = build_device_requests(sandbox, selected_default_device)?;
let mut labels = template.labels.clone();
labels.insert(
Expand All @@ -2281,7 +2445,11 @@ fn build_container_create_body_with_default(
Ok(ContainerCreateBody {
image: Some(template.image.clone()),
user: Some("0".to_string()),
env: Some(build_environment(sandbox, config)),
env: Some(build_environment_with_driver_config(
sandbox,
config,
Some(&driver_config),
)),
entrypoint: Some(vec![SUPERVISOR_MOUNT_PATH.to_string()]),
// Clear the image CMD so Docker does not append inherited args to the
// supervisor entrypoint.
Expand Down
Loading
Loading