diff --git a/BREAKING_CHANGES.md b/BREAKING_CHANGES.md index 4083871..98a2355 100644 --- a/BREAKING_CHANGES.md +++ b/BREAKING_CHANGES.md @@ -1,5 +1,40 @@ # Breaking Changes +## Shell now matches GitHub Actions invocation (v0.7.3) + +The `bash` shell now executes with `bash --noprofile --norc -e -o pipefail -c`, matching GitHub Actions behavior. The `sh` shell uses `sh -e -c`. This means: + +- Scripts exit immediately on the first command that returns a non-zero exit code (`-e` / errexit) +- In bash, a failure in any command of a pipeline causes the whole pipeline to fail (`-o pipefail`) +- User profile/rc files are not sourced (`--noprofile --norc`) + +### Why + +GitHub Actions runs `bash` steps with `bash --noprofile --norc -e -o pipefail {0}`. The previous wrkflw behavior of `bash -c` (without `-e` or `pipefail`) allowed scripts to silently continue past failing commands, which diverged from GHA semantics and could mask real failures. + +### Impact + +Multi-command `run:` scripts that relied on intermediate commands failing without aborting the step will now fail at the first non-zero exit. Piped commands where an earlier stage fails will also now fail. For example: + +```yaml +- run: | + false # This now aborts the step + echo "This no longer runs" + +- run: | + false | echo "pipeline now fails too" +``` + +### Migration + +If a step intentionally tolerates command failures, either: + +- Append `|| true` to the specific command: `might-fail || true` +- Use `continue-on-error: true` on the step +- Add `set +e` or `set +o pipefail` at the top of the script to opt out selectively + +--- + ## EncryptedSecretStore serialization format (v0.7.3) The `EncryptedSecretStore` struct in `crates/secrets/src/storage.rs` has changed its serialization format: diff --git a/Cargo.lock b/Cargo.lock index 61acddb..04c47a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -982,6 +982,12 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + [[package]] name = "globset" version = "0.4.16" @@ -3374,6 +3380,7 @@ dependencies = [ "dirs", "futures", "futures-util", + "glob", "ignore", "lazy_static", "num_cpus", @@ -3383,6 +3390,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "sha2", "shlex", "tar", "tempfile", diff --git a/Cargo.toml b/Cargo.toml index 66f116c..538c24f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,6 +66,8 @@ libc = "0.2" nix = { version = "0.27.1", features = ["fs"] } urlencoding = "2.1.3" wiremock = "0.5" +glob = "0.3" +sha2 = "0.10" shlex = "1.3" [profile.release] diff --git a/INDEX.md b/INDEX.md index 6563474..5dbe275 100644 --- a/INDEX.md +++ b/INDEX.md @@ -1,16 +1,18 @@ # Codebase Index: wrkflw -> Generated: 2026-03-28 05:02:20 UTC | Files: 150 | Lines: 35146 -> Languages: JSON (4), Markdown (24), Rust (72), Shell (5), TOML (16), YAML (29) +> Generated: 2026-04-02 07:55:32 UTC | Files: 158 | Lines: 44819 +> Languages: C++ (1), JSON (4), Markdown (26), Python (1), Rust (75), Shell (5), TOML (16), YAML (30) ## Directory Structure ``` wrkflw/ AGENTS.md + BREAKING_CHANGES.md CLAUDE.md Cargo.toml GITLAB_USAGE.md + INDEX.md README.md VERSION_MANAGEMENT.md cliff.toml @@ -25,6 +27,7 @@ wrkflw/ Cargo.toml README.md src/ + action_resolver.rs dependency.rs docker.rs docker_test.rs @@ -152,16 +155,21 @@ wrkflw/ src/ lib.rs main.rs + tests/ + target_job_test.rs examples/ secrets-demo/ README.md secrets-workflow.yml + hello.cpp + hello.rs publish_crates.sh schemas/ github-workflow.json gitlab-ci.json scripts/ bump-crate.sh + test.py tests/ README.md TESTING_PODMAN.md @@ -197,6 +205,7 @@ wrkflw/ cpp-test.yml example.yml matrix-example.yml + multi-runtime-test.yml node-test.yml python-test.yml runs-on-array-test.yml @@ -213,6 +222,9 @@ wrkflw/ **AGENTS.md** - `# Codebase Navigation — Use indxr MCP tools` +**BREAKING_CHANGES.md** +- `# Breaking Changes` + **CLAUDE.md** - `# wrkflw` @@ -228,6 +240,9 @@ wrkflw/ - `# Trigger on a specific branch` - `# Trigger with custom variables` +**INDEX.md** +- `# Codebase Index: wrkflw` + **README.md** - `# WRKFLW` - `# Install Podman (varies by OS)` @@ -329,12 +344,21 @@ wrkflw/ **crates/executor/Cargo.toml** - `[package]` - `[dependencies]` +- `[dev-dependencies]` **crates/executor/README.md** - `## wrkflw-executor` +**crates/executor/src/action_resolver.rs** +- `pub enum ActionType` +- `pub struct ResolvedAction` +- `pub async fn resolve_remote_action( repo: &str, version: &str, sub_path: Option<&str>, ) -> Result` + **crates/executor/src/dependency.rs** - `pub fn resolve_dependencies(workflow: &WorkflowDefinition) -> Result>, String>` +- `pub fn collect_transitive_deps(target_job: &str, jobs: &HashMap) -> HashSet` +- `pub fn filter_plan_to_job( plan: Vec>, target_job: &str, jobs: &HashMap, kind: &str, ) -> Result>, String>` +- `pub fn filter_plan_to_job_by_stage( plan: Vec>, target_job: &str, jobs: &HashMap, kind: &str, ) -> Result>, String>` **crates/executor/src/docker.rs** - `pub struct DockerRuntime` @@ -367,6 +391,7 @@ wrkflw/ - `pub fn add_matrix_context( env: &mut HashMap, matrix_combination: &MatrixCombination, )` **crates/executor/src/lib.rs** +- `pub mod action_resolver` - `pub mod dependency` - `pub mod docker` - `pub mod engine` @@ -509,7 +534,10 @@ wrkflw/ - `pub struct SchemaValidator` **crates/parser/src/workflow.rs** +- `pub struct ContainerCredentials` +- `pub struct JobContainer` - `pub struct WorkflowDefinition` +- `pub struct Strategy` - `pub struct Job` - `pub struct Service` - `pub struct Step` @@ -532,6 +560,8 @@ wrkflw/ - `# This workflow will run successfully in secure emulation mode` **crates/runtime/src/container.rs** +- `pub const LOCAL_IMAGE_PREFIX: &str = "wrkflw-"` +- `pub const COMBINED_IMAGE_PREFIX: &str = "wrkflw-combined:"` - `pub trait ContainerRuntime` - `pub struct ContainerOutput` - `pub enum ContainerError` @@ -647,7 +677,7 @@ wrkflw/ - `# })?;` **crates/ui/src/app/mod.rs** -- `pub async fn run_wrkflw_tui( path: Option<&PathBuf>, runtime_type: RuntimeType, verbose: bool, preserve_containers_on_failure: bool, ) -> io::Result<()>` +- `pub async fn run_wrkflw_tui( path: Option<&PathBuf>, runtime_type: RuntimeType, verbose: bool, preserve_containers_on_failure: bool, show_action_messages: bool, ) -> io::Result<()>` **crates/ui/src/app/state.rs** - `pub struct App` @@ -666,7 +696,7 @@ wrkflw/ **crates/ui/src/handlers/workflow.rs** - `pub fn validate_workflow(path: &Path, verbose: bool) -> io::Result<()>` -- `pub async fn execute_workflow_cli( path: &Path, runtime_type: RuntimeType, verbose: bool, ) -> io::Result<()>` +- `pub async fn execute_workflow_cli( path: &Path, runtime_type: RuntimeType, verbose: bool, show_action_messages: bool, ) -> io::Result<()>` - `pub async fn execute_curl_trigger( workflow_name: &str, branch: Option<&str>, ) -> Result<(Vec, ()), String>` - `pub fn start_next_workflow_execution( app: &mut App, tx_clone: &mpsc::Sender, verbose: bool, )` @@ -811,6 +841,9 @@ wrkflw/ - `on:` - `jobs:` +**hello.cpp** +- `int main()` + **publish_crates.sh** - `show_help()` - `update_versions()` @@ -1042,6 +1075,11 @@ wrkflw/ - `env:` - `jobs:` +**tests/workflows/multi-runtime-test.yml** +- `name:` +- `on:` +- `jobs:` + **tests/workflows/node-test.yml** - `name:` - `on:` @@ -1085,6 +1123,14 @@ wrkflw/ --- +## BREAKING_CHANGES.md + +**Language:** Markdown | **Size:** 1.3 KB | **Lines:** 30 + +**Declarations:** + +--- + ## CLAUDE.md **Language:** Markdown | **Size:** 4.4 KB | **Lines:** 66 @@ -1095,7 +1141,7 @@ wrkflw/ ## Cargo.toml -**Language:** TOML | **Size:** 2.2 KB | **Lines:** 71 +**Language:** TOML | **Size:** 2.2 KB | **Lines:** 73 **Declarations:** @@ -1109,6 +1155,14 @@ wrkflw/ --- +## INDEX.md + +**Language:** Markdown | **Size:** 85.9 KB | **Lines:** 3732 + +**Declarations:** + +--- + ## README.md **Language:** Markdown | **Size:** 24.5 KB | **Lines:** 611 @@ -1175,7 +1229,7 @@ wrkflw/ ## crates/executor/Cargo.toml -**Language:** TOML | **Size:** 1.0 KB | **Lines:** 42 +**Language:** TOML | **Size:** 1.1 KB | **Lines:** 47 **Imports:** - `ignore` @@ -1186,27 +1240,73 @@ wrkflw/ ## crates/executor/README.md -**Language:** Markdown | **Size:** 880 B | **Lines:** 29 +**Language:** Markdown | **Size:** 902 B | **Lines:** 30 **Declarations:** --- +## crates/executor/src/action_resolver.rs + +**Language:** Rust | **Size:** 23.4 KB | **Lines:** 736 + +**Imports:** +- `once_cell::sync::Lazy` +- `std::collections::{HashMap, VecDeque}` +- `tokio::sync::RwLock` + +**Declarations:** + +`const MAX_CACHE_ENTRIES: usize = 256` + +`struct BoundedCache` +> Fields: `map: HashMap`, `order: VecDeque` + +**`impl BoundedCache`** + `fn new() -> Self` + + `fn get(&self, key: &str) -> Option<&ResolvedAction>` + + `fn insert(&mut self, key: String, value: ResolvedAction)` + + +`static ACTION_CACHE: Lazy> = Lazy::new(|| RwLock::new(BoundedCache::new()))` + +`static HTTP_CLIENT: Lazy = Lazy::new(||` + +`static NO_REDIRECT_CLIENT: Lazy = Lazy::new(||` + +`const GITHUB_RAW_BASE_URL: &str = "https://raw.githubusercontent.com"` + +`async fn fetch_and_parse( base_url: &str, repo: &str, version: &str, sub_path: Option<&str>, filename: &str, token: Option<&str>, ) -> Result` + +`fn parse_action_definition(content: &str) -> Result` + +`fn parse_using(using: &str, runs: &serde_yaml::Value) -> Result` + +`mod tests` + +--- + ## crates/executor/src/dependency.rs -**Language:** Rust | **Size:** 4.0 KB | **Lines:** 112 +**Language:** Rust | **Size:** 17.4 KB | **Lines:** 507 **Imports:** -- `std::collections::{HashMap, HashSet}` -- `wrkflw_parser::workflow::WorkflowDefinition` +- `std::collections::{HashMap, HashSet, VecDeque}` +- `wrkflw_parser::workflow::{Job, WorkflowDefinition}` **Declarations:** +`fn job_not_found_error(target_job: &str, jobs: &HashMap, kind: &str) -> String` + +`mod tests` + --- ## crates/executor/src/docker.rs -**Language:** Rust | **Size:** 44.0 KB | **Lines:** 1188 +**Language:** Rust | **Size:** 49.0 KB | **Lines:** 1303 **Imports:** - `async_trait::async_trait` @@ -1222,7 +1322,9 @@ wrkflw/ - `std::path::Path` - `std::sync::Mutex` - `wrkflw_logging` -- `wrkflw_runtime::container::{ContainerError, ContainerOutput, ContainerRuntime}` +- `wrkflw_runtime::container::{ + ContainerError, ContainerOutput, ContainerRuntime, COMBINED_IMAGE_PREFIX, LOCAL_IMAGE_PREFIX, +}` - `wrkflw_utils` - *... and 1 more imports* @@ -1253,28 +1355,30 @@ wrkflw/ **`impl ContainerRuntime for DockerRuntime`** - `async fn run_container( &self, image: &str, cmd: &[&str], env_vars: &[(&str, &str)], working_dir: &Path, volumes: &[(&Path, &Path)], ) -> Result` + `async fn run_container( &self, image: &str, cmd: &[&str], env_vars: &[(&str, &str)], working_dir: &Path, volumes: &[(&Path, &Path)], entrypoint: Option<&str>, ) -> Result` `async fn pull_image(&self, image: &str) -> Result<(), ContainerError>` - `async fn build_image(&self, dockerfile: &Path, tag: &str) -> Result<(), ContainerError>` + `async fn build_image( &self, dockerfile: &Path, tag: &str, context_dir: &Path, ) -> Result<(), ContainerError>` `async fn prepare_language_environment( &self, language: &str, version: Option<&str>, additional_packages: Option>, ) -> Result` + `async fn image_exists(&self, tag: &str) -> Result` + **`impl DockerRuntime`** - `async fn run_container_inner( &self, image: &str, cmd: &[&str], env_vars: &[(&str, &str)], working_dir: &Path, volumes: &[(&Path, &Path)], ) -> Result` + `async fn run_container_inner( &self, image: &str, cmd: &[&str], env_vars: &[(&str, &str)], working_dir: &Path, volumes: &[(&Path, &Path)], entrypoint: Option<&str>, ) -> Result` `async fn pull_image_inner(&self, image: &str) -> Result<(), ContainerError>` - `async fn build_image_inner(&self, dockerfile: &Path, tag: &str) -> Result<(), ContainerError>` + `async fn build_image_inner( &self, dockerfile: &Path, tag: &str, context_dir: &Path, ) -> Result<(), ContainerError>` --- ## crates/executor/src/docker_test.rs -**Language:** Rust | **Size:** 6.4 KB | **Lines:** 197 +**Language:** Rust | **Size:** 6.4 KB | **Lines:** 198 **Imports:** - `bollard::Docker` @@ -1293,19 +1397,19 @@ wrkflw/ ## crates/executor/src/engine.rs -**Language:** Rust | **Size:** 96.4 KB | **Lines:** 2608 +**Language:** Rust | **Size:** 196.9 KB | **Lines:** 5511 **Imports:** - `bollard::Docker` - `futures::future` -- `regex` - `serde_yaml::Value` - `std::collections::HashMap` - `std::fs` -- `std::path::Path` +- `std::path::{Path, PathBuf}` - `std::process::Command` - `thiserror::Error` - `ignore::{gitignore::GitignoreBuilder, Match}` +- `crate::action_resolver` - *... and 12 more imports* **Declarations:** @@ -1326,10 +1430,49 @@ wrkflw/ `fn from(err: String) -> Self` -`async fn prepare_action( action: &ActionInfo, runtime: &dyn ContainerRuntime, ) -> Result` +`enum PreparedAction` +> Variants: `NativeDocker`, `Image`, `Composite` + +`async fn prepare_action( action: &ActionInfo, runtime: &dyn ContainerRuntime, ) -> Result` + +`async fn execute_native_docker_step( ctx: &StepExecutionContext<'_>, step_env: &mut HashMap, step_name: String, uses: &str, image: String, entrypoint: Option, args: Vec, ) -> Result` + +`fn sanitize_sub_path(raw: &str) -> Result<(), String>` + +`fn sanitize_dockerfile_rel(raw: &str) -> Result` + +`fn extract_docker_runs_config( definition: Option<&serde_yaml::Value>, ) -> Result<(Option, Vec), String>` + +`async fn shallow_clone( repo_url: &str, git_ref: &str, target_dir: &Path, ) -> Result<(), ExecutionError>` + +`fn is_git_sha(git_ref: &str) -> bool` `fn determine_action_image(repository: &str) -> String` +`struct SetupRuntime` +> Fields: `language: String`, `version: String`, `install_script: String` + +`struct SetupActionDef` +> Fields: `repos: &'static [&'static str]`, `with_key: &'static str`, `default_version: &'static str`, `language: &'static str`, `version_from_ref: bool` + +`const SETUP_ACTIONS: &[SetupActionDef] = &[ SetupActionDef` + +`fn is_safe_version(version: &str) -> bool` + +`fn detect_setup_runtimes(steps: &[Step]) -> Vec` + +`fn get_install_script(language: &str, version: &str) -> String` + +`fn generate_combined_dockerfile(runtimes: &[SetupRuntime], base_image: &str) -> String` + +`fn fnv1a_hash(data: &[u8]) -> u64` + +`fn combined_image_tag(runtimes: &[SetupRuntime], dockerfile: &str) -> String` + +`async fn build_combined_runtime_image( runtimes: &[SetupRuntime], base_image: &str, runtime: &dyn ContainerRuntime, ) -> Result` + +`async fn resolve_runner_image( job: &Job, runtime: &dyn ContainerRuntime, ) -> Result` + `async fn execute_job_batch( jobs: &[String], workflow: &WorkflowDefinition, runtime: &dyn ContainerRuntime, env_context: &HashMap, verbose: bool, secret_manager: Option<&SecretManager>, secret_masker: Option<&SecretMasker>, ) -> Result, ExecutionError>` `struct JobExecutionContext<'a>` @@ -1346,8 +1489,13 @@ wrkflw/ `async fn execute_matrix_job( job_name: &str, job_template: &Job, combination: &MatrixCombination, workflow: &WorkflowDefinition, runtime: &dyn ContainerRuntime, base_env_context: &HashMap, verbose: bool, ) -> Result` +`enum StepOutcome` +> Variants: `Completed`, `Skipped` + +`async fn run_step_with_guards( step: &Step, step_idx: usize, job_env: &HashMap, workflow: &WorkflowDefinition, step_exec_ctx: StepExecutionContext<'_>, ) -> Result` + `struct StepExecutionContext<'a>` -> Fields: `step: &'a workflow::Step`, `step_idx: usize`, `job_env: &'a HashMap`, `working_dir: &'a Path`, `runtime: &'a dyn ContainerRuntime`, `workflow: &'a WorkflowDefinition`, `runner_image: &'a str`, `verbose: bool`, `matrix_combination: &'a Option>`, `secret_manager: Option<&'a SecretManager>`, `secret_masker: Option<&'a SecretMasker>` +> Fields: `step: &'a workflow::Step`, `step_idx: usize`, `job_env: &'a HashMap`, `working_dir: &'a Path`, `runtime: &'a dyn ContainerRuntime`, `workflow: &'a WorkflowDefinition`, `runner_image: &'a str`, `verbose: bool`, `matrix_combination: &'a Option>`, `secret_manager: Option<&'a SecretManager>`, `secret_masker: Option<&'a SecretMasker>`, `container_config: Option<&'a JobContainer>` `async fn execute_step(ctx: StepExecutionContext<'_>) -> Result` @@ -1361,6 +1509,23 @@ wrkflw/ `fn get_runner_image_from_opt(runs_on: &Option>) -> String` +`fn get_effective_runner_image(job: &Job) -> String` + +`struct StepContainerContext` +> Fields: `owned_volume_paths: Vec`, `github_mount: Option` + +**`impl StepContainerContext`** + `fn build_volumes<'a>( &'a self, working_dir: &'a Path, container_workspace: &'a Path, ) -> Vec<(&'a Path, &'a Path)>` + + +`fn prepare_step_container_context( step_env: &mut HashMap, job_env: &HashMap, container_config: Option<&JobContainer>, ) -> StepContainerContext` + +`type VolumePathPair = (PathBuf, PathBuf)` + +`fn prepare_container_mounts( step_env: &mut HashMap, job_env: &HashMap, container_config: Option<&JobContainer>, ) -> (Vec, Option)` + +`fn warn_unsupported_container_fields(container: &JobContainer)` + `async fn execute_reusable_workflow_job( ctx: &JobExecutionContext<'_>, uses: &str, with: Option<&HashMap>, secrets: Option<&serde_yaml::Value>, ) -> Result` `async fn prepare_runner_image( image: &str, runtime: &dyn ContainerRuntime, verbose: bool, ) -> Result<(), ExecutionError>` @@ -1373,6 +1538,8 @@ wrkflw/ `fn evaluate_job_condition( condition: &str, env_context: &HashMap, workflow: &WorkflowDefinition, ) -> bool` +`mod tests` + --- ## crates/executor/src/environment.rs @@ -1410,7 +1577,7 @@ wrkflw/ ## crates/executor/src/lib.rs -**Language:** Rust | **Size:** 360 B | **Lines:** 16 +**Language:** Rust | **Size:** 385 B | **Lines:** 17 **Imports:** - `pub use docker::cleanup_resources` @@ -1424,7 +1591,7 @@ wrkflw/ ## crates/executor/src/podman.rs -**Language:** Rust | **Size:** 33.0 KB | **Lines:** 877 +**Language:** Rust | **Size:** 34.2 KB | **Lines:** 922 **Imports:** - `async_trait::async_trait` @@ -1436,7 +1603,9 @@ wrkflw/ - `tempfile` - `tokio::process::Command` - `wrkflw_logging` -- `wrkflw_runtime::container::{ContainerError, ContainerOutput, ContainerRuntime}` +- `wrkflw_runtime::container::{ + ContainerError, ContainerOutput, ContainerRuntime, LOCAL_IMAGE_PREFIX, +}` - *... and 2 more imports* **Declarations:** @@ -1464,28 +1633,30 @@ wrkflw/ **`impl ContainerRuntime for PodmanRuntime`** - `async fn run_container( &self, image: &str, cmd: &[&str], env_vars: &[(&str, &str)], working_dir: &Path, volumes: &[(&Path, &Path)], ) -> Result` + `async fn run_container( &self, image: &str, cmd: &[&str], env_vars: &[(&str, &str)], working_dir: &Path, volumes: &[(&Path, &Path)], entrypoint: Option<&str>, ) -> Result` `async fn pull_image(&self, image: &str) -> Result<(), ContainerError>` - `async fn build_image(&self, dockerfile: &Path, tag: &str) -> Result<(), ContainerError>` + `async fn build_image( &self, dockerfile: &Path, tag: &str, context_dir: &Path, ) -> Result<(), ContainerError>` `async fn prepare_language_environment( &self, language: &str, version: Option<&str>, additional_packages: Option>, ) -> Result` + `async fn image_exists(&self, tag: &str) -> Result` + **`impl PodmanRuntime`** - `async fn run_container_inner( &self, image: &str, cmd: &[&str], env_vars: &[(&str, &str)], working_dir: &Path, volumes: &[(&Path, &Path)], ) -> Result` + `async fn run_container_inner( &self, image: &str, cmd: &[&str], env_vars: &[(&str, &str)], working_dir: &Path, volumes: &[(&Path, &Path)], entrypoint: Option<&str>, ) -> Result` `async fn pull_image_inner(&self, image: &str) -> Result<(), ContainerError>` - `async fn build_image_inner(&self, dockerfile: &Path, tag: &str) -> Result<(), ContainerError>` + `async fn build_image_inner( &self, dockerfile: &Path, tag: &str, context_dir: &Path, ) -> Result<(), ContainerError>` --- ## crates/executor/src/substitution.rs -**Language:** Rust | **Size:** 3.6 KB | **Lines:** 108 +**Language:** Rust | **Size:** 3.6 KB | **Lines:** 106 **Imports:** - `lazy_static::lazy_static` @@ -1517,7 +1688,7 @@ wrkflw/ ## crates/github/src/lib.rs -**Language:** Rust | **Size:** 10.7 KB | **Lines:** 329 +**Language:** Rust | **Size:** 11.1 KB | **Lines:** 340 **Imports:** - `lazy_static::lazy_static` @@ -1554,7 +1725,7 @@ wrkflw/ ## crates/gitlab/src/lib.rs -**Language:** Rust | **Size:** 8.7 KB | **Lines:** 278 +**Language:** Rust | **Size:** 9.1 KB | **Lines:** 284 **Imports:** - `lazy_static::lazy_static` @@ -1626,7 +1797,7 @@ wrkflw/ ## crates/matrix/src/lib.rs -**Language:** Rust | **Size:** 7.2 KB | **Lines:** 248 +**Language:** Rust | **Size:** 13.6 KB | **Lines:** 422 **Imports:** - `indexmap::IndexMap` @@ -1657,6 +1828,8 @@ wrkflw/ `fn value_to_string(value: &Value) -> String` +`mod tests` + --- ## crates/models/Cargo.toml @@ -1677,7 +1850,7 @@ wrkflw/ ## crates/models/src/lib.rs -**Language:** Rust | **Size:** 11.3 KB | **Lines:** 338 +**Language:** Rust | **Size:** 14.8 KB | **Lines:** 444 **Declarations:** @@ -1730,7 +1903,7 @@ wrkflw/ ## crates/parser/src/gitlab.rs -**Language:** Rust | **Size:** 8.8 KB | **Lines:** 278 +**Language:** Rust | **Size:** 8.3 KB | **Lines:** 264 **Imports:** - `crate::schema::{SchemaType, SchemaValidator}` @@ -1784,7 +1957,7 @@ wrkflw/ ## crates/parser/src/workflow.rs -**Language:** Rust | **Size:** 6.4 KB | **Lines:** 231 +**Language:** Rust | **Size:** 24.3 KB | **Lines:** 796 **Imports:** - `serde::{Deserialize, Deserializer, Serialize}` @@ -1800,6 +1973,28 @@ wrkflw/ `fn deserialize_runs_on<'de, D>(deserializer: D) -> Result>, D::Error> where D: Deserializer<'de>,` +`fn deserialize_container<'de, D>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de>,` + +**`impl serde::Serialize for ContainerCredentials`** + `fn serialize(&self, serializer: S) -> Result where S: serde::Serializer,` + + +**`impl std::fmt::Debug for ContainerCredentials`** + `fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result` + + +**`impl Job`** + `pub fn matrix_config(&self) -> Option<&MatrixConfig>` + + `pub fn fail_fast(&self) -> bool` + + `pub fn max_parallel(&self) -> Option` + + +**`impl Step`** + `pub fn with_run(name: impl Into, run: impl Into) -> Self` + + **`impl WorkflowDefinition`** `pub fn resolve_action(&self, action_ref: &str) -> ActionInfo` @@ -1839,7 +2034,7 @@ wrkflw/ ## crates/runtime/src/container.rs -**Language:** Rust | **Size:** 1.9 KB | **Lines:** 65 +**Language:** Rust | **Size:** 2.7 KB | **Lines:** 89 **Imports:** - `async_trait::async_trait` @@ -1856,7 +2051,7 @@ wrkflw/ ## crates/runtime/src/emulation.rs -**Language:** Rust | **Size:** 33.0 KB | **Lines:** 887 +**Language:** Rust | **Size:** 32.5 KB | **Lines:** 896 **Imports:** - `crate::container::{ContainerError, ContainerOutput, ContainerRuntime}` @@ -1888,11 +2083,13 @@ wrkflw/ **`impl ContainerRuntime for EmulationRuntime`** - `async fn run_container( &self, _image: &str, command: &[&str], env_vars: &[(&str, &str)], working_dir: &Path, _volumes: &[(&Path, &Path)], ) -> Result` + `async fn run_container( &self, _image: &str, command: &[&str], env_vars: &[(&str, &str)], working_dir: &Path, _volumes: &[(&Path, &Path)], _entrypoint: Option<&str>, ) -> Result` `async fn pull_image(&self, image: &str) -> Result<(), ContainerError>` - `async fn build_image(&self, dockerfile: &Path, tag: &str) -> Result<(), ContainerError>` + `async fn build_image( &self, dockerfile: &Path, tag: &str, _context_dir: &Path, ) -> Result<(), ContainerError>` + + `async fn image_exists(&self, _tag: &str) -> Result` `async fn prepare_language_environment( &self, language: &str, version: Option<&str>, _additional_packages: Option>, ) -> Result` @@ -1911,11 +2108,13 @@ wrkflw/ `async fn cleanup_workspaces()` +`mod tests` + --- ## crates/runtime/src/emulation_test.rs -**Language:** Rust | **Size:** 8.6 KB | **Lines:** 240 +**Language:** Rust | **Size:** 8.7 KB | **Lines:** 241 **Imports:** - `std::path::{Path, PathBuf}` @@ -1996,7 +2195,7 @@ wrkflw/ ## crates/runtime/src/secure_emulation.rs -**Language:** Rust | **Size:** 12.7 KB | **Lines:** 339 +**Language:** Rust | **Size:** 13.3 KB | **Lines:** 359 **Imports:** - `crate::container::{ContainerError, ContainerOutput, ContainerRuntime}` @@ -2018,11 +2217,13 @@ wrkflw/ **`impl ContainerRuntime for SecureEmulationRuntime`** - `async fn run_container( &self, image: &str, command: &[&str], env_vars: &[(&str, &str)], working_dir: &Path, _volumes: &[(&Path, &Path)], ) -> Result` + `async fn run_container( &self, image: &str, command: &[&str], env_vars: &[(&str, &str)], working_dir: &Path, _volumes: &[(&Path, &Path)], entrypoint: Option<&str>, ) -> Result` `async fn pull_image(&self, image: &str) -> Result<(), ContainerError>` - `async fn build_image(&self, dockerfile: &Path, tag: &str) -> Result<(), ContainerError>` + `async fn build_image( &self, dockerfile: &Path, tag: &str, _context_dir: &Path, ) -> Result<(), ContainerError>` + + `async fn image_exists(&self, _tag: &str) -> Result` `async fn prepare_language_environment( &self, language: &str, version: Option<&str>, _additional_packages: Option>, ) -> Result` @@ -2199,7 +2400,7 @@ wrkflw/ ## crates/secrets/src/masking.rs -**Language:** Rust | **Size:** 10.4 KB | **Lines:** 348 +**Language:** Rust | **Size:** 10.4 KB | **Lines:** 345 **Imports:** - `regex::Regex` @@ -2409,7 +2610,7 @@ wrkflw/ ## crates/secrets/src/storage.rs -**Language:** Rust | **Size:** 10.9 KB | **Lines:** 351 +**Language:** Rust | **Size:** 13.0 KB | **Lines:** 394 **Imports:** - `crate::{SecretError, SecretResult}` @@ -2426,7 +2627,7 @@ wrkflw/ **`impl EncryptedSecretStore`** `pub fn new() -> SecretResult<(Self, [u8; 32])>` - `pub fn from_data(secrets: HashMap, salt: String, nonce: String) -> Self` + `pub fn from_data(secrets: HashMap, salt: String) -> Self` `pub fn add_secret(&mut self, key: &[u8; 32], name: &str, value: &str) -> SecretResult<()>` @@ -2442,9 +2643,9 @@ wrkflw/ `pub fn clear(&mut self)` - `fn encrypt_value(&self, key: &[u8; 32], value: &str) -> SecretResult` + `fn encrypt_value(key: &[u8; 32], value: &str) -> SecretResult` - `fn decrypt_value(&self, key: &[u8; 32], encrypted: &str) -> SecretResult` + `fn decrypt_value(key: &[u8; 32], encrypted: &str) -> SecretResult` `fn generate_salt() -> [u8; 32]` @@ -2572,7 +2773,7 @@ wrkflw/ ## crates/ui/src/app/mod.rs -**Language:** Rust | **Size:** 21.1 KB | **Lines:** 496 +**Language:** Rust | **Size:** 21.3 KB | **Lines:** 503 **Imports:** - `crate::handlers::workflow::start_next_workflow_execution` @@ -2601,7 +2802,7 @@ wrkflw/ ## crates/ui/src/app/state.rs -**Language:** Rust | **Size:** 40.9 KB | **Lines:** 1069 +**Language:** Rust | **Size:** 40.9 KB | **Lines:** 1065 **Imports:** - `crate::log_processor::{LogProcessingRequest, LogProcessor, ProcessedLogEntry}` @@ -2619,7 +2820,7 @@ wrkflw/ **Declarations:** **`impl App`** - `pub fn new( runtime_type: RuntimeType, tx: mpsc::Sender, preserve_containers_on_failure: bool, ) -> App` + `pub fn new( runtime_type: RuntimeType, tx: mpsc::Sender, preserve_containers_on_failure: bool, show_action_messages: bool, ) -> App` `pub fn toggle_selected(&mut self)` @@ -2807,7 +3008,7 @@ wrkflw/ ## crates/ui/src/handlers/workflow.rs -**Language:** Rust | **Size:** 22.1 KB | **Lines:** 569 +**Language:** Rust | **Size:** 22.3 KB | **Lines:** 575 **Imports:** - `crate::app::App` @@ -2839,7 +3040,7 @@ wrkflw/ ## crates/ui/src/log_processor.rs -**Language:** Rust | **Size:** 10.4 KB | **Lines:** 305 +**Language:** Rust | **Size:** 11.2 KB | **Lines:** 330 **Imports:** - `crate::models::LogFilterLevel` @@ -2880,6 +3081,8 @@ wrkflw/ `fn default() -> Self` +`mod tests` + --- ## crates/ui/src/models/mod.rs @@ -3025,7 +3228,7 @@ wrkflw/ ## crates/ui/src/views/status_bar.rs -**Language:** Rust | **Size:** 7.3 KB | **Lines:** 212 +**Language:** Rust | **Size:** 7.3 KB | **Lines:** 211 **Imports:** - `crate::app::App` @@ -3066,7 +3269,7 @@ wrkflw/ ## crates/ui/src/views/workflows_tab.rs -**Language:** Rust | **Size:** 4.5 KB | **Lines:** 131 +**Language:** Rust | **Size:** 4.7 KB | **Lines:** 137 **Imports:** - `crate::app::App` @@ -3143,7 +3346,7 @@ wrkflw/ ## crates/validators/src/gitlab.rs -**Language:** Rust | **Size:** 7.7 KB | **Lines:** 234 +**Language:** Rust | **Size:** 7.8 KB | **Lines:** 235 **Imports:** - `std::collections::HashMap` @@ -3168,15 +3371,22 @@ wrkflw/ ## crates/validators/src/jobs.rs -**Language:** Rust | **Size:** 4.7 KB | **Lines:** 102 +**Language:** Rust | **Size:** 12.0 KB | **Lines:** 354 **Imports:** +- `std::collections::{HashMap, HashSet}` - `crate::{validate_matrix, validate_steps}` - `serde_yaml::Value` - `wrkflw_models::ValidationResult` **Declarations:** +`fn detect_cyclic_needs(jobs_map: &serde_yaml::Mapping, result: &mut ValidationResult)` + +`fn dfs_detect_cycle( node: &str, graph: &HashMap>, visited: &mut HashSet, in_stack: &mut HashSet, rec_stack: &mut Vec, reported_cycles: &mut HashSet>, result: &mut ValidationResult, )` + +`mod tests` + --- ## crates/validators/src/lib.rs @@ -3227,7 +3437,7 @@ wrkflw/ ## crates/validators/src/steps.rs -**Language:** Rust | **Size:** 2.2 KB | **Lines:** 57 +**Language:** Rust | **Size:** 3.5 KB | **Lines:** 107 **Imports:** - `crate::validate_action_reference` @@ -3237,11 +3447,13 @@ wrkflw/ **Declarations:** +`mod tests` + --- ## crates/validators/src/triggers.rs -**Language:** Rust | **Size:** 3.0 KB | **Lines:** 96 +**Language:** Rust | **Size:** 8.3 KB | **Lines:** 263 **Imports:** - `serde_yaml::Value` @@ -3251,6 +3463,12 @@ wrkflw/ `fn validate_cron_syntax(cron: &str, result: &mut ValidationResult)` +`fn is_valid_cron_field(field: &str, min: u32, max: u32) -> bool` + +`fn is_valid_cron_atom(atom: &str, min: u32, max: u32) -> bool` + +`mod tests` + --- ## crates/wrkflw/Cargo.toml @@ -3266,7 +3484,7 @@ wrkflw/ ## crates/wrkflw/README.md -**Language:** Markdown | **Size:** 3.6 KB | **Lines:** 112 +**Language:** Markdown | **Size:** 3.6 KB | **Lines:** 113 **Declarations:** @@ -3293,7 +3511,7 @@ wrkflw/ ## crates/wrkflw/src/main.rs -**Language:** Rust | **Size:** 26.7 KB | **Lines:** 708 +**Language:** Rust | **Size:** 29.2 KB | **Lines:** 783 **Imports:** - `bollard::Docker` @@ -3331,7 +3549,28 @@ wrkflw/ `fn validate_gitlab_pipeline(path: &Path, verbose: bool) -> bool` -`fn list_workflows_and_pipelines(verbose: bool)` +`fn list_workflows_and_pipelines(verbose: bool, show_jobs: bool)` + +--- + +## crates/wrkflw/tests/target_job_test.rs + +**Language:** Rust | **Size:** 3.6 KB | **Lines:** 141 + +**Imports:** +- `std::fs` +- `tempfile::tempdir` +- `wrkflw_lib::executor::engine::{execute_workflow, ExecutionConfig, RuntimeType}` + +**Declarations:** + +`fn write_file(path: &std::path::Path, content: &str)` + +`async fn test_target_job_runs_only_specified_job()` + +`async fn test_target_job_not_found_returns_error()` + +`async fn test_target_job_with_no_deps_runs_alone()` --- @@ -3351,6 +3590,20 @@ wrkflw/ --- +## hello.cpp + +**Language:** C++ | **Size:** 127 B | **Lines:** 6 + +**Declarations:** + +--- + +## hello.rs + +**Language:** Rust | **Size:** 70 B | **Lines:** 4 + +--- + ## publish_crates.sh **Language:** Shell | **Size:** 5.1 KB | **Lines:** 179 @@ -3381,6 +3634,15 @@ wrkflw/ --- +## test.py + +**Language:** Python | **Size:** 53 B | **Lines:** 2 + +**Imports:** +- `import sys` + +--- + ## tests/README.md **Language:** Markdown | **Size:** 1.8 KB | **Lines:** 61 @@ -3516,7 +3778,7 @@ wrkflw/ ## tests/reusable_workflow_execution_test.rs -**Language:** Rust | **Size:** 3.0 KB | **Lines:** 120 +**Language:** Rust | **Size:** 3.1 KB | **Lines:** 122 **Imports:** - `std::fs` @@ -3676,6 +3938,14 @@ wrkflw/ --- +## tests/workflows/multi-runtime-test.yml + +**Language:** YAML | **Size:** 688 B | **Lines:** 27 + +**Declarations:** + +--- + ## tests/workflows/node-test.yml **Language:** YAML | **Size:** 639 B | **Lines:** 31 diff --git a/crates/executor/Cargo.toml b/crates/executor/Cargo.toml index d687af8..2432bd1 100644 --- a/crates/executor/Cargo.toml +++ b/crates/executor/Cargo.toml @@ -33,7 +33,9 @@ num_cpus.workspace = true once_cell.workspace = true regex.workspace = true reqwest.workspace = true +glob.workspace = true serde.workspace = true +sha2.workspace = true serde_json.workspace = true serde_yaml.workspace = true shlex.workspace = true diff --git a/crates/executor/src/dependency.rs b/crates/executor/src/dependency.rs index ea562df..72e4dbe 100644 --- a/crates/executor/src/dependency.rs +++ b/crates/executor/src/dependency.rs @@ -236,6 +236,8 @@ mod tests { uses: None, with: None, secrets: None, + timeout_minutes: None, + defaults: None, } } diff --git a/crates/executor/src/engine.rs b/crates/executor/src/engine.rs index 281aaf5..e09bf64 100644 --- a/crates/executor/src/engine.rs +++ b/crates/executor/src/engine.rs @@ -1695,6 +1695,9 @@ async fn execute_job(ctx: JobExecutionContext<'_>) -> Result) -> Result { - step_results.push(result); - } - StepOutcome::Completed { result, abort_job } => { - // Add step output to logs only in verbose mode or if there's an error - if ctx.verbose || result.status == StepStatus::Failure { - job_logs.push_str(&format!( - "\n=== Output from step '{}' ===\n{}\n=== End output ===\n\n", - result.name, result.output - )); - } else { - job_logs.push_str(&format!( - "Step '{}' completed with status: {:?}\n", - result.name, result.status - )); + match outcome { + StepOutcome::Skipped(result) => { + step_results.push(result); } + StepOutcome::Completed { result, abort_job } => { + // Add step output to logs only in verbose mode or if there's an error + if ctx.verbose || result.status == StepStatus::Failure { + job_logs.push_str(&format!( + "\n=== Output from step '{}' ===\n{}\n=== End output ===\n\n", + result.name, result.output + )); + } else { + job_logs.push_str(&format!( + "Step '{}' completed with status: {:?}\n", + result.name, result.status + )); + } - step_results.push(result); + step_results.push(result); - if abort_job { - job_success = false; - break; + if abort_job { + job_success = false; + break; + } } } } + + Ok::<(), ExecutionError>(()) + }; + + match tokio::time::timeout(job_timeout, step_loop).await { + Ok(Ok(())) => {} + Ok(Err(e)) => return Err(e), + Err(_) => { + wrkflw_logging::error(&format!( + "Job '{}' exceeded timeout of {} minutes", + ctx.job_name, timeout_mins + )); + return Ok(JobResult { + name: ctx.job_name.to_string(), + status: JobStatus::Failure, + steps: step_results, + logs: format!("{}\nJob timed out after {} minutes", job_logs, timeout_mins), + }); + } } Ok(JobResult { @@ -1935,6 +1965,8 @@ async fn execute_matrix_job( secret_manager: None, secret_masker: None, container_config: job_template.container.as_ref(), + workflow_defaults: workflow.defaults.as_ref(), + job_defaults: job_template.defaults.as_ref(), }, ) .await?; @@ -2019,7 +2051,29 @@ async fn run_step_with_guards( } } - match execute_step(step_exec_ctx).await { + // Wrap step execution with optional timeout; sanitize to avoid panic on negative/NaN + let step_result = if let Some(minutes) = step.timeout_minutes { + let safe_mins = sanitize_timeout_minutes(Some(minutes), 360.0); + let dur = std::time::Duration::from_secs_f64(safe_mins * 60.0); + match tokio::time::timeout(dur, execute_step(step_exec_ctx)).await { + Ok(result) => result, + Err(_) => { + wrkflw_logging::error(&format!( + " Step '{}' exceeded timeout of {} minutes", + step_name, minutes + )); + Ok(StepResult { + name: step_name.clone(), + status: StepStatus::Failure, + output: format!("Step timed out after {} minutes", minutes), + }) + } + } + } else { + execute_step(step_exec_ctx).await + }; + + match step_result { Ok(result) => { let abort_job = if result.status == StepStatus::Failure { if step.continue_on_error == Some(true) { @@ -2064,6 +2118,18 @@ async fn run_step_with_guards( } } +/// Sanitize a timeout-minutes value, returning a safe positive finite number. +/// Falls back to `default` for `None`, `NaN`, `Infinity`, zero, or negative values. +/// Clamps to a maximum of 8640 minutes (6 days). +fn sanitize_timeout_minutes(raw: Option, default: f64) -> f64 { + let mins = raw.unwrap_or(default); + if mins.is_finite() && mins > 0.0 { + mins.min(360.0 * 24.0) + } else { + default + } +} + // Before the execute_step function, add this struct struct StepExecutionContext<'a> { step: &'a workflow::Step, @@ -2080,6 +2146,8 @@ struct StepExecutionContext<'a> { #[allow(dead_code)] // Planned for future implementation secret_masker: Option<&'a SecretMasker>, container_config: Option<&'a JobContainer>, + workflow_defaults: Option<&'a workflow::Defaults>, + job_defaults: Option<&'a workflow::Defaults>, } async fn execute_step(ctx: StepExecutionContext<'_>) -> Result { @@ -2651,15 +2719,105 @@ async fn execute_step(ctx: StepExecutionContext<'_>) -> Result r, + Err(e) => { + return Ok(StepResult { + name: step_name, + status: StepStatus::Failure, + output: format!("Expression substitution failed: {}", e), + }); + } + }; + // Check if this is a cargo command let is_cargo_cmd = resolved_run.trim().starts_with("cargo"); - // For complex shell commands, use bash to execute them properly - // This handles quotes, pipes, redirections, and command substitutions correctly - let cmd_parts = vec!["bash", "-c", &resolved_run]; + // Resolve effective shell: step > job defaults > workflow defaults > "bash" + let effective_shell = ctx + .step + .shell + .as_deref() + .or_else(|| { + ctx.job_defaults + .and_then(|d| d.run.as_ref()?.shell.as_deref()) + }) + .or_else(|| { + ctx.workflow_defaults + .and_then(|d| d.run.as_ref()?.shell.as_deref()) + }) + .unwrap_or("bash"); + + let cmd_parts = match effective_shell { + "bash" => vec![ + "bash", + "--noprofile", + "--norc", + "-e", + "-o", + "pipefail", + "-c", + &resolved_run, + ], + "sh" => vec!["sh", "-e", "-c", &resolved_run], + "python" => vec!["python", "-c", &resolved_run], + "pwsh" | "powershell" => vec!["pwsh", "-command", &resolved_run], + other => { + wrkflw_logging::warning(&format!( + " Unrecognized shell '{}', falling back to '{} -c'", + other, other + )); + vec![other, "-c", &resolved_run] + } + }; + + // Resolve effective working directory: step > job defaults > workflow defaults + let effective_wd = ctx + .step + .working_directory + .as_deref() + .or_else(|| { + ctx.job_defaults + .and_then(|d| d.run.as_ref()?.working_directory.as_deref()) + }) + .or_else(|| { + ctx.workflow_defaults + .and_then(|d| d.run.as_ref()?.working_directory.as_deref()) + }); // Define the standard workspace path inside the container let container_workspace = Path::new("/github/workspace"); + let final_workspace = if let Some(wd) = effective_wd { + let joined = container_workspace.join(wd); + // Canonicalize logically to catch ".." traversal and absolute path replacement + let mut normalized = std::path::PathBuf::new(); + for component in joined.components() { + match component { + std::path::Component::ParentDir => { + normalized.pop(); + } + c => normalized.push(c.as_os_str()), + } + } + if !normalized.starts_with(container_workspace) { + return Ok(StepResult { + name: step_name, + status: StepStatus::Failure, + output: format!( + "Invalid working-directory '{}': must be within workspace", + wd + ), + }); + } + normalized + } else { + container_workspace.to_path_buf() + }; let mount_ctx = prepare_step_container_context(&mut step_env, ctx.job_env, ctx.container_config); @@ -2676,7 +2834,7 @@ async fn execute_step(ctx: StepExecutionContext<'_>) -> Result Step { + Step { + name: Some("run-step".to_string()), + uses: None, + run: Some(run.to_string()), + with: None, + env: HashMap::new(), + continue_on_error: None, + if_condition: None, + id: None, + working_directory: None, + shell: None, + timeout_minutes: None, + } + } + + #[tokio::test] + async fn bash_shell_uses_errexit_and_pipefail() { + let runtime = MockContainerRuntime::default(); + let workflow = minimal_workflow(); + let job_env = HashMap::new(); + let working_dir = std::env::current_dir().unwrap(); + + let step = make_run_step("echo hello"); + + let ctx = StepExecutionContext { + step: &step, + step_idx: 0, + job_env: &job_env, + working_dir: &working_dir, + runtime: &runtime, + workflow: &workflow, + runner_image: "ubuntu:latest", + verbose: false, + matrix_combination: &None, + secret_manager: None, + secret_masker: None, + container_config: None, + workflow_defaults: None, + job_defaults: None, + }; + + let result = execute_step(ctx).await.unwrap(); + assert_eq!(result.status, StepStatus::Success); + + let calls = runtime.run_calls.lock().unwrap(); + assert_eq!(calls.len(), 1); + let cmd = &calls[0].cmd; + // Should be: bash --noprofile --norc -e -o pipefail -c