From 8005cbb7eee33ffc540d1193fcfaa233472293e0 Mon Sep 17 00:00:00 2001 From: bahdotsh Date: Thu, 21 Aug 2025 15:27:00 +0530 Subject: [PATCH] feat: Add .gitignore support for file copying - Add ignore crate dependency to executor and runtime crates - Implement gitignore-aware file copying in engine.rs and emulation.rs - Support for .gitignore patterns, whitelist rules, and default ignore patterns - Maintain backward compatibility with projects without .gitignore files - Add proper error handling and debug logging for ignored files This ensures that files marked in .gitignore are not copied to containers or emulation workspaces, improving performance and security. --- crates/executor/Cargo.toml | 1 + crates/executor/src/engine.rs | 77 +++++++++++++++++++++++++++++++-- crates/runtime/Cargo.toml | 1 + crates/runtime/src/emulation.rs | 75 ++++++++++++++++++++++++++++---- 4 files changed, 142 insertions(+), 12 deletions(-) diff --git a/crates/executor/Cargo.toml b/crates/executor/Cargo.toml index bc1c38c..4759048 100644 --- a/crates/executor/Cargo.toml +++ b/crates/executor/Cargo.toml @@ -27,6 +27,7 @@ chrono.workspace = true dirs.workspace = true futures.workspace = true futures-util.workspace = true +ignore = "0.4" lazy_static.workspace = true num_cpus.workspace = true once_cell.workspace = true diff --git a/crates/executor/src/engine.rs b/crates/executor/src/engine.rs index b122080..10a9594 100644 --- a/crates/executor/src/engine.rs +++ b/crates/executor/src/engine.rs @@ -9,6 +9,8 @@ use std::path::Path; use std::process::Command; use thiserror::Error; +use ignore::{gitignore::GitignoreBuilder, Match}; + use crate::dependency; use crate::docker; use crate::environment; @@ -1785,16 +1787,77 @@ async fn execute_step(ctx: StepExecutionContext<'_>) -> Result Result, ExecutionError> { + let mut builder = GitignoreBuilder::new(dir); + + // Try to add .gitignore file if it exists + let gitignore_path = dir.join(".gitignore"); + if gitignore_path.exists() { + builder.add(&gitignore_path); + } + + // Add some common ignore patterns as fallback + builder.add_line(None, "target/").map_err(|e| { + ExecutionError::Execution(format!("Failed to add default ignore pattern: {}", e)) + })?; + builder.add_line(None, ".git/").map_err(|e| { + ExecutionError::Execution(format!("Failed to add default ignore pattern: {}", e)) + })?; + + match builder.build() { + Ok(gitignore) => Ok(Some(gitignore)), + Err(e) => { + wrkflw_logging::warning(&format!("Failed to build gitignore matcher: {}", e)); + Ok(None) + } + } +} + fn copy_directory_contents(from: &Path, to: &Path) -> Result<(), ExecutionError> { + copy_directory_contents_with_gitignore(from, to, None) +} + +fn copy_directory_contents_with_gitignore( + from: &Path, + to: &Path, + gitignore: Option<&ignore::gitignore::Gitignore>, +) -> Result<(), ExecutionError> { + // If no gitignore provided, try to create one for the root directory + let root_gitignore; + let gitignore = if gitignore.is_none() { + root_gitignore = create_gitignore_matcher(from)?; + root_gitignore.as_ref() + } else { + gitignore + }; + for entry in std::fs::read_dir(from) .map_err(|e| ExecutionError::Execution(format!("Failed to read directory: {}", e)))? { let entry = entry.map_err(|e| ExecutionError::Execution(format!("Failed to read entry: {}", e)))?; let path = entry.path(); + + // Check if the file should be ignored according to .gitignore + if let Some(gitignore) = gitignore { + let relative_path = path.strip_prefix(from).unwrap_or(&path); + match gitignore.matched(relative_path, path.is_dir()) { + Match::Ignore(_) => { + wrkflw_logging::debug(&format!("Skipping ignored file/directory: {path:?}")); + continue; + } + Match::Whitelist(_) | Match::None => { + // File is not ignored or explicitly whitelisted + } + } + } + wrkflw_logging::debug(&format!("Copying entry: {path:?} -> {to:?}")); - // Skip hidden files/dirs and target directory for efficiency + // Additional basic filtering for hidden files (but allow .gitignore and .github) let file_name = match path.file_name() { Some(name) => name.to_string_lossy(), None => { @@ -1804,7 +1867,13 @@ fn copy_directory_contents(from: &Path, to: &Path) -> Result<(), ExecutionError> ))); } }; - if file_name.starts_with(".") || file_name == "target" { + + // Skip most hidden files but allow important ones + if file_name.starts_with(".") + && file_name != ".gitignore" + && file_name != ".github" + && !file_name.starts_with(".env") + { continue; } @@ -1822,8 +1891,8 @@ fn copy_directory_contents(from: &Path, to: &Path) -> Result<(), ExecutionError> std::fs::create_dir_all(&dest_path) .map_err(|e| ExecutionError::Execution(format!("Failed to create dir: {}", e)))?; - // Recursively copy subdirectories - copy_directory_contents(&path, &dest_path)?; + // Recursively copy subdirectories with the same gitignore + copy_directory_contents_with_gitignore(&path, &dest_path, gitignore)?; } else { std::fs::copy(&path, &dest_path) .map_err(|e| ExecutionError::Execution(format!("Failed to copy file: {}", e)))?; diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index d5f51a8..cbb5ac0 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -23,6 +23,7 @@ serde_yaml.workspace = true tempfile = "3.9" tokio.workspace = true futures = "0.3" +ignore = "0.4" wrkflw-utils = { path = "../utils", version = "0.7.0" } which = "4.4" regex = "1.10" diff --git a/crates/runtime/src/emulation.rs b/crates/runtime/src/emulation.rs index c54e954..04c9549 100644 --- a/crates/runtime/src/emulation.rs +++ b/crates/runtime/src/emulation.rs @@ -10,6 +10,8 @@ use tempfile::TempDir; use which; use wrkflw_logging; +use ignore::{gitignore::GitignoreBuilder, Match}; + // Global collection of resources to clean up static EMULATION_WORKSPACES: Lazy>> = Lazy::new(|| Mutex::new(Vec::new())); static EMULATION_PROCESSES: Lazy>> = Lazy::new(|| Mutex::new(Vec::new())); @@ -490,14 +492,75 @@ impl ContainerRuntime for EmulationRuntime { } #[allow(dead_code)] +/// Create a gitignore matcher for the given directory +fn create_gitignore_matcher( + dir: &Path, +) -> Result, std::io::Error> { + let mut builder = GitignoreBuilder::new(dir); + + // Try to add .gitignore file if it exists + let gitignore_path = dir.join(".gitignore"); + if gitignore_path.exists() { + builder.add(&gitignore_path); + } + + // Add some common ignore patterns as fallback + if let Err(e) = builder.add_line(None, "target/") { + wrkflw_logging::warning(&format!("Failed to add default ignore pattern: {}", e)); + } + if let Err(e) = builder.add_line(None, ".git/") { + wrkflw_logging::warning(&format!("Failed to add default ignore pattern: {}", e)); + } + + match builder.build() { + Ok(gitignore) => Ok(Some(gitignore)), + Err(e) => { + wrkflw_logging::warning(&format!("Failed to build gitignore matcher: {}", e)); + Ok(None) + } + } +} + fn copy_directory_contents(source: &Path, dest: &Path) -> std::io::Result<()> { + copy_directory_contents_with_gitignore(source, dest, None) +} + +fn copy_directory_contents_with_gitignore( + source: &Path, + dest: &Path, + gitignore: Option<&ignore::gitignore::Gitignore>, +) -> std::io::Result<()> { // Create the destination directory if it doesn't exist fs::create_dir_all(dest)?; + // If no gitignore provided, try to create one for the root directory + let root_gitignore; + let gitignore = if gitignore.is_none() { + root_gitignore = create_gitignore_matcher(source)?; + root_gitignore.as_ref() + } else { + gitignore + }; + // Iterate through all entries in the source directory for entry in fs::read_dir(source)? { let entry = entry?; let path = entry.path(); + + // Check if the file should be ignored according to .gitignore + if let Some(gitignore) = gitignore { + let relative_path = path.strip_prefix(source).unwrap_or(&path); + match gitignore.matched(relative_path, path.is_dir()) { + Match::Ignore(_) => { + wrkflw_logging::debug(&format!("Skipping ignored file/directory: {path:?}")); + continue; + } + Match::Whitelist(_) | Match::None => { + // File is not ignored or explicitly whitelisted + } + } + } + let file_name = match path.file_name() { Some(name) => name, None => { @@ -507,23 +570,19 @@ fn copy_directory_contents(source: &Path, dest: &Path) -> std::io::Result<()> { }; let dest_path = dest.join(file_name); - // Skip hidden files (except .gitignore and .github might be useful) + // Skip most hidden files but allow important ones let file_name_str = file_name.to_string_lossy(); if file_name_str.starts_with(".") && file_name_str != ".gitignore" && file_name_str != ".github" + && !file_name_str.starts_with(".env") { continue; } - // Skip target directory for Rust projects - if file_name_str == "target" { - continue; - } - if path.is_dir() { - // Recursively copy subdirectories - copy_directory_contents(&path, &dest_path)?; + // Recursively copy subdirectories with the same gitignore + copy_directory_contents_with_gitignore(&path, &dest_path, gitignore)?; } else { // Copy files fs::copy(&path, &dest_path)?;