Merge pull request #51 from bahdotsh/feature/gitignore-support

feat: Add .gitignore support for file copying

Cargo.lock

@@ -293,6 +293,16 @@ dependencies = [
  "serde_with",
 ]
 
+[[package]]
+name = "bstr"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4"
+dependencies = [
+ "memchr",
+ "serde",
+]
+
 [[package]]
 name = "bumpalo"
 version = "3.17.0"
@@ -891,6 +901,19 @@ version = "0.31.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
 
+[[package]]
+name = "globset"
+version = "0.4.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "54a1028dfc5f5df5da8a56a73e6c153c9a9708ec57232470703592a3f18e49f5"
+dependencies = [
+ "aho-corasick",
+ "bstr",
+ "log",
+ "regex-automata",
+ "regex-syntax",
+]
+
 [[package]]
 name = "h2"
 version = "0.3.26"
@@ -1227,6 +1250,22 @@ dependencies = [
  "icu_properties",
 ]
 
+[[package]]
+name = "ignore"
+version = "0.4.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d89fd380afde86567dfba715db065673989d6253f42b88179abd3eae47bda4b"
+dependencies = [
+ "crossbeam-deque",
+ "globset",
+ "log",
+ "memchr",
+ "regex-automata",
+ "same-file",
+ "walkdir",
+ "winapi-util",
+]
+
 [[package]]
 name = "indexmap"
 version = "1.9.3"
@@ -3061,6 +3100,7 @@ dependencies = [
  "dirs",
  "futures",
  "futures-util",
+ "ignore",
  "lazy_static",
  "num_cpus",
  "once_cell",
@@ -3163,6 +3203,7 @@ version = "0.7.0"
 dependencies = [
  "async-trait",
  "futures",
+ "ignore",
  "once_cell",
  "regex",
  "serde",

crates/executor/Cargo.toml

@@ -27,6 +27,7 @@ chrono.workspace = true
 dirs.workspace = true
 futures.workspace = true
 futures-util.workspace = true
+ignore = "0.4"
 lazy_static.workspace = true
 num_cpus.workspace = true
 once_cell.workspace = true

crates/executor/src/engine.rs

@@ -9,6 +9,8 @@ use std::path::Path;
 use std::process::Command;
 use thiserror::Error;
 
+use ignore::{gitignore::GitignoreBuilder, Match};
+
 use crate::dependency;
 use crate::docker;
 use crate::environment;
@@ -1785,16 +1787,77 @@ async fn execute_step(ctx: StepExecutionContext<'_>) -> Result<StepResult, Execu
     Ok(step_result)
 }
 
+/// Create a gitignore matcher for the given directory
+fn create_gitignore_matcher(
+    dir: &Path,
+) -> Result<Option<ignore::gitignore::Gitignore>, ExecutionError> {
+    let mut builder = GitignoreBuilder::new(dir);
+
+    // Try to add .gitignore file if it exists
+    let gitignore_path = dir.join(".gitignore");
+    if gitignore_path.exists() {
+        builder.add(&gitignore_path);
+    }
+
+    // Add some common ignore patterns as fallback
+    builder.add_line(None, "target/").map_err(|e| {
+        ExecutionError::Execution(format!("Failed to add default ignore pattern: {}", e))
+    })?;
+    builder.add_line(None, ".git/").map_err(|e| {
+        ExecutionError::Execution(format!("Failed to add default ignore pattern: {}", e))
+    })?;
+
+    match builder.build() {
+        Ok(gitignore) => Ok(Some(gitignore)),
+        Err(e) => {
+            wrkflw_logging::warning(&format!("Failed to build gitignore matcher: {}", e));
+            Ok(None)
+        }
+    }
+}
+
 fn copy_directory_contents(from: &Path, to: &Path) -> Result<(), ExecutionError> {
+    copy_directory_contents_with_gitignore(from, to, None)
+}
+
+fn copy_directory_contents_with_gitignore(
+    from: &Path,
+    to: &Path,
+    gitignore: Option<&ignore::gitignore::Gitignore>,
+) -> Result<(), ExecutionError> {
+    // If no gitignore provided, try to create one for the root directory
+    let root_gitignore;
+    let gitignore = if gitignore.is_none() {
+        root_gitignore = create_gitignore_matcher(from)?;
+        root_gitignore.as_ref()
+    } else {
+        gitignore
+    };
+
     for entry in std::fs::read_dir(from)
         .map_err(|e| ExecutionError::Execution(format!("Failed to read directory: {}", e)))?
     {
         let entry =
             entry.map_err(|e| ExecutionError::Execution(format!("Failed to read entry: {}", e)))?;
         let path = entry.path();
+
+        // Check if the file should be ignored according to .gitignore
+        if let Some(gitignore) = gitignore {
+            let relative_path = path.strip_prefix(from).unwrap_or(&path);
+            match gitignore.matched(relative_path, path.is_dir()) {
+                Match::Ignore(_) => {
+                    wrkflw_logging::debug(&format!("Skipping ignored file/directory: {path:?}"));
+                    continue;
+                }
+                Match::Whitelist(_) | Match::None => {
+                    // File is not ignored or explicitly whitelisted
+                }
+            }
+        }
+
         wrkflw_logging::debug(&format!("Copying entry: {path:?} -> {to:?}"));
 
-        // Skip hidden files/dirs and target directory for efficiency
+        // Additional basic filtering for hidden files (but allow .gitignore and .github)
         let file_name = match path.file_name() {
             Some(name) => name.to_string_lossy(),
             None => {
@@ -1804,7 +1867,13 @@ fn copy_directory_contents(from: &Path, to: &Path) -> Result<(), ExecutionError>
                 )));
             }
         };
-        if file_name.starts_with(".") || file_name == "target" {
+
+        // Skip most hidden files but allow important ones
+        if file_name.starts_with(".")
+            && file_name != ".gitignore"
+            && file_name != ".github"
+            && !file_name.starts_with(".env")
+        {
             continue;
         }
 
@@ -1822,8 +1891,8 @@ fn copy_directory_contents(from: &Path, to: &Path) -> Result<(), ExecutionError>
             std::fs::create_dir_all(&dest_path)
                 .map_err(|e| ExecutionError::Execution(format!("Failed to create dir: {}", e)))?;
 
-            // Recursively copy subdirectories
-            copy_directory_contents(&path, &dest_path)?;
+            // Recursively copy subdirectories with the same gitignore
+            copy_directory_contents_with_gitignore(&path, &dest_path, gitignore)?;
         } else {
             std::fs::copy(&path, &dest_path)
                 .map_err(|e| ExecutionError::Execution(format!("Failed to copy file: {}", e)))?;

crates/runtime/Cargo.toml

@@ -23,6 +23,7 @@ serde_yaml.workspace = true
 tempfile = "3.9"
 tokio.workspace = true
 futures = "0.3"
+ignore = "0.4"
 wrkflw-utils = { path = "../utils", version = "0.7.0" }
 which = "4.4"
 regex = "1.10"

crates/runtime/src/emulation.rs

@@ -10,6 +10,8 @@ use tempfile::TempDir;
 use which;
 use wrkflw_logging;
 
+use ignore::{gitignore::GitignoreBuilder, Match};
+
 // Global collection of resources to clean up
 static EMULATION_WORKSPACES: Lazy<Mutex<Vec<PathBuf>>> = Lazy::new(|| Mutex::new(Vec::new()));
 static EMULATION_PROCESSES: Lazy<Mutex<Vec<u32>>> = Lazy::new(|| Mutex::new(Vec::new()));
@@ -490,14 +492,75 @@ impl ContainerRuntime for EmulationRuntime {
 }
 
 #[allow(dead_code)]
+/// Create a gitignore matcher for the given directory
+fn create_gitignore_matcher(
+    dir: &Path,
+) -> Result<Option<ignore::gitignore::Gitignore>, std::io::Error> {
+    let mut builder = GitignoreBuilder::new(dir);
+
+    // Try to add .gitignore file if it exists
+    let gitignore_path = dir.join(".gitignore");
+    if gitignore_path.exists() {
+        builder.add(&gitignore_path);
+    }
+
+    // Add some common ignore patterns as fallback
+    if let Err(e) = builder.add_line(None, "target/") {
+        wrkflw_logging::warning(&format!("Failed to add default ignore pattern: {}", e));
+    }
+    if let Err(e) = builder.add_line(None, ".git/") {
+        wrkflw_logging::warning(&format!("Failed to add default ignore pattern: {}", e));
+    }
+
+    match builder.build() {
+        Ok(gitignore) => Ok(Some(gitignore)),
+        Err(e) => {
+            wrkflw_logging::warning(&format!("Failed to build gitignore matcher: {}", e));
+            Ok(None)
+        }
+    }
+}
+
 fn copy_directory_contents(source: &Path, dest: &Path) -> std::io::Result<()> {
+    copy_directory_contents_with_gitignore(source, dest, None)
+}
+
+fn copy_directory_contents_with_gitignore(
+    source: &Path,
+    dest: &Path,
+    gitignore: Option<&ignore::gitignore::Gitignore>,
+) -> std::io::Result<()> {
     // Create the destination directory if it doesn't exist
     fs::create_dir_all(dest)?;
 
+    // If no gitignore provided, try to create one for the root directory
+    let root_gitignore;
+    let gitignore = if gitignore.is_none() {
+        root_gitignore = create_gitignore_matcher(source)?;
+        root_gitignore.as_ref()
+    } else {
+        gitignore
+    };
+
     // Iterate through all entries in the source directory
     for entry in fs::read_dir(source)? {
         let entry = entry?;
         let path = entry.path();
+
+        // Check if the file should be ignored according to .gitignore
+        if let Some(gitignore) = gitignore {
+            let relative_path = path.strip_prefix(source).unwrap_or(&path);
+            match gitignore.matched(relative_path, path.is_dir()) {
+                Match::Ignore(_) => {
+                    wrkflw_logging::debug(&format!("Skipping ignored file/directory: {path:?}"));
+                    continue;
+                }
+                Match::Whitelist(_) | Match::None => {
+                    // File is not ignored or explicitly whitelisted
+                }
+            }
+        }
+
         let file_name = match path.file_name() {
             Some(name) => name,
             None => {
@@ -507,23 +570,19 @@ fn copy_directory_contents(source: &Path, dest: &Path) -> std::io::Result<()> {
         };
         let dest_path = dest.join(file_name);
 
-        // Skip hidden files (except .gitignore and .github might be useful)
+        // Skip most hidden files but allow important ones
         let file_name_str = file_name.to_string_lossy();
         if file_name_str.starts_with(".")
             && file_name_str != ".gitignore"
             && file_name_str != ".github"
+            && !file_name_str.starts_with(".env")
         {
             continue;
         }
 
-        // Skip target directory for Rust projects
-        if file_name_str == "target" {
-            continue;
-        }
-
         if path.is_dir() {
-            // Recursively copy subdirectories
-            copy_directory_contents(&path, &dest_path)?;
+            // Recursively copy subdirectories with the same gitignore
+            copy_directory_contents_with_gitignore(&path, &dest_path, gitignore)?;
         } else {
             // Copy files
             fs::copy(&path, &dest_path)?;