Fix massive memory allocation issue

Not exactly sure what was going on, other than cloning of the FileMatch
with its results, since it is recursive I believe this blew up the
stack/heap with unnecessary allocations. By using an iterator and filter
this seemed to greatly reduce memory use and improve speed.
This commit is contained in:
Cameron Cordes
2020-05-19 18:19:18 -04:00
parent e1aa96dd78
commit 0dc8383d2e

View File

@@ -19,34 +19,40 @@ fn main() {
} }
} }
fn traverse_dir(config: &config::Config, dir: &Path, operation: fn(&config::Config, &Path) -> io::Result<FileMatch>) fn traverse_dir(
-> io::Result<Vec<FileMatch>> { config: &config::Config,
dir: &Path,
operation: fn(&config::Config, &Path) -> io::Result<FileMatch>,
) -> io::Result<Vec<FileMatch>> {
let mut matches: Vec<FileMatch> = Vec::new();
for entry in fs::read_dir(dir)? {
let entry = entry?;
let file_type = entry.file_type()?;
let mut matches: Vec<FileMatch> = Vec::new(); if file_type.is_dir() {
for entry in fs::read_dir(dir)? { let mut results = traverse_dir(&config, &entry.path(), operation)?;
let entry = entry?; let has_results = results
let file_type = entry.file_type()?; .iter()
.filter({ |mat| mat.matches.len() > 0 })
.count()
> 0;
if file_type.is_dir() { if has_results {
let results = traverse_dir(&config, &entry.path(), operation)?; matches.append(&mut results);
for mat in results.as_slice() { }
if mat.matches.len() > 0 { } else {
matches.append(&mut results.clone()); let result = operation(&config, &entry.path());
} match result {
} Result::Ok(result) if result.matches.len() > 0 => matches.push(result),
} else { _ => (),
let result = operation(&config, &entry.path());
match result {
Result::Ok(result) if result.matches.len() > 0 => matches.push(result),
_ => ()
}
} }
} }
Ok(matches)
} }
#[derive(Debug, Clone)] Ok(matches)
}
#[derive(Debug)]
struct FileMatch { struct FileMatch {
file_name: String, file_name: String,
matches: Vec<Match> matches: Vec<Match>