Fix massive memory allocation issue

Not exactly sure what was going on, other than cloning of the FileMatch
with its results, since it is recursive I believe this blew up the
stack/heap with unnecessary allocations. By using an iterator and filter
this seemed to greatly reduce memory use and improve speed.
This commit is contained in:
Cameron Cordes
2020-05-19 18:19:18 -04:00
parent e1aa96dd78
commit 0dc8383d2e

View File

@@ -19,26 +19,32 @@ fn main() {
} }
} }
fn traverse_dir(config: &config::Config, dir: &Path, operation: fn(&config::Config, &Path) -> io::Result<FileMatch>) fn traverse_dir(
-> io::Result<Vec<FileMatch>> { config: &config::Config,
dir: &Path,
operation: fn(&config::Config, &Path) -> io::Result<FileMatch>,
) -> io::Result<Vec<FileMatch>> {
let mut matches: Vec<FileMatch> = Vec::new(); let mut matches: Vec<FileMatch> = Vec::new();
for entry in fs::read_dir(dir)? { for entry in fs::read_dir(dir)? {
let entry = entry?; let entry = entry?;
let file_type = entry.file_type()?; let file_type = entry.file_type()?;
if file_type.is_dir() { if file_type.is_dir() {
let results = traverse_dir(&config, &entry.path(), operation)?; let mut results = traverse_dir(&config, &entry.path(), operation)?;
for mat in results.as_slice() { let has_results = results
if mat.matches.len() > 0 { .iter()
matches.append(&mut results.clone()); .filter({ |mat| mat.matches.len() > 0 })
} .count()
> 0;
if has_results {
matches.append(&mut results);
} }
} else { } else {
let result = operation(&config, &entry.path()); let result = operation(&config, &entry.path());
match result { match result {
Result::Ok(result) if result.matches.len() > 0 => matches.push(result), Result::Ok(result) if result.matches.len() > 0 => matches.push(result),
_ => () _ => (),
} }
} }
} }
@@ -46,7 +52,7 @@ fn traverse_dir(config: &config::Config, dir: &Path, operation: fn(&config::Conf
Ok(matches) Ok(matches)
} }
#[derive(Debug, Clone)] #[derive(Debug)]
struct FileMatch { struct FileMatch {
file_name: String, file_name: String,
matches: Vec<Match> matches: Vec<Match>