Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ cli = [
"strip-ansi-escapes",
"target-triple",
"tempfile",
"thread_local",
"time-humanize",
"tokio",
"twox-hash",
Expand All @@ -61,6 +62,7 @@ signal-hook = { version = "0.4.3", optional = true }
strip-ansi-escapes = { version = "0.2.1", optional = true }
target-triple = { version = "1.0.0", optional = true }
tempfile = { version = "3.27.0", optional = true }
thread_local = { version = "1.1.9", optional = true }
time-humanize = { version = "0.1.3", optional = true }
tokio = { version = "1.50.0", features = [
"process",
Expand Down
72 changes: 45 additions & 27 deletions src/bin/cargo-ziggy/coverage.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
use crate::{Common, Cover, util::Context, util::Utf8PathBuf};
use crate::{
Common, Cover,
util::Utf8PathBuf,
util::{Context, progress_bar},
};
use anyhow::{Context as _, Result, bail};
use indicatif::{ProgressBar, ProgressStyle};
use rayon::iter::{IntoParallelIterator, ParallelIterator};
use std::{
env, fmt, fs,
Expand All @@ -10,10 +13,6 @@ use std::{
str::FromStr,
};

thread_local! {
static BUF: std::cell::RefCell<Vec<u8>> = std::cell::RefCell::new(Vec::with_capacity(8 * 1024));
}

impl Cover {
pub fn generate_coverage(&self, common: &Common) -> Result<(), anyhow::Error> {
let cx = Context::new(common, self.target.clone())?;
Expand Down Expand Up @@ -75,37 +74,32 @@ impl Cover {
}

eprintln!(" Generating raw profiles");
let pb = ProgressBar::new(coverage_corpus.len() as u64);
pb.set_style(
ProgressStyle::with_template(
" [{elapsed_precise}] [{wide_bar}] {pos}/{len} ({eta})",
)
.unwrap()
.progress_chars("#--"),
);
let pb = progress_bar(coverage_corpus.len() as u64);
let log_dir = self.ziggy_output.join(format!("{}/logs", &cx.bin_target));
fs::create_dir_all(&log_dir).context("output dir for logs")?;
let log_file = std::sync::Mutex::new(
std::fs::File::create(log_dir.join("coverage.log")).context("logfile")?,
);
let tls = thread_local::ThreadLocal::new();
#[expect(clippy::significant_drop_tightening)]
coverage_corpus.into_par_iter().for_each(|file| {
#[expect(clippy::significant_drop_tightening)]
BUF.with_borrow_mut(|buf| {
buf.clear();
let _ = cfg.profile(file.as_path(), buf);
let mut log_file = log_file.lock().unwrap();
let _ = log_file.write_all(buf);
// use `lock_file` mutex to avoid contention on progress bar
pb.inc(1);
});
let mut buf = tls
.get_or(|| std::cell::RefCell::new(Vec::with_capacity(8 * 1024)))
.borrow_mut();
buf.clear();
let _ = cfg.profile(file.as_path(), &mut buf);
let mut log_file = log_file.lock().unwrap();
let _ = log_file.write_all(&buf);
// use `lock_file` mutex to avoid contention on progress bar
pb.inc(1);
});
pb.finish();

// We generate the code coverage report
eprintln!("\n Generating coverage report");

let out_dir = PathBuf::from(coverage_dir);
let profdata = out_dir.join("coverage.profraw");
let profdata = out_dir.join("coverage.prodata");
fs::create_dir_all(&out_dir).context("output dir for coverage")?;
cfg.merge_profraw(&profdata, self.jobs)
.context("llvm_profdata: merging profraw files")?;
Expand Down Expand Up @@ -251,6 +245,28 @@ impl Cfg {
})
}

pub fn llvm_profdata(&self) -> process::Command {
process::Command::new(&self.llvm_profdata)
}

pub fn llvm_cov(&self) -> process::Command {
process::Command::new(&self.llvm_cov)
}

pub fn runner_path(&self) -> &Path {
self.runner.as_std_path()
}

pub fn profile_simple(&self, seed: &Path, profraw_pattern: &Path) {
let _ = process::Command::new(&self.runner)
.arg(seed)
.stdin(process::Stdio::null())
.stdout(process::Stdio::null())
.stderr(process::Stdio::null())
.env("LLVM_PROFILE_FILE", profraw_pattern)
.status();
}

fn profile(&self, seed: &Path, output: &mut Vec<u8>) -> Result<(), anyhow::Error> {
// redirect stderr and stdout into buffer (via pipe)
let (mut rx, tx) = std::io::pipe()?;
Expand Down Expand Up @@ -293,9 +309,11 @@ impl Cfg {
}
}

let status = std::process::Command::new(&self.llvm_profdata)
let status = self
.llvm_profdata()
.arg("merge")
.arg("-sparse")
.arg("--sparse")
.arg("--failure-mode=warn")
.arg("-f")
.arg(
list_file
Expand All @@ -322,7 +340,7 @@ impl Cfg {
) -> Result<()> {
use ReportType::{Html, Json, LCov, Text};

let mut cov_cmd = std::process::Command::new(&self.llvm_cov);
let mut cov_cmd = self.llvm_cov();
match format {
Html => cov_cmd.args(["show", "-format=html"]),
Text => cov_cmd.args(["show", "-format=text"]),
Expand Down
4 changes: 2 additions & 2 deletions src/bin/cargo-ziggy/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -394,9 +394,9 @@ pub struct Stability {
)]
ziggy_output: PathBuf,

/// Number of repeated runs over the corpus (default: 3)
/// Number of repeated runs over the corpus
#[clap(short = 'n', long, default_value_t = 3)]
runs: u32,
runs: usize,

/// Number of concurrent jobs
#[clap(short, long, value_name = "NUM")]
Expand Down
106 changes: 49 additions & 57 deletions src/bin/cargo-ziggy/stability.rs
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
use crate::{Common, Cover, Stability, util::Context};
use crate::{
Common, Cover, Stability,
util::{Context, progress_bar},
};
use anyhow::{Context as _, Result, bail};
use console::style;
use indicatif::{ProgressBar, ProgressStyle};
use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
use std::{
collections::{HashMap, HashSet},
env, fs,
io::Write,
path::{Path, PathBuf},
process,
};

/// Maps (filename, line_number) to execution count.
Expand All @@ -25,18 +27,17 @@ impl Stability {
}

// Check that llvm tools are available
check_tool("llvm-profdata")?;
check_tool("llvm-cov")?;
let cfg = crate::coverage::Cfg::new(
cx.target_dir
.join(format!("coverage/debug/{}", cx.bin_target)),
"dummy.profraw".into(),
)?;

// Build coverage-instrumented binary
eprintln!(" {} coverage binary", style("Building").red().bold());
Cover::build_runner(common)?;
eprintln!(" {} coverage binary", style("Finished").cyan().bold());

let runner = cx
.target_dir
.join(format!("coverage/debug/{}", cx.bin_target));

// Collect corpus files
let corpus = collect_corpus(&self.input, &self.ziggy_output, &cx)?;
if corpus.is_empty() {
Expand All @@ -54,7 +55,7 @@ impl Stability {
}

// Run the corpus N times, collecting LCOV data each time
let mut run_data: Vec<LineCounts> = Vec::with_capacity(self.runs as usize);
let mut run_data: Vec<LineCounts> = Vec::with_capacity(self.runs);

for run_idx in 0..self.runs {
eprintln!(
Expand All @@ -69,39 +70,32 @@ impl Stability {
let profraw_pattern = run_dir.join("cov-%p-%m.profraw");

// Run all inputs in parallel
let pb = ProgressBar::new(corpus.len() as u64);
pb.set_style(
ProgressStyle::with_template(
" [{elapsed_precise}] [{wide_bar}] {pos}/{len} ({eta})",
)
.unwrap()
.progress_chars("#>-"),
);
let pb = progress_bar(corpus.len() as u64);

corpus.par_iter().for_each(|input| {
let _ = process::Command::new(runner.as_str())
.arg(input)
.stdin(process::Stdio::null())
.stdout(process::Stdio::null())
.stderr(process::Stdio::null())
.env("LLVM_PROFILE_FILE", &profraw_pattern)
.status();
cfg.profile_simple(input, &profraw_pattern);
pb.inc(1);
});
pb.finish();

// Collect profraw files, skipping empty ones (from processes that were
// killed before the LLVM runtime initialized)
let profraw_files: Vec<PathBuf> = fs::read_dir(&run_dir)?
.flatten()
.map(|e| e.path())
.filter(|p| {
p.extension().is_some_and(|ext| ext == "profraw")
&& p.metadata().is_ok_and(|m| m.len() > 0)
})
.collect();

if profraw_files.is_empty() {
let mut list_file = std::io::BufWriter::new(
tempfile::NamedTempFile::new()
.context("creating temp file for llvm-profdata input list")?,
);
let mut valid_profile = false;
for path in fs::read_dir(&run_dir)?.flatten().map(|e| e.path()) {
if path.extension().is_some_and(|ext| ext == "profraw")
&& path.metadata().is_ok_and(|m| m.len() > 0)
{
writeln!(list_file, "{}", path.display())
.context("writing profraw path to input list")?;
valid_profile = true;
}
}

if !valid_profile {
eprintln!(
" {} No coverage data for run {} — all inputs may have crashed",
style("!!").yellow().bold(),
Expand All @@ -114,13 +108,21 @@ impl Stability {
// Use --failure-mode=warn to skip corrupt profiles (from crashing inputs)
// instead of aborting the entire merge.
let profdata = run_dir.join("merged.profdata");
let merge = process::Command::new("llvm-profdata")
let merge = cfg
.llvm_profdata()
.arg("merge")
.arg("--sparse")
.arg("--failure-mode=warn")
.args(&profraw_files)
.arg("-f")
.arg(
list_file
.into_inner()
.context("flushing llvm-profdata input list")?
.path(),
)
.arg("-o")
.arg(&profdata)
.args(self.jobs.map(|n| format!("-j={n}")))
.output()
.context("Failed to run llvm-profdata merge")?;

Expand All @@ -134,11 +136,15 @@ impl Stability {
}

// Export as LCOV
let export = process::Command::new("llvm-cov")
let export = cfg
.llvm_cov()
.arg("export")
.arg("--format=lcov")
.arg(format!("--instr-profile={}", profdata.display()))
.arg(runner.as_str())
.arg("--instr-profile")
.arg(profdata)
.args(["--skip-branches", "--skip-functions"])
.args(self.jobs.map(|n| format!("-j={n}")))
.arg(cfg.runner_path())
.output()
.context("Failed to run llvm-cov export")?;

Expand Down Expand Up @@ -171,7 +177,7 @@ impl Stability {
.map(|s| fs::canonicalize(s).unwrap_or_else(|_| s.clone()));

// Analyze and report
let successful_runs = run_data.len() as u32;
let successful_runs = run_data.len();
let report = analyze_runs(&run_data, source_filter.as_deref());
let cwd = env::current_dir().ok();
print_report(
Expand All @@ -186,16 +192,6 @@ impl Stability {
}
}

fn check_tool(name: &str) -> Result<()> {
process::Command::new(name)
.arg("--version")
.stdout(process::Stdio::null())
.stderr(process::Stdio::null())
.status()
.with_context(|| format!("{name} not found — please install LLVM tools"))?;
Ok(())
}

fn collect_corpus(input: &Path, ziggy_output: &Path, cx: &Context) -> Result<Vec<PathBuf>> {
let input_path = PathBuf::from(
input
Expand Down Expand Up @@ -270,11 +266,7 @@ fn analyze_runs(runs: &[LineCounts], source_filter: Option<&Path>) -> StabilityR
let executed_keys: Vec<_> = all_keys
.into_iter()
.filter(|key| runs.iter().any(|r| r.get(key).copied().unwrap_or(0) > 0))
.filter(|key| {
source_filter
.map(|filter| Path::new(&key.0).starts_with(filter))
.unwrap_or(true)
})
.filter(|key| source_filter.is_none_or(|filter| Path::new(&key.0).starts_with(filter)))
.collect();

let mut unstable = Vec::new();
Expand Down Expand Up @@ -356,7 +348,7 @@ fn print_report(
report: &StabilityReport,
target: &str,
corpus_size: usize,
runs: u32,
runs: usize,
cwd: Option<&Path>,
) {
let stability_pct = if report.total_lines > 0 {
Expand Down
Loading