1use std::collections::HashMap;
4use std::fmt::Display;
5use std::fs::{File, OpenOptions};
6use std::io::{BufWriter, Write};
7use std::path::{Path, PathBuf};
8use std::time::{Duration, Instant, SystemTime};
9
10use dyn_clone::DynClone;
11use libafl::monitors::stats::*;
12use libafl::monitors::tui::TuiMonitor;
13use libafl::prelude::*;
14use libafl_bolts::prelude::{current_time, ClientId};
15use serde::Serialize;
16use serde_json::Serializer as JSONSerializer;
17
18use crate::fuzzer::libafl_setup::MAP_FEEDBACK_NAME;
19use crate::fuzzer::stats_stage::{RuntimeStats, DUPLICATES, STATS};
20
21trait ClonableMonitor: Monitor + DynClone {}
22impl ClonableMonitor for TuiMonitor {}
23impl ClonableMonitor for NopMonitor {}
24dyn_clone::clone_trait_object!(ClonableMonitor);
25
26pub struct StatsMonitor {
28 monitor: Box<dyn ClonableMonitor>,
29 handlers: Vec<Box<dyn EventHandler>>,
30 stats_interval: Duration,
31 last_client_write: HashMap<ClientId, Instant>,
32 last_global_write: Instant,
33}
34
35impl Clone for StatsMonitor {
36 fn clone(&self) -> Self {
37 Self {
38 monitor: self.monitor.clone(),
39 handlers: self.handlers.clone(),
40 stats_interval: self.stats_interval,
41 last_client_write: HashMap::new(),
42 last_global_write: Instant::now() - self.stats_interval,
43 }
44 }
45}
46
47impl StatsMonitor {
48 pub fn with_tui_output(stats_file: PathBuf, stats_interval: Duration) -> Self {
49 let monitor = Box::new(
50 TuiMonitor::builder()
51 .title(String::from("tlspuffin [press q to exit]"))
52 .enhanced_graphics(true)
53 .build(),
54 );
55 let handlers: Vec<Box<dyn EventHandler>> =
56 vec![Box::new(JSONEventHandler::new(stats_file))];
57
58 Self::new(monitor, handlers, stats_interval)
59 }
60
61 pub fn with_raw_output(stats_file: PathBuf, stats_interval: Duration) -> Self {
62 let monitor = Box::new(NopMonitor::new());
63 let handlers: Vec<Box<dyn EventHandler>> = vec![
64 Box::new(|_, msg: &str, stats: &Statistics| log::info!("[{}] {}", msg, stats)),
65 Box::new(JSONEventHandler::new(stats_file)),
66 ];
67
68 Self::new(monitor, handlers, stats_interval)
69 }
70
71 fn new(
72 monitor: Box<dyn ClonableMonitor>,
73 handlers: Vec<Box<dyn EventHandler>>,
74 stats_interval: Duration,
75 ) -> Self {
76 Self {
77 monitor,
78 handlers,
79 stats_interval,
80 last_client_write: HashMap::new(),
81 last_global_write: Instant::now() - stats_interval,
82 }
83 }
84
85 fn client(
86 &mut self,
87 client_stats_manager: &mut ClientStatsManager,
88 id: ClientId,
89 ) -> Result<Statistics, Error> {
90 client_stats_manager.update_client_stats_for(id, |client| {
91 #[cfg(feature = "introspection")]
92 let introspect_feature = {
93 let intro_stats = &client.introspection_stats;
94 let elapsed_cycles = intro_stats.elapsed_cycles();
95 let elapsed = if elapsed_cycles == 0 {
96 1.0
97 } else {
98 elapsed_cycles as f32
99 };
100
101 let mut introspect_features = IntrospectFeatures::new();
103
104 for (_, features) in intro_stats.used_stages() {
105 for (feature_index, feature) in features.iter().enumerate() {
106 let feature_percent = *feature as f32 / elapsed;
108
109 if feature_percent == 0.0 {
111 continue;
112 }
113
114 let feature: PerfFeature = feature_index.into();
116
117 introspect_features.record(&feature, feature_percent);
119 }
120
121 }
123
124 IntrospectStatistics {
125 scheduler: intro_stats.scheduler_cycles() as f32 / elapsed,
126 manager: intro_stats.manager_cycles() as f32 / elapsed,
127 elapsed_cycles,
128 introspect_features,
129 }
130 };
131
132 let cur_time = current_time();
133 let exec_sec = client.execs_per_sec(cur_time);
134 let total_execs = client.executions();
135
136 let trace = TraceStatistics::new(client);
137 let mut error_counter = ErrorStatistics::new(total_execs);
138
139 error_counter.count(client);
140
141 let corpus_size = client.corpus_size();
142 let objective_size = client.objective_size();
143
144 let coverage =
145 client
146 .user_stats()
147 .get(MAP_FEEDBACK_NAME)
148 .and_then(|s| match s.value() {
149 UserStatsValue::Ratio(a, b) => {
150 Some(CoverageStatistics { hit: *a, max: *b })
151 }
152 _ => None,
153 });
154
155 let duplicates = get_number(client, DUPLICATES.name);
156
157 Statistics::Client(ClientStatistics {
158 id: id.0,
159 time: SystemTime::now(),
160 trace,
161 errors: error_counter,
162 #[cfg(feature = "introspection")]
163 intro: introspect_feature,
164 coverage,
165 corpus_size,
166 objective_size,
167 total_execs,
168 exec_per_sec: exec_sec as u64,
169 duplicates,
170 })
171 })
172 }
173
174 fn global(&mut self, client_stats_manager: &mut ClientStatsManager) -> Statistics {
175 let duplicates: u64 = client_stats_manager
176 .client_stats()
177 .values()
178 .map(|client| get_number(client, DUPLICATES.name))
179 .sum();
180
181 let global_stats = client_stats_manager.global_stats();
182
183 Statistics::Global(GlobalStatistics {
184 time: SystemTime::now(),
185
186 clients: global_stats.client_stats_count as u32,
187 corpus_size: global_stats.corpus_size,
188 objective_size: global_stats.objective_size,
189 duplicates,
190 total_execs: global_stats.total_execs,
191 exec_per_sec: global_stats.execs_per_sec as u64,
192 })
193 }
194
195 fn dispatch(&mut self, sender: ClientId, msg: &str, stats: &Statistics) {
196 self.handlers
197 .iter_mut()
198 .for_each(|h| h.process(sender, msg, stats));
199 }
200}
201
202impl Monitor for StatsMonitor {
203 fn display(
204 &mut self,
205 client_stats_manager: &mut ClientStatsManager,
206 _event_msg: &str,
207 sender_id: ClientId,
208 ) -> Result<(), Error> {
209 client_stats_manager.client_stats_insert(sender_id)?;
210
211 let now = Instant::now();
215 let event_msg = "Monitor";
216
217 let last_client = self
218 .last_client_write
219 .get(&sender_id)
220 .copied()
221 .unwrap_or(now - self.stats_interval);
222
223 if now.duration_since(last_client) >= self.stats_interval {
224 self.last_client_write.insert(sender_id, now);
225 let client_stats = self.client(client_stats_manager, sender_id)?;
226 self.dispatch(sender_id, event_msg, &client_stats);
227 }
228
229 if now.duration_since(self.last_global_write) >= self.stats_interval {
232 self.last_global_write = now;
233 let global_stats = self.global(client_stats_manager);
234 self.dispatch(sender_id, event_msg, &global_stats);
235 }
236
237 self.monitor
238 .display(client_stats_manager, event_msg, sender_id)?;
239 Ok(())
240 }
241}
242
243#[derive(Serialize)]
244#[serde(tag = "type")]
245enum Statistics {
246 #[serde(rename = "client")]
247 Client(ClientStatistics),
248 #[serde(rename = "global")]
249 Global(GlobalStatistics),
250}
251
252impl Display for Statistics {
253 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
254 match self {
255 Self::Client(client_stats) => {
256 write!(
257 f,
258 "(CLIENT) id: {}, corpus: {}, obj: {}, execs: {}, exec/sec: {}",
259 client_stats.id,
260 client_stats.corpus_size,
261 client_stats.objective_size,
262 client_stats.total_execs,
263 client_stats.exec_per_sec
264 )?;
265
266 if let Some(CoverageStatistics { hit, max }) = client_stats.coverage {
267 match max {
268 0 => write!(f, ", edges: {hit}/{max}"),
269 _ => write!(f, ", edges: {hit}/{max} ({}%)", hit * 100 / max),
270 }
271 } else {
272 Ok(())
273 }
274 }
275
276 Self::Global(global_stats) => {
277 write!(
278 f,
279 "(GLOBAL) clients: {}, corpus: {}, obj: {}, execs: {}, exec/sec: {}",
280 global_stats.clients,
281 global_stats.corpus_size,
282 global_stats.objective_size,
283 global_stats.total_execs,
284 global_stats.exec_per_sec,
285 )
286 }
287 }
288 }
289}
290
291#[derive(Serialize)]
292struct GlobalStatistics {
293 time: SystemTime,
294
295 clients: u32,
296
297 corpus_size: u64,
298 objective_size: u64,
299
300 total_execs: u64,
301 exec_per_sec: u64,
302 duplicates: u64,
303}
304
305#[derive(Serialize)]
306struct ClientStatistics {
307 id: u32,
309 time: SystemTime,
310 errors: ErrorStatistics,
311 trace: TraceStatistics,
312 #[cfg(feature = "introspection")]
313 intro: IntrospectStatistics,
314 coverage: Option<CoverageStatistics>,
315
316 corpus_size: u64,
317 objective_size: u64,
318 total_execs: u64,
319 exec_per_sec: u64,
320 duplicates: u64,
321}
322
323#[derive(Serialize)]
324struct CoverageStatistics {
325 hit: u64,
326 max: u64,
327}
328
329#[cfg(feature = "introspection")]
330#[derive(Serialize)]
331struct IntrospectStatistics {
332 scheduler: f32,
333 manager: f32,
334 elapsed_cycles: u64,
335 introspect_features: IntrospectFeatures,
336}
337
338#[cfg(feature = "introspection")]
339#[derive(Serialize)]
340struct IntrospectFeatures {
341 get_input_from_corpus: f32,
342 mutate: f32,
343 mutate_post_exec: f32,
344 target_execution: f32,
345 pre_exec: f32,
346 post_exec: f32,
347 pre_exec_observers: f32,
348 post_exec_observers: f32,
349 get_feedback_interesting_all: f32,
350 get_objectives_interesting_all: f32,
351}
352
353#[derive(Serialize)]
390struct ErrorStatistics {
391 #[serde(skip)]
392 #[allow(dead_code)]
393 total_execs: u64,
394 eval_fn_crypto_error: u64,
396 eval_fn_malformed_error: u64,
397 eval_fn_unknown_error: u64,
398 eval_fn_codec_error: u64,
399 eval_term_error: u64,
400 eval_termbug_error: u64,
401 eval_codec_error: u64,
402 all_term_eval: u64,
403 all_term_eval_success: u64,
404
405 fn_error: u64,
407 term_error: u64,
408 term_bug_error: u64,
409 codec_error: u64,
410 put_error: u64,
411 io_error: u64,
412 agent_error: u64,
413 stream_error: u64,
414 extraction_error: u64,
415 all_exec: u64,
416 all_exec_success: u64,
417 all_exec_agent_success: u64,
418 harness_exec: u64,
419 harness_exec_success: u64,
420 harness_exec_agent_success: u64,
421 bit_exec: u64,
422 bit_exec_success: u64,
423 mm_exec: u64,
424 mm_exec_success: u64,
425
426 corpus_exec: u64,
427 corpus_exec_minimal: u64,
428 duplicates: u64,
429}
430
431#[derive(Serialize)]
432struct TraceStatistics {
433 min_trace_length: Option<u64>,
434 max_trace_length: Option<u64>,
435 mean_trace_length: Option<u64>,
436
437 max_nb_payload: Option<u64>,
438 mean_nb_payload: Option<u64>,
439
440 min_payload_size: Option<u64>,
441 max_payload_size: Option<u64>,
442 mean_payload_size: Option<u64>,
443
444 min_term_size: Option<u64>,
445 max_term_size: Option<u64>,
446 mean_term_size: Option<u64>,
447}
448
449#[cfg(feature = "introspection")]
450impl IntrospectFeatures {
451 pub fn new() -> Self {
452 Self {
453 get_input_from_corpus: 0.0,
454 mutate: 0.0,
455 mutate_post_exec: 0.0,
456 target_execution: 0.0,
457 pre_exec: 0.0,
458 post_exec: 0.0,
459 pre_exec_observers: 0.0,
460 post_exec_observers: 0.0,
461 get_feedback_interesting_all: 0.0,
462 get_objectives_interesting_all: 0.0,
463 }
464 }
465
466 fn make_mean(value: &mut f32, new_value: f32) {
467 if *value == 0.0 {
468 *value = new_value
469 } else {
470 *value = (*value + new_value) / 2_f32
471 }
472 }
473
474 pub fn record(&mut self, feature: &PerfFeature, relative_cycles: f32) {
475 match feature {
476 PerfFeature::GetInputFromCorpus => {
477 Self::make_mean(&mut self.get_input_from_corpus, relative_cycles)
478 }
479 PerfFeature::Mutate => Self::make_mean(&mut self.mutate, relative_cycles),
480 PerfFeature::MutatePostExec => {
481 Self::make_mean(&mut self.mutate_post_exec, relative_cycles)
482 }
483 PerfFeature::TargetExecution => {
484 Self::make_mean(&mut self.target_execution, relative_cycles)
485 }
486 PerfFeature::PreExec => Self::make_mean(&mut self.pre_exec, relative_cycles),
487 PerfFeature::PostExec => Self::make_mean(&mut self.post_exec, relative_cycles),
488 PerfFeature::PreExecObservers => {
489 Self::make_mean(&mut self.pre_exec_observers, relative_cycles)
490 }
491 PerfFeature::PostExecObservers => {
492 Self::make_mean(&mut self.post_exec_observers, relative_cycles)
493 }
494 PerfFeature::GetFeedbackInterestingAll => {
495 Self::make_mean(&mut self.get_feedback_interesting_all, relative_cycles)
496 }
497 PerfFeature::GetObjectivesInterestingAll => {
498 Self::make_mean(&mut self.get_objectives_interesting_all, relative_cycles)
499 }
500 PerfFeature::Count => {}
501 }
502 }
503}
504
505#[cfg(feature = "introspection")]
506impl Default for IntrospectFeatures {
507 fn default() -> Self {
508 Self::new()
509 }
510}
511
512impl ErrorStatistics {
513 pub const fn new(total_execs: u64) -> Self {
514 Self {
515 total_execs,
516 eval_fn_crypto_error: 0,
517 eval_fn_malformed_error: 0,
518 eval_fn_unknown_error: 0,
519 eval_fn_codec_error: 0,
520 eval_term_error: 0,
521 eval_termbug_error: 0,
522 fn_error: 0,
523 term_error: 0,
524 term_bug_error: 0,
525 all_term_eval: 0,
526 all_term_eval_success: 0,
527 all_exec: 0,
528 all_exec_success: 0,
529 all_exec_agent_success: 0,
530 harness_exec: 0,
531 harness_exec_success: 0,
532 harness_exec_agent_success: 0,
533 bit_exec: 0,
534 bit_exec_success: 0,
535 mm_exec: 0,
536 mm_exec_success: 0,
537 codec_error: 0,
538 put_error: 0,
539 io_error: 0,
540 agent_error: 0,
541 stream_error: 0,
542 extraction_error: 0,
543 corpus_exec: 0,
544 corpus_exec_minimal: 0,
545 duplicates: 0,
546 eval_codec_error: 0,
547 }
548 }
549
550 pub fn count(&mut self, client_stats: &ClientStats) {
551 for stat_definition in &STATS {
552 match stat_definition {
553 RuntimeStats::EvalFnCryptoError(c) => {
554 self.eval_fn_crypto_error += get_number(client_stats, c.name)
555 }
556 RuntimeStats::EvalFnMalformedError(c) => {
557 self.eval_fn_malformed_error += get_number(client_stats, c.name)
558 }
559 RuntimeStats::EvalFnUnknownError(c) => {
560 self.eval_fn_unknown_error += get_number(client_stats, c.name)
561 }
562 RuntimeStats::EvalFnCodecError(c) => {
563 self.eval_fn_codec_error += get_number(client_stats, c.name)
564 }
565 RuntimeStats::EvalTermError(c) => {
566 self.eval_term_error += get_number(client_stats, c.name)
567 }
568 RuntimeStats::EvalTermBugError(c) => {
569 self.eval_termbug_error += get_number(client_stats, c.name)
570 }
571 RuntimeStats::EvalCodecError(c) => {
572 self.eval_codec_error += get_number(client_stats, c.name)
573 }
574 RuntimeStats::AllFnError(c) => self.fn_error += get_number(client_stats, c.name),
575 RuntimeStats::AllTermError(c) => {
576 self.term_error += get_number(client_stats, c.name)
577 }
578 RuntimeStats::AllTermBugError(c) => {
579 self.term_bug_error += get_number(client_stats, c.name)
580 }
581 RuntimeStats::AllCodecError(c) => {
582 self.codec_error += get_number(client_stats, c.name)
583 }
584 RuntimeStats::AllPutError(c) => self.put_error += get_number(client_stats, c.name),
585 RuntimeStats::AllIOError(c) => self.io_error += get_number(client_stats, c.name),
586 RuntimeStats::AllAgentError(c) => {
587 self.agent_error += get_number(client_stats, c.name)
588 }
589 RuntimeStats::AllStreamError(c) => {
590 self.stream_error += get_number(client_stats, c.name)
591 }
592 RuntimeStats::AllExtractionError(c) => {
593 self.extraction_error += get_number(client_stats, c.name)
594 }
595 RuntimeStats::AllTermEval(c) => {
596 self.all_term_eval += get_number(client_stats, c.name)
597 }
598 RuntimeStats::AllTermEvalSuccess(c) => {
599 self.all_term_eval_success += get_number(client_stats, c.name)
600 }
601 RuntimeStats::AllExec(c) => self.all_exec += get_number(client_stats, c.name),
602 RuntimeStats::AllExecSuccess(c) => {
603 self.all_exec_success += get_number(client_stats, c.name)
604 }
605 RuntimeStats::AllExecAgentSuccess(c) => {
606 self.all_exec_agent_success += get_number(client_stats, c.name)
607 }
608 RuntimeStats::HarnessExec(c) => {
609 self.harness_exec += get_number(client_stats, c.name)
610 }
611 RuntimeStats::HarnessExecSuccess(c) => {
612 self.harness_exec_success += get_number(client_stats, c.name)
613 }
614 RuntimeStats::HarnessExecAgentSuccess(c) => {
615 self.harness_exec_agent_success += get_number(client_stats, c.name)
616 }
617 RuntimeStats::BitExec(c) => self.bit_exec += get_number(client_stats, c.name),
618 RuntimeStats::BitExecSuccess(c) => {
619 self.bit_exec_success += get_number(client_stats, c.name)
620 }
621 RuntimeStats::MMExec(c) => self.mm_exec += get_number(client_stats, c.name),
622 RuntimeStats::MMNExecSuccess(c) => {
623 self.mm_exec_success += get_number(client_stats, c.name)
624 }
625 RuntimeStats::CorpusExec(c) => self.corpus_exec += get_number(client_stats, c.name),
626 RuntimeStats::CorpusExecMinimal(c) => {
627 self.corpus_exec_minimal += get_number(client_stats, c.name)
628 }
629 RuntimeStats::Duplicates(c) => self.duplicates += get_number(client_stats, c.name),
630 RuntimeStats::TraceLength(_) => {}
632 RuntimeStats::NbPayload(_) => {}
633 RuntimeStats::PayloadLength(_) => {}
634 RuntimeStats::TermSize(_) => {}
635 }
636 }
637 }
638}
639
640fn get_number(user_stats: &ClientStats, name: &str) -> u64 {
641 user_stats
642 .user_stats()
643 .get(name)
644 .and_then(|s| match s.value() {
645 UserStatsValue::Number(n) => Some(*n),
646 _ => None,
647 })
648 .unwrap_or(0u64)
649}
650
651impl TraceStatistics {
652 pub fn new(user_stats: &ClientStats) -> Self {
653 let mut trace_stats = Self {
654 min_trace_length: None,
655 max_trace_length: None,
656 mean_trace_length: None,
657 max_nb_payload: None,
658 mean_nb_payload: None,
659 min_payload_size: None,
660 max_payload_size: None,
661 mean_payload_size: None,
662 min_term_size: None,
663 max_term_size: None,
664 mean_term_size: None,
665 };
666
667 for stat_definition in &STATS {
669 match stat_definition {
670 RuntimeStats::TraceLength(mmm) => {
671 trace_stats.min_trace_length =
672 Some(get_number(user_stats, &(mmm.name.to_owned() + "-min")));
673 trace_stats.max_trace_length =
674 Some(get_number(user_stats, &(mmm.name.to_owned() + "-max")));
675 trace_stats.mean_trace_length =
676 Some(get_number(user_stats, &(mmm.name.to_owned() + "-mean")));
677 }
678 RuntimeStats::TermSize(mmm) => {
679 trace_stats.min_term_size =
680 Some(get_number(user_stats, &(mmm.name.to_owned() + "-min")));
681 trace_stats.max_term_size =
682 Some(get_number(user_stats, &(mmm.name.to_owned() + "-max")));
683 trace_stats.mean_term_size =
684 Some(get_number(user_stats, &(mmm.name.to_owned() + "-mean")));
685 }
686 RuntimeStats::NbPayload(mmm) => {
687 trace_stats.max_nb_payload =
688 Some(get_number(user_stats, &(mmm.name.to_owned() + "-max")));
689 trace_stats.mean_nb_payload =
690 Some(get_number(user_stats, &(mmm.name.to_owned() + "-mean")));
691 }
692 RuntimeStats::PayloadLength(mmm) => {
693 trace_stats.min_payload_size =
694 Some(get_number(user_stats, &(mmm.name.to_owned() + "-min")));
695 trace_stats.max_payload_size =
696 Some(get_number(user_stats, &(mmm.name.to_owned() + "-max")));
697 trace_stats.mean_payload_size =
698 Some(get_number(user_stats, &(mmm.name.to_owned() + "-mean")));
699 }
700 _ => {}
701 }
702 }
703
704 trace_stats
705 }
706}
707
708trait EventHandler: DynClone {
709 fn process(&mut self, source: ClientId, msg: &str, stats: &Statistics);
710}
711
712dyn_clone::clone_trait_object!(EventHandler);
713
714impl<F> EventHandler for F
715where
716 F: FnMut(ClientId, &str, &Statistics) + Clone + 'static,
717{
718 fn process(&mut self, source: ClientId, msg: &str, stats: &Statistics) {
719 self(source, msg, stats);
720 }
721}
722
723struct JSONEventHandler {
724 output_path: PathBuf,
725 writer: BufWriter<File>,
726}
727
728impl JSONEventHandler {
729 fn new<P>(output_path: P) -> Self
730 where
731 P: AsRef<Path>,
732 {
733 let writer = BufWriter::new({
734 let mut o = OpenOptions::new();
735 OpenOptions::append(&mut o, true);
736 o.create(true).open(output_path.as_ref()).unwrap()
737 });
738
739 Self {
740 output_path: output_path.as_ref().to_path_buf(),
741 writer,
742 }
743 }
744}
745
746impl Clone for JSONEventHandler {
747 fn clone(&self) -> Self {
748 Self::new(self.output_path.clone())
749 }
750}
751
752impl EventHandler for JSONEventHandler {
753 fn process(&mut self, _source: ClientId, _msg: &str, stats: &Statistics) {
754 let mut serializer = JSONSerializer::new(&mut self.writer);
755 stats.serialize(&mut serializer).unwrap();
756 self.writer.flush().unwrap();
757 }
758}
759
760#[cfg(test)]
761mod tests {
762 use std::path::PathBuf;
763
764 use libafl::monitors::stats::ClientStatsManager;
765 use libafl_bolts::prelude::ClientId;
766
767 use super::*;
768
769 #[test]
770 fn test_display_with_unregistered_client_id_0() {
771 let mut monitor =
772 StatsMonitor::with_raw_output(PathBuf::from("/dev/null"), Duration::from_secs(1));
773 let mut mgr = ClientStatsManager::default();
774 monitor
776 .display(&mut mgr, "Broker Heartbeat", ClientId(0))
777 .expect("display should not fail for unregistered ClientId(0)");
778 }
779}