use std::{collections::BTreeMap, io, path::Path}; use test_rpc::meta::Os; use tokio::{ fs, io::{AsyncBufReadExt, AsyncWriteExt}, }; use crate::tests::should_run_on_os; #[derive(thiserror::Error, Debug)] pub enum Error { #[error("Failed to open log file {1:?}")] Open(#[source] io::Error, std::path::PathBuf), #[error("Failed to write to log file")] Write(#[source] io::Error), #[error("Failed to read from log file")] Read(#[source] io::Error), #[error("Failed to parse log file")] Parse, #[error("Failed to serialize value")] Serialize(#[source] serde_json::Error), #[error("Failed to deserialize value")] Deserialize(#[source] serde_json::Error), } #[derive(Clone, Copy)] pub enum TestResult { Pass, Fail, Skip, Unknown, } impl TestResult { const PASS_STR: &'static str = "✅"; const FAIL_STR: &'static str = "❌"; const SKIP_STR: &'static str = "↪️"; const UNKNOWN_STR: &'static str = " "; } impl std::str::FromStr for TestResult { type Err = Error; fn from_str(s: &str) -> Result { match s { TestResult::PASS_STR => Ok(TestResult::Pass), TestResult::FAIL_STR => Ok(TestResult::Fail), TestResult::SKIP_STR => Ok(TestResult::Skip), _ => Ok(TestResult::Unknown), } } } impl std::fmt::Display for TestResult { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { TestResult::Pass => f.write_str(TestResult::PASS_STR), TestResult::Fail => f.write_str(TestResult::FAIL_STR), TestResult::Skip => f.write_str(TestResult::SKIP_STR), TestResult::Unknown => f.write_str(TestResult::UNKNOWN_STR), } } } /// Logger that outputs test results in a structured format pub struct SummaryLogger { file: fs::File, } impl SummaryLogger { /// Create a new logger and log to `path`. If `path` does not exist, it will be created. If it /// already exists, it is truncated and overwritten. pub async fn new(vm: &str, os: Os, path: &Path) -> Result { let mut file = fs::OpenOptions::new() .create(true) .write(true) .truncate(true) .open(path) .await .map_err(|err| Error::Open(err, path.to_path_buf()))?; file.write_all(vm.as_bytes()).await.map_err(Error::Write)?; file.write_u8(b'\n').await.map_err(Error::Write)?; file.write_all(&serde_json::to_vec(&os).map_err(Error::Serialize)?) .await .map_err(Error::Write)?; file.write_u8(b'\n').await.map_err(Error::Write)?; Ok(SummaryLogger { file }) } pub async fn log_test_result( &mut self, test_name: &str, test_result: TestResult, ) -> Result<(), Error> { self.file .write_all(test_name.as_bytes()) .await .map_err(Error::Write)?; self.file.write_u8(b' ').await.map_err(Error::Write)?; self.file .write_all(test_result.to_string().as_bytes()) .await .map_err(Error::Write)?; self.file.write_u8(b'\n').await.map_err(Error::Write)?; Ok(()) } } /// Parsed summary results pub struct Summary { /// Name of the configuration config_name: String, /// Pairs of test names mapped to test results results: BTreeMap, } impl Summary { /// Read test summary from `path`. pub async fn parse_log>( all_tests: &[crate::tests::TestDescription], path: P, ) -> Result { let file = fs::OpenOptions::new() .read(true) .open(&path) .await .map_err(|err| Error::Open(err, path.as_ref().to_path_buf()))?; let mut lines = tokio::io::BufReader::new(file).lines(); let config_name = lines .next_line() .await .map_err(Error::Read)? .ok_or(Error::Parse)?; let os = lines .next_line() .await .map_err(Error::Read)? .ok_or(Error::Parse)?; let os: Os = serde_json::from_str(&os).map_err(Error::Deserialize)?; let mut results = BTreeMap::new(); while let Some(line) = lines.next_line().await.map_err(Error::Read)? { let mut cols = line.split_whitespace(); let test_name = cols.next().ok_or(Error::Parse)?; let test_result = cols.next().ok_or(Error::Parse)?.parse()?; results.insert(test_name.to_owned(), test_result); } for test in all_tests { // Add missing test results let entry = results.entry(test.name.to_owned()); if should_run_on_os(test.targets, os) { entry.or_insert(TestResult::Unknown); } else { entry.or_insert(TestResult::Skip); } } Ok(Summary { config_name, results, }) } // Return all tests which passed. fn passed(&self) -> Vec<&TestResult> { self.results .values() .filter(|x| matches!(x, TestResult::Pass)) .collect() } } /// Outputs an HTML table, to stdout, containing the results of the given log files. /// /// This is a best effort attempt at summarizing the log files which do /// exist. If some log file which is expected to exist, but for any reason fails to /// be parsed, we should not abort the entire summarization. pub async fn print_summary_table>(summary_files: &[P]) { // Collect test details let tests = crate::tests::get_test_descriptions(); let mut summaries = vec![]; let mut failed_to_parse = vec![]; for sumfile in summary_files { match Summary::parse_log(&tests[..], sumfile).await { Ok(summary) => summaries.push(summary), Err(_) => failed_to_parse.push(sumfile), } } // Print a table println!(""); // First row: Print summary names println!(""); println!(""); for summary in &summaries { let total_tests = summary.results.len(); let total_passed = summary.passed().len(); let counter_text = if total_passed == total_tests { String::from(TestResult::PASS_STR) } else { format!("({total_passed}/{total_tests})") }; println!( "", summary.config_name, counter_text ); } // A summary of all OSes println!(""); // List all tests again println!(""); println!(""); // Remaining rows: Print results for each test and each summary for test in &tests { println!(""); println!("", test.name,); let mut failed_platforms = vec![]; for summary in &summaries { let result = summary .results .get(test.name) .unwrap_or(&TestResult::Unknown); match result { TestResult::Fail | TestResult::Unknown => { failed_platforms.push(summary.config_name.clone()) } TestResult::Pass | TestResult::Skip => (), } println!(""); } // Print a summary of all OSes at the end of the table // For each test, collect the result for each platform. // - If the test passed on all platforms, we print a symbol declaring success // - If the test failed on any platform, we print the platform println!(""); // List the test name again (Useful for the summary across the different platforms) println!("", test.name); // End row println!(""); } println!("
Test ⬇️ / Platform ➡️ {} {}"); println!("{}", { let oses_passed: Vec<_> = summaries .iter() .filter(|summary| summary.passed().len() == summary.results.len()) .collect(); if oses_passed.len() == summaries.len() { "🎉 All Platforms passed 🎉".to_string() } else { let failed: usize = summaries .iter() .map(|summary| { if summary.passed().len() == summary.results.len() { 0 } else { 1 } }) .sum(); format!("🌧️ ️ {failed} Platform(s) failed 🌧️") } }); println!("Test ⬇️
{}{result}"); print!( "{}", if failed_platforms.is_empty() { TestResult::PASS_STR.to_string() } else { failed_platforms.join(", ") } ); println!("{}
"); // Print explanation of test result println!("

{} = Test passed

", TestResult::PASS_STR); println!("

{} = Test failed

", TestResult::FAIL_STR); println!("

{} = Test skipped

", TestResult::SKIP_STR); }