diff options
| -rw-r--r-- | test/test-manager/src/config.rs | 10 | ||||
| -rw-r--r-- | test/test-manager/src/main.rs | 23 | ||||
| -rw-r--r-- | test/test-manager/src/run_tests.rs | 8 | ||||
| -rw-r--r-- | test/test-manager/src/summary.rs | 75 | ||||
| -rw-r--r-- | test/test-manager/src/tests/config.rs | 3 | ||||
| -rw-r--r-- | test/test-manager/src/tests/install.rs | 2 | ||||
| -rw-r--r-- | test/test-manager/src/tests/test_metadata.rs | 10 | ||||
| -rw-r--r-- | test/test-manager/src/tests/tunnel.rs | 4 | ||||
| -rw-r--r-- | test/test-manager/src/tests/ui.rs | 2 | ||||
| -rw-r--r-- | test/test-manager/test_macro/src/lib.rs | 31 | ||||
| -rw-r--r-- | test/test-rpc/src/client.rs | 8 | ||||
| -rw-r--r-- | test/test-rpc/src/lib.rs | 3 | ||||
| -rw-r--r-- | test/test-rpc/src/meta.rs | 26 | ||||
| -rw-r--r-- | test/test-runner/src/main.rs | 5 |
14 files changed, 144 insertions, 66 deletions
diff --git a/test/test-manager/src/config.rs b/test/test-manager/src/config.rs index 7145dca8a5..0acbeb322e 100644 --- a/test/test-manager/src/config.rs +++ b/test/test-manager/src/config.rs @@ -195,6 +195,16 @@ pub enum OsType { Macos, } +impl From<OsType> for test_rpc::meta::Os { + fn from(ostype: OsType) -> Self { + match ostype { + OsType::Windows => Self::Windows, + OsType::Linux => Self::Linux, + OsType::Macos => Self::Macos, + } + } +} + #[derive(clap::ValueEnum, Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum PackageType { diff --git a/test/test-manager/src/main.rs b/test/test-manager/src/main.rs index 37a46c2580..f5e7a70d9f 100644 --- a/test/test-manager/src/main.rs +++ b/test/test-manager/src/main.rs @@ -208,15 +208,6 @@ async fn main() -> Result<()> { verbose, test_report, } => { - let summary_logger = match test_report { - Some(path) => Some( - summary::SummaryLogger::new(&name, &path) - .await - .context("Failed to create summary logger")?, - ), - None => None, - }; - let mut config = config.clone(); config.runtime_opts.display = match (display, vnc.is_some()) { (false, false) => config::Display::None, @@ -233,6 +224,19 @@ async fn main() -> Result<()> { let vm_config = vm::get_vm_config(&config, &name).context("Cannot get VM config")?; + let summary_logger = match test_report { + Some(path) => Some( + summary::SummaryLogger::new( + &name, + test_rpc::meta::Os::from(vm_config.os_type), + &path, + ) + .await + .context("Failed to create summary logger")?, + ), + None => None, + }; + let manifest = package::get_app_manifest(vm_config, current_app, previous_app) .await .context("Could not find the specified app packages")?; @@ -273,6 +277,7 @@ async fn main() -> Result<()> { host_bridge_name: crate::vm::network::macos::find_vm_bridge()?, #[cfg(not(target_os = "macos"))] host_bridge_name: crate::vm::network::linux::BRIDGE_NAME.to_owned(), + os: test_rpc::meta::Os::from(vm_config.os_type), }, &*instance, &test_filters, diff --git a/test/test-manager/src/run_tests.rs b/test/test-manager/src/run_tests.rs index 4a296ce288..b316511b06 100644 --- a/test/test-manager/src/run_tests.rs +++ b/test/test-manager/src/run_tests.rs @@ -1,5 +1,5 @@ use crate::summary::{self, maybe_log_test_result}; -use crate::tests::TestContext; +use crate::tests::{config::TEST_CONFIG, TestContext}; use crate::{ logging::{panic_as_string, TestOutput}, mullvad_daemon, tests, @@ -26,7 +26,7 @@ pub async fn run( mut summary_logger: Option<summary::SummaryLogger>, ) -> Result<()> { log::trace!("Setting test constants"); - tests::config::TEST_CONFIG.init(config); + TEST_CONFIG.init(config); let pty_path = instance.get_pty(); @@ -47,7 +47,9 @@ pub async fn run( let mullvad_client = mullvad_daemon::new_rpc_client(connection_handle, mullvad_daemon_transport); - let mut tests: Vec<_> = inventory::iter::<tests::TestMetadata>().collect(); + let mut tests: Vec<_> = inventory::iter::<tests::TestMetadata>() + .filter(|test| test.should_run_on_os(TEST_CONFIG.os)) + .collect(); tests.sort_by_key(|test| test.priority.unwrap_or(0)); if !test_filters.is_empty() { diff --git a/test/test-manager/src/summary.rs b/test/test-manager/src/summary.rs index b8a6d5d809..27ff2db572 100644 --- a/test/test-manager/src/summary.rs +++ b/test/test-manager/src/summary.rs @@ -1,4 +1,5 @@ use std::{collections::BTreeMap, io, path::Path}; +use test_rpc::meta::Os; use tokio::{ fs, io::{AsyncBufReadExt, AsyncWriteExt}, @@ -15,18 +16,24 @@ pub enum Error { Read(#[error(source)] io::Error), #[error(display = "Failed to parse log file")] Parse, + #[error(display = "Failed to serialize value")] + Serialize(#[error(source)] serde_json::Error), + #[error(display = "Failed to deserialize value")] + Deserialize(#[error(source)] serde_json::Error), } #[derive(Clone, Copy)] pub enum TestResult { Pass, Fail, + Skip, Unknown, } impl TestResult { const PASS_STR: &'static str = "✅"; const FAIL_STR: &'static str = "❌"; + const SKIP_STR: &'static str = "↪️"; const UNKNOWN_STR: &'static str = " "; } @@ -37,6 +44,7 @@ impl std::str::FromStr for TestResult { match s { TestResult::PASS_STR => Ok(TestResult::Pass), TestResult::FAIL_STR => Ok(TestResult::Fail), + TestResult::SKIP_STR => Ok(TestResult::Skip), _ => Ok(TestResult::Unknown), } } @@ -47,6 +55,7 @@ impl std::fmt::Display for TestResult { match self { TestResult::Pass => f.write_str(TestResult::PASS_STR), TestResult::Fail => f.write_str(TestResult::FAIL_STR), + TestResult::Skip => f.write_str(TestResult::SKIP_STR), TestResult::Unknown => f.write_str(TestResult::UNKNOWN_STR), } } @@ -60,7 +69,7 @@ pub struct SummaryLogger { impl SummaryLogger { /// Create a new logger and log to `path`. If `path` does not exist, it will be created. If it /// already exists, it is truncated and overwritten. - pub async fn new(name: &str, path: &Path) -> Result<SummaryLogger, Error> { + pub async fn new(name: &str, os: Os, path: &Path) -> Result<SummaryLogger, Error> { let mut file = fs::OpenOptions::new() .create(true) .write(true) @@ -69,11 +78,14 @@ impl SummaryLogger { .await .map_err(|err| Error::Open(err, path.to_path_buf()))?; - // The first row is the summary name file.write_all(name.as_bytes()) .await .map_err(Error::Write)?; file.write_u8(b'\n').await.map_err(Error::Write)?; + file.write_all(&serde_json::to_vec(&os).map_err(Error::Serialize)?) + .await + .map_err(Error::Write)?; + file.write_u8(b'\n').await.map_err(Error::Write)?; Ok(SummaryLogger { file }) } @@ -113,15 +125,18 @@ pub async fn maybe_log_test_result( /// Parsed summary results pub struct Summary { - /// Summary name - name: String, + /// Name of the configuration + config_name: String, /// Pairs of test names mapped to test results results: BTreeMap<String, TestResult>, } impl Summary { /// Read test summary from `path`. - pub async fn parse_log<P: AsRef<Path>>(path: P) -> Result<Summary, Error> { + pub async fn parse_log<P: AsRef<Path>>( + all_tests: &[&crate::tests::TestMetadata], + path: P, + ) -> Result<Summary, Error> { let file = fs::OpenOptions::new() .read(true) .open(&path) @@ -130,11 +145,17 @@ impl Summary { let mut lines = tokio::io::BufReader::new(file).lines(); - let name = lines + let config_name = lines .next_line() .await .map_err(Error::Read)? .ok_or(Error::Parse)?; + let os = lines + .next_line() + .await + .map_err(Error::Read)? + .ok_or(Error::Parse)?; + let os: Os = serde_json::from_str(&os).map_err(Error::Deserialize)?; let mut results = BTreeMap::new(); @@ -147,7 +168,20 @@ impl Summary { results.insert(test_name.to_owned(), test_result); } - Ok(Summary { name, results }) + for test in all_tests { + // Add missing test results + let entry = results.entry(test.name.to_owned()); + if test.should_run_on_os(os) { + entry.or_insert(TestResult::Unknown); + } else { + entry.or_insert(TestResult::Skip); + } + } + + Ok(Summary { + config_name, + results, + }) } // Return all tests which passed. @@ -165,18 +199,18 @@ impl Summary { /// exist. If some log file which is expected to exist, but for any reason fails to /// be parsed, we should not abort the entire summarization. pub async fn print_summary_table<P: AsRef<Path>>(summary_files: &[P]) { - let mut summaries = Vec::new(); - let mut failed_to_parse = Vec::new(); + // Collect test details + let tests: Vec<_> = inventory::iter::<crate::tests::TestMetadata>().collect(); + + let mut summaries = vec![]; + let mut failed_to_parse = vec![]; for sumfile in summary_files { - match Summary::parse_log(sumfile).await { + match Summary::parse_log(&tests, sumfile).await { Ok(summary) => summaries.push(summary), Err(_) => failed_to_parse.push(sumfile), } } - // Collect test details - let tests: Vec<_> = inventory::iter::<crate::tests::TestMetadata>().collect(); - // Print a table println!("<table>"); @@ -185,7 +219,7 @@ pub async fn print_summary_table<P: AsRef<Path>>(summary_files: &[P]) { println!("<td style='text-align: center;'>Test ⬇️ / Platform ➡️ </td>"); for summary in &summaries { - let total_tests = tests.len(); + let total_tests = summary.results.len(); let total_passed = summary.passed().len(); let counter_text = if total_passed == total_tests { String::from(TestResult::PASS_STR) @@ -194,7 +228,7 @@ pub async fn print_summary_table<P: AsRef<Path>>(summary_files: &[P]) { }; println!( "<td style='text-align: center;'>{} {}</td>", - summary.name, counter_text + summary.config_name, counter_text ); } @@ -203,7 +237,7 @@ pub async fn print_summary_table<P: AsRef<Path>>(summary_files: &[P]) { println!("{}", { let oses_passed: Vec<_> = summaries .iter() - .filter(|summary| summary.passed().len() == tests.len()) + .filter(|summary| summary.passed().len() == summary.results.len()) .collect(); if oses_passed.len() == summaries.len() { "🎉 All Platforms passed 🎉".to_string() @@ -211,7 +245,7 @@ pub async fn print_summary_table<P: AsRef<Path>>(summary_files: &[P]) { let failed: usize = summaries .iter() .map(|summary| { - if summary.passed().len() == tests.len() { + if summary.passed().len() == summary.results.len() { 0 } else { 1 @@ -246,9 +280,9 @@ pub async fn print_summary_table<P: AsRef<Path>>(summary_files: &[P]) { .unwrap_or(&TestResult::Unknown); match result { TestResult::Fail | TestResult::Unknown => { - failed_platforms.push(summary.name.clone()) + failed_platforms.push(summary.config_name.clone()) } - TestResult::Pass => (), + TestResult::Pass | TestResult::Skip => (), } println!("<td style='text-align: center;'>{}</td>", result); } @@ -267,7 +301,7 @@ pub async fn print_summary_table<P: AsRef<Path>>(summary_files: &[P]) { ); println!("</td>"); - // List the test name again (Useful for the summary accross the different platforms) + // List the test name again (Useful for the summary across the different platforms) println!("<td>{}</td>", test.name); // End row @@ -279,4 +313,5 @@ pub async fn print_summary_table<P: AsRef<Path>>(summary_files: &[P]) { // Print explanation of test result println!("<p>{} = Test passed</p>", TestResult::PASS_STR); println!("<p>{} = Test failed</p>", TestResult::FAIL_STR); + println!("<p>{} = Test skipped</p>", TestResult::SKIP_STR); } diff --git a/test/test-manager/src/tests/config.rs b/test/test-manager/src/tests/config.rs index a0a22368dd..7ffe737aa7 100644 --- a/test/test-manager/src/tests/config.rs +++ b/test/test-manager/src/tests/config.rs @@ -1,5 +1,6 @@ use once_cell::sync::OnceCell; use std::ops::Deref; +use test_rpc::meta::Os; // Default `mullvad_host`. This should match the production env. pub const DEFAULT_MULLVAD_HOST: &str = "mullvad.net"; @@ -20,6 +21,8 @@ pub struct TestConfig { pub mullvad_host: String, pub host_bridge_name: String, + + pub os: Os, } #[derive(Debug, Clone)] diff --git a/test/test-manager/src/tests/install.rs b/test/test-manager/src/tests/install.rs index 48ce233493..ec1344ede0 100644 --- a/test/test-manager/src/tests/install.rs +++ b/test/test-manager/src/tests/install.rs @@ -360,7 +360,7 @@ async fn replace_openvpn_cert(rpc: &ServiceClient) -> Result<(), Error> { const SOURCE_CERT_FILENAME: &str = "openvpn.ca.crt"; const DEST_CERT_FILENAME: &str = "ca.crt"; - let dest_dir = match rpc.get_os().await.expect("failed to get OS") { + let dest_dir = match TEST_CONFIG.os { Os::Windows => "C:\\Program Files\\Mullvad VPN\\resources", Os::Linux => "/opt/Mullvad VPN/resources", Os::Macos => "/Applications/Mullvad VPN.app/Contents/Resources", diff --git a/test/test-manager/src/tests/test_metadata.rs b/test/test-manager/src/tests/test_metadata.rs index 39d802e5e0..3e28a4380b 100644 --- a/test/test-manager/src/tests/test_metadata.rs +++ b/test/test-manager/src/tests/test_metadata.rs @@ -1,9 +1,11 @@ use super::TestWrapperFunction; +use test_rpc::meta::Os; use test_rpc::mullvad_daemon::MullvadClientVersion; pub struct TestMetadata { pub name: &'static str, pub command: &'static str, + pub target_os: Option<Os>, pub mullvad_client_version: MullvadClientVersion, pub func: TestWrapperFunction, pub priority: Option<i32>, @@ -12,5 +14,13 @@ pub struct TestMetadata { pub cleanup: bool, } +impl TestMetadata { + pub fn should_run_on_os(&self, os: Os) -> bool { + self.target_os + .map(|target_os| target_os == os) + .unwrap_or(true) + } +} + // Register our test metadata struct with inventory to allow submitting tests of this type. inventory::collect!(TestMetadata); diff --git a/test/test-manager/src/tests/tunnel.rs b/test/test-manager/src/tests/tunnel.rs index d36ba4febe..9544f098b2 100644 --- a/test/test-manager/src/tests/tunnel.rs +++ b/test/test-manager/src/tests/tunnel.rs @@ -1,7 +1,7 @@ use super::helpers::{ self, connect_and_wait, disconnect_and_wait, set_bridge_settings, set_relay_settings, }; -use super::{Error, TestContext}; +use super::{config::TEST_CONFIG, Error, TestContext}; use crate::network_monitor::{start_packet_monitor, MonitorOptions}; use mullvad_management_interface::{types, ManagementServiceClient}; @@ -502,7 +502,7 @@ async fn check_tunnel_psk( mullvad_client: &ManagementServiceClient, should_have_psk: bool, ) { - match rpc.get_os().await.expect("failed to get OS") { + match TEST_CONFIG.os { Os::Linux => { let name = helpers::get_tunnel_interface(mullvad_client.clone()) .await diff --git a/test/test-manager/src/tests/ui.rs b/test/test-manager/src/tests/ui.rs index 3600eb1d27..461d20ae10 100644 --- a/test/test-manager/src/tests/ui.rs +++ b/test/test-manager/src/tests/ui.rs @@ -32,7 +32,7 @@ pub async fn run_test_env< let new_params: Vec<String>; let bin_path; - match rpc.get_os().await? { + match TEST_CONFIG.os { Os::Linux => { bin_path = PathBuf::from("/usr/bin/xvfb-run"); diff --git a/test/test-manager/test_macro/src/lib.rs b/test/test-manager/test_macro/src/lib.rs index b82b796eba..07f353ad7a 100644 --- a/test/test-manager/test_macro/src/lib.rs +++ b/test/test-manager/test_macro/src/lib.rs @@ -37,6 +37,9 @@ use syn::{AttributeArgs, Lit, Meta, NestedMeta}; /// filters are provided by the user. /// `always_run` defaults to false. /// +/// * `target_os` - The test should only run on the specified OS. This can currently be +/// set to `linux`, `windows`, or `macos`. +/// /// # Examples /// /// ## Create a standard test. @@ -102,12 +105,14 @@ fn get_test_macro_parameters(attributes: &syn::AttributeArgs) -> MacroParameters let mut cleanup = true; let mut always_run = false; let mut must_succeed = false; + let mut target_os = None; + for attribute in attributes { if let NestedMeta::Meta(Meta::NameValue(nv)) = attribute { if nv.path.is_ident("priority") { match &nv.lit { Lit::Int(lit_int) => { - priority = Some(lit_int.clone()); + priority = Some(lit_int.base10_parse().unwrap()); } _ => panic!("'priority' should have an integer value"), } @@ -132,6 +137,13 @@ fn get_test_macro_parameters(attributes: &syn::AttributeArgs) -> MacroParameters } _ => panic!("'cleanup' should have a bool value"), } + } else if nv.path.is_ident("target_os") { + match &nv.lit { + Lit::Str(lit_str) => { + target_os = Some(lit_str.value()); + } + _ => panic!("'target_os' should have a string value"), + } } } } @@ -141,13 +153,22 @@ fn get_test_macro_parameters(attributes: &syn::AttributeArgs) -> MacroParameters cleanup, always_run, must_succeed, + target_os, } } fn create_test(test_function: TestFunction) -> proc_macro2::TokenStream { let test_function_priority = match test_function.macro_parameters.priority { - Some(priority) => quote! {Some(#priority)}, - None => quote! {None}, + Some(priority) => quote! { Some(#priority) }, + None => quote! { None }, + }; + let target_os = match test_function.macro_parameters.target_os { + Some(target_os) => { + quote! { + Some(::std::str::FromStr::from_str(#target_os).expect("invalid target os")) + } + } + None => quote! { None }, }; let should_cleanup = test_function.macro_parameters.cleanup; let always_run = test_function.macro_parameters.always_run; @@ -191,6 +212,7 @@ fn create_test(test_function: TestFunction) -> proc_macro2::TokenStream { inventory::submit!(crate::tests::test_metadata::TestMetadata { name: stringify!(#func_name), command: stringify!(#func_name), + target_os: #target_os, mullvad_client_version: #function_mullvad_version, func: Box::new(#wrapper_closure), priority: #test_function_priority, @@ -208,10 +230,11 @@ struct TestFunction { } struct MacroParameters { - priority: Option<syn::LitInt>, + priority: Option<i32>, cleanup: bool, always_run: bool, must_succeed: bool, + target_os: Option<String>, } enum MullvadClient { diff --git a/test/test-rpc/src/client.rs b/test/test-rpc/src/client.rs index 29f86b9443..2c47328e00 100644 --- a/test/test-rpc/src/client.rs +++ b/test/test-rpc/src/client.rs @@ -109,14 +109,6 @@ impl ServiceClient { .map_err(Error::Tarpc) } - /// Return the OS of the guest. - pub async fn get_os(&self) -> Result<meta::Os, Error> { - self.client - .get_os(tarpc::context::current()) - .await - .map_err(Error::Tarpc) - } - /// Wait for the Mullvad service to enter a specified state. The state is inferred from the presence /// of a named pipe or UDS, not the actual system service state. pub async fn mullvad_daemon_wait_for_state( diff --git a/test/test-rpc/src/lib.rs b/test/test-rpc/src/lib.rs index 73ab467ab8..5919a894d1 100644 --- a/test/test-rpc/src/lib.rs +++ b/test/test-rpc/src/lib.rs @@ -111,9 +111,6 @@ mod service { async fn get_mullvad_app_logs() -> logging::LogOutput; - /// Return the OS of the guest. - async fn get_os() -> meta::Os; - /// Return status of the system service. async fn mullvad_daemon_get_status() -> mullvad_daemon::ServiceStatus; diff --git a/test/test-rpc/src/meta.rs b/test/test-rpc/src/meta.rs index 67c87738e0..09200f8690 100644 --- a/test/test-rpc/src/meta.rs +++ b/test/test-rpc/src/meta.rs @@ -1,12 +1,27 @@ use serde::{Deserialize, Serialize}; +use std::str::FromStr; -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Copy)] +#[serde(rename_all = "snake_case")] pub enum Os { Linux, Macos, Windows, } +impl FromStr for Os { + type Err = Box<dyn std::error::Error>; + + fn from_str(s: &str) -> Result<Self, Self::Err> { + match s { + "linux" => Ok(Os::Linux), + "macos" => Ok(Os::Macos), + "windows" => Ok(Os::Windows), + other => Err(format!("unknown os {other}").into()), + } + } +} + impl std::fmt::Display for Os { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { @@ -16,12 +31,3 @@ impl std::fmt::Display for Os { } } } - -#[cfg(target_os = "linux")] -pub const CURRENT_OS: Os = Os::Linux; - -#[cfg(target_os = "windows")] -pub const CURRENT_OS: Os = Os::Windows; - -#[cfg(target_os = "macos")] -pub const CURRENT_OS: Os = Os::Macos; diff --git a/test/test-runner/src/main.rs b/test/test-runner/src/main.rs index c67876526b..d7ace9d15a 100644 --- a/test/test-runner/src/main.rs +++ b/test/test-runner/src/main.rs @@ -9,7 +9,6 @@ use std::{ use tarpc::context; use tarpc::server::Channel; use test_rpc::{ - meta, mullvad_daemon::{ServiceStatus, SOCKET_PATH}, package::Package, transport::GrpcForwarder, @@ -96,10 +95,6 @@ impl Service for TestServer { Ok(result) } - async fn get_os(self, _: context::Context) -> meta::Os { - meta::CURRENT_OS - } - async fn mullvad_daemon_get_status( self, _: context::Context, |