# Copyright (c) 2023-2024 Arista Networks, Inc. # Use of this source code is governed by the Apache License 2.0 # that can be found in the LICENSE file. """Result Manager module for ANTA.""" from __future__ import annotations import json import logging from collections import defaultdict from functools import cached_property from itertools import chain from typing import Any from anta.result_manager.models import AntaTestStatus, TestResult from .models import CategoryStats, DeviceStats, TestStats logger = logging.getLogger(__name__) # pylint: disable=too-many-instance-attributes class ResultManager: """Helper to manage Test Results and generate reports. Examples -------- Create Inventory: inventory_anta = AntaInventory.parse( filename='examples/inventory.yml', username='ansible', password='ansible', ) Create Result Manager: manager = ResultManager() Run tests for all connected devices: for device in inventory_anta.get_inventory().devices: manager.add( VerifyNTP(device=device).test() ) manager.add( VerifyEOSVersion(device=device).test(version='4.28.3M') ) Print result in native format: manager.results [ TestResult( name="pf1", test="VerifyZeroTouch", categories=["configuration"], description="Verifies ZeroTouch is disabled", result="success", messages=[], custom_field=None, ), TestResult( name="pf1", test='VerifyNTP', categories=["software"], categories=['system'], description='Verifies if NTP is synchronised.', result='failure', messages=["The device is not synchronized with the configured NTP server(s): 'NTP is disabled.'"], custom_field=None, ), ] """ _result_entries: list[TestResult] status: AntaTestStatus error_status: bool _device_stats: defaultdict[str, DeviceStats] _category_stats: defaultdict[str, CategoryStats] _test_stats: defaultdict[str, TestStats] _stats_in_sync: bool def __init__(self) -> None: """Class constructor. The status of the class is initialized to "unset" Then when adding a test with a status that is NOT 'error' the following table shows the updated status: | Current Status | Added test Status | Updated Status | | -------------- | ------------------------------- | -------------- | | unset | Any | Any | | skipped | unset, skipped | skipped | | skipped | success | success | | skipped | failure | failure | | success | unset, skipped, success | success | | success | failure | failure | | failure | unset, skipped success, failure | failure | If the status of the added test is error, the status is untouched and the error_status is set to True. """ self.reset() def reset(self) -> None: """Create or reset the attributes of the ResultManager instance.""" self._result_entries: list[TestResult] = [] self.status: AntaTestStatus = AntaTestStatus.UNSET self.error_status = False # Initialize the statistics attributes self._reset_stats() def __len__(self) -> int: """Implement __len__ method to count number of results.""" return len(self._result_entries) @property def results(self) -> list[TestResult]: """Get the list of TestResult.""" return self._result_entries @results.setter def results(self, value: list[TestResult]) -> None: """Set the list of TestResult.""" # When setting the results, we need to reset the state of the current instance self.reset() for result in value: self.add(result) @property def dump(self) -> list[dict[str, Any]]: """Get a list of dictionary of the results.""" return [result.model_dump() for result in self._result_entries] @property def json(self) -> str: """Get a JSON representation of the results.""" return json.dumps(self.dump, indent=4) @property def device_stats(self) -> defaultdict[str, DeviceStats]: """Get the device statistics.""" self._ensure_stats_in_sync() return self._device_stats @property def category_stats(self) -> defaultdict[str, CategoryStats]: """Get the category statistics.""" self._ensure_stats_in_sync() return self._category_stats @property def test_stats(self) -> defaultdict[str, TestStats]: """Get the test statistics.""" self._ensure_stats_in_sync() return self._test_stats @property def sorted_category_stats(self) -> dict[str, CategoryStats]: """A property that returns the category_stats dictionary sorted by key name.""" self._ensure_stats_in_sync() return dict(sorted(self.category_stats.items())) @cached_property def results_by_status(self) -> dict[AntaTestStatus, list[TestResult]]: """A cached property that returns the results grouped by status.""" return {status: [result for result in self._result_entries if result.result == status] for status in AntaTestStatus} def _update_status(self, test_status: AntaTestStatus) -> None: """Update the status of the ResultManager instance based on the test status. Parameters ---------- test_status AntaTestStatus to update the ResultManager status. """ if test_status == "error": self.error_status = True return if self.status == "unset" or (self.status == "skipped" and test_status in {"success", "failure"}): self.status = test_status elif self.status == "success" and test_status == "failure": self.status = AntaTestStatus.FAILURE def _reset_stats(self) -> None: """Create or reset the statistics attributes.""" self._device_stats = defaultdict(DeviceStats) self._category_stats = defaultdict(CategoryStats) self._test_stats = defaultdict(TestStats) self._stats_in_sync = False def _update_stats(self, result: TestResult) -> None: """Update the statistics based on the test result. Parameters ---------- result TestResult to update the statistics. """ count_attr = f"tests_{result.result}_count" # Update device stats device_stats: DeviceStats = self._device_stats[result.name] setattr(device_stats, count_attr, getattr(device_stats, count_attr) + 1) if result.result in ("failure", "error"): device_stats.tests_failure.add(result.test) device_stats.categories_failed.update(result.categories) elif result.result == "skipped": device_stats.categories_skipped.update(result.categories) # Update category stats for category in result.categories: category_stats: CategoryStats = self._category_stats[category] setattr(category_stats, count_attr, getattr(category_stats, count_attr) + 1) # Update test stats count_attr = f"devices_{result.result}_count" test_stats: TestStats = self._test_stats[result.test] setattr(test_stats, count_attr, getattr(test_stats, count_attr) + 1) if result.result in ("failure", "error"): test_stats.devices_failure.add(result.name) def _compute_stats(self) -> None: """Compute all statistics from the current results.""" logger.info("Computing statistics for all results.") # Reset all stats self._reset_stats() # Recompute stats for all results for result in self._result_entries: self._update_stats(result) self._stats_in_sync = True def _ensure_stats_in_sync(self) -> None: """Ensure statistics are in sync with current results.""" if not self._stats_in_sync: self._compute_stats() def add(self, result: TestResult) -> None: """Add a result to the ResultManager instance. The result is added to the internal list of results and the overall status of the ResultManager instance is updated based on the added test status. Parameters ---------- result TestResult to add to the ResultManager instance. """ self._result_entries.append(result) self._update_status(result.result) self._stats_in_sync = False # Every time a new result is added, we need to clear the cached property self.__dict__.pop("results_by_status", None) def get_results(self, status: set[AntaTestStatus] | None = None, sort_by: list[str] | None = None) -> list[TestResult]: """Get the results, optionally filtered by status and sorted by TestResult fields. If no status is provided, all results are returned. Parameters ---------- status Optional set of AntaTestStatus enum members to filter the results. sort_by Optional list of TestResult fields to sort the results. Returns ------- list[TestResult] List of results. """ # Return all results if no status is provided, otherwise return results for multiple statuses results = self._result_entries if status is None else list(chain.from_iterable(self.results_by_status.get(status, []) for status in status)) if sort_by: accepted_fields = TestResult.model_fields.keys() if not set(sort_by).issubset(set(accepted_fields)): msg = f"Invalid sort_by fields: {sort_by}. Accepted fields are: {list(accepted_fields)}" raise ValueError(msg) results = sorted(results, key=lambda result: [getattr(result, field) for field in sort_by]) return results def get_total_results(self, status: set[AntaTestStatus] | None = None) -> int: """Get the total number of results, optionally filtered by status. If no status is provided, the total number of results is returned. Parameters ---------- status Optional set of AntaTestStatus enum members to filter the results. Returns ------- int Total number of results. """ if status is None: # Return the total number of results return sum(len(results) for results in self.results_by_status.values()) # Return the total number of results for multiple statuses return sum(len(self.results_by_status.get(status, [])) for status in status) def get_status(self, *, ignore_error: bool = False) -> str: """Return the current status including error_status if ignore_error is False.""" return "error" if self.error_status and not ignore_error else self.status def filter(self, hide: set[AntaTestStatus]) -> ResultManager: """Get a filtered ResultManager based on test status. Parameters ---------- hide Set of AntaTestStatus enum members to select tests to hide based on their status. Returns ------- ResultManager A filtered `ResultManager`. """ possible_statuses = set(AntaTestStatus) manager = ResultManager() manager.results = self.get_results(possible_statuses - hide) return manager def filter_by_tests(self, tests: set[str]) -> ResultManager: """Get a filtered ResultManager that only contains specific tests. Parameters ---------- tests Set of test names to filter the results. Returns ------- ResultManager A filtered `ResultManager`. """ manager = ResultManager() manager.results = [result for result in self._result_entries if result.test in tests] return manager def filter_by_devices(self, devices: set[str]) -> ResultManager: """Get a filtered ResultManager that only contains specific devices. Parameters ---------- devices Set of device names to filter the results. Returns ------- ResultManager A filtered `ResultManager`. """ manager = ResultManager() manager.results = [result for result in self._result_entries if result.name in devices] return manager def get_tests(self) -> set[str]: """Get the set of all the test names. Returns ------- set[str] Set of test names. """ return {str(result.test) for result in self._result_entries} def get_devices(self) -> set[str]: """Get the set of all the device names. Returns ------- set[str] Set of device names. """ return {str(result.name) for result in self._result_entries}