312 lines
12 KiB
Python
312 lines
12 KiB
Python
|
# Copyright (c) 2023-2024 Arista Networks, Inc.
|
||
|
|
# Use of this source code is governed by the Apache License 2.0
|
||
|
|
# that can be found in the LICENSE file.
|
||
|
|
"""
|
||
|
|
test anta.device.py
|
||
|
|
"""
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
from pathlib import Path
|
||
|
|
from typing import Any
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
from pydantic import ValidationError
|
||
|
|
from yaml import safe_load
|
||
|
|
|
||
|
|
from anta.catalog import AntaCatalog, AntaTestDefinition
|
||
|
|
from anta.models import AntaTest
|
||
|
|
from anta.tests.interfaces import VerifyL3MTU
|
||
|
|
from anta.tests.mlag import VerifyMlagStatus
|
||
|
|
from anta.tests.software import VerifyEOSVersion
|
||
|
|
from anta.tests.system import (
|
||
|
|
VerifyAgentLogs,
|
||
|
|
VerifyCoredump,
|
||
|
|
VerifyCPUUtilization,
|
||
|
|
VerifyFileSystemUtilization,
|
||
|
|
VerifyMemoryUtilization,
|
||
|
|
VerifyNTP,
|
||
|
|
VerifyReloadCause,
|
||
|
|
VerifyUptime,
|
||
|
|
)
|
||
|
|
from tests.lib.utils import generate_test_ids_list
|
||
|
|
from tests.units.test_models import FakeTestWithInput
|
||
|
|
|
||
|
|
# Test classes used as expected values
|
||
|
|
|
||
|
|
DATA_DIR: Path = Path(__file__).parent.parent.resolve() / "data"
|
||
|
|
|
||
|
|
INIT_CATALOG_DATA: list[dict[str, Any]] = [
|
||
|
|
{
|
||
|
|
"name": "test_catalog",
|
||
|
|
"filename": "test_catalog.yml",
|
||
|
|
"tests": [
|
||
|
|
(VerifyEOSVersion, VerifyEOSVersion.Input(versions=["4.31.1F"])),
|
||
|
|
],
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "test_catalog_with_tags",
|
||
|
|
"filename": "test_catalog_with_tags.yml",
|
||
|
|
"tests": [
|
||
|
|
(
|
||
|
|
VerifyUptime,
|
||
|
|
VerifyUptime.Input(
|
||
|
|
minimum=10,
|
||
|
|
filters=VerifyUptime.Input.Filters(tags=["fabric"]),
|
||
|
|
),
|
||
|
|
),
|
||
|
|
(VerifyReloadCause, {"filters": {"tags": ["leaf", "spine"]}}),
|
||
|
|
(VerifyCoredump, VerifyCoredump.Input()),
|
||
|
|
(VerifyAgentLogs, AntaTest.Input()),
|
||
|
|
(VerifyCPUUtilization, VerifyCPUUtilization.Input(filters=VerifyCPUUtilization.Input.Filters(tags=["leaf"]))),
|
||
|
|
(VerifyMemoryUtilization, VerifyMemoryUtilization.Input(filters=VerifyMemoryUtilization.Input.Filters(tags=["testdevice"]))),
|
||
|
|
(VerifyFileSystemUtilization, None),
|
||
|
|
(VerifyNTP, {}),
|
||
|
|
(VerifyMlagStatus, None),
|
||
|
|
(VerifyL3MTU, {"mtu": 1500, "filters": {"tags": ["demo"]}}),
|
||
|
|
],
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "test_empty_catalog",
|
||
|
|
"filename": "test_empty_catalog.yml",
|
||
|
|
"tests": [],
|
||
|
|
},
|
||
|
|
]
|
||
|
|
CATALOG_PARSE_FAIL_DATA: list[dict[str, Any]] = [
|
||
|
|
{
|
||
|
|
"name": "undefined_tests",
|
||
|
|
"filename": "test_catalog_with_undefined_tests.yml",
|
||
|
|
"error": "FakeTest is not defined in Python module anta.tests.software",
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "undefined_module",
|
||
|
|
"filename": "test_catalog_with_undefined_module.yml",
|
||
|
|
"error": "Module named anta.tests.undefined cannot be imported",
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "undefined_module",
|
||
|
|
"filename": "test_catalog_with_undefined_module.yml",
|
||
|
|
"error": "Module named anta.tests.undefined cannot be imported",
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "syntax_error",
|
||
|
|
"filename": "test_catalog_with_syntax_error_module.yml",
|
||
|
|
"error": "Value error, Module named tests.data.syntax_error cannot be imported. Verify that the module exists and there is no Python syntax issues.",
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "undefined_module_nested",
|
||
|
|
"filename": "test_catalog_with_undefined_module_nested.yml",
|
||
|
|
"error": "Module named undefined from package anta.tests cannot be imported",
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "not_a_list",
|
||
|
|
"filename": "test_catalog_not_a_list.yml",
|
||
|
|
"error": "Value error, Syntax error when parsing: True\nIt must be a list of ANTA tests. Check the test catalog.",
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "test_definition_not_a_dict",
|
||
|
|
"filename": "test_catalog_test_definition_not_a_dict.yml",
|
||
|
|
"error": "Value error, Syntax error when parsing: VerifyEOSVersion\nIt must be a dictionary. Check the test catalog.",
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "test_definition_multiple_dicts",
|
||
|
|
"filename": "test_catalog_test_definition_multiple_dicts.yml",
|
||
|
|
"error": "Value error, Syntax error when parsing: {'VerifyEOSVersion': {'versions': ['4.25.4M', '4.26.1F']}, "
|
||
|
|
"'VerifyTerminAttrVersion': {'versions': ['4.25.4M']}}\nIt must be a dictionary with a single entry. Check the indentation in the test catalog.",
|
||
|
|
},
|
||
|
|
{"name": "wrong_type_after_parsing", "filename": "test_catalog_wrong_type.yml", "error": "must be a dict, got str"},
|
||
|
|
]
|
||
|
|
CATALOG_FROM_DICT_FAIL_DATA: list[dict[str, Any]] = [
|
||
|
|
{
|
||
|
|
"name": "undefined_tests",
|
||
|
|
"filename": "test_catalog_with_undefined_tests.yml",
|
||
|
|
"error": "FakeTest is not defined in Python module anta.tests.software",
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "wrong_type",
|
||
|
|
"filename": "test_catalog_wrong_type.yml",
|
||
|
|
"error": "Wrong input type for catalog data, must be a dict, got str",
|
||
|
|
},
|
||
|
|
]
|
||
|
|
CATALOG_FROM_LIST_FAIL_DATA: list[dict[str, Any]] = [
|
||
|
|
{
|
||
|
|
"name": "wrong_inputs",
|
||
|
|
"tests": [
|
||
|
|
(
|
||
|
|
FakeTestWithInput,
|
||
|
|
AntaTest.Input(),
|
||
|
|
),
|
||
|
|
],
|
||
|
|
"error": "Test input has type AntaTest.Input but expected type FakeTestWithInput.Input",
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "no_test",
|
||
|
|
"tests": [(None, None)],
|
||
|
|
"error": "Input should be a subclass of AntaTest",
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "no_input_when_required",
|
||
|
|
"tests": [(FakeTestWithInput, None)],
|
||
|
|
"error": "Field required",
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "wrong_input_type",
|
||
|
|
"tests": [(FakeTestWithInput, True)],
|
||
|
|
"error": "Value error, Coud not instantiate inputs as type bool is not valid",
|
||
|
|
},
|
||
|
|
]
|
||
|
|
|
||
|
|
TESTS_SETTER_FAIL_DATA: list[dict[str, Any]] = [
|
||
|
|
{
|
||
|
|
"name": "not_a_list",
|
||
|
|
"tests": "not_a_list",
|
||
|
|
"error": "The catalog must contain a list of tests",
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "not_a_list_of_test_definitions",
|
||
|
|
"tests": [42, 43],
|
||
|
|
"error": "A test in the catalog must be an AntaTestDefinition instance",
|
||
|
|
},
|
||
|
|
]
|
||
|
|
|
||
|
|
|
||
|
|
class Test_AntaCatalog:
|
||
|
|
"""
|
||
|
|
Test for anta.catalog.AntaCatalog
|
||
|
|
"""
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("catalog_data", INIT_CATALOG_DATA, ids=generate_test_ids_list(INIT_CATALOG_DATA))
|
||
|
|
def test_parse(self, catalog_data: dict[str, Any]) -> None:
|
||
|
|
"""
|
||
|
|
Instantiate AntaCatalog from a file
|
||
|
|
"""
|
||
|
|
catalog: AntaCatalog = AntaCatalog.parse(str(DATA_DIR / catalog_data["filename"]))
|
||
|
|
|
||
|
|
assert len(catalog.tests) == len(catalog_data["tests"])
|
||
|
|
for test_id, (test, inputs) in enumerate(catalog_data["tests"]):
|
||
|
|
assert catalog.tests[test_id].test == test
|
||
|
|
if inputs is not None:
|
||
|
|
if isinstance(inputs, dict):
|
||
|
|
inputs = test.Input(**inputs)
|
||
|
|
assert inputs == catalog.tests[test_id].inputs
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("catalog_data", INIT_CATALOG_DATA, ids=generate_test_ids_list(INIT_CATALOG_DATA))
|
||
|
|
def test_from_list(self, catalog_data: dict[str, Any]) -> None:
|
||
|
|
"""
|
||
|
|
Instantiate AntaCatalog from a list
|
||
|
|
"""
|
||
|
|
catalog: AntaCatalog = AntaCatalog.from_list(catalog_data["tests"])
|
||
|
|
|
||
|
|
assert len(catalog.tests) == len(catalog_data["tests"])
|
||
|
|
for test_id, (test, inputs) in enumerate(catalog_data["tests"]):
|
||
|
|
assert catalog.tests[test_id].test == test
|
||
|
|
if inputs is not None:
|
||
|
|
if isinstance(inputs, dict):
|
||
|
|
inputs = test.Input(**inputs)
|
||
|
|
assert inputs == catalog.tests[test_id].inputs
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("catalog_data", INIT_CATALOG_DATA, ids=generate_test_ids_list(INIT_CATALOG_DATA))
|
||
|
|
def test_from_dict(self, catalog_data: dict[str, Any]) -> None:
|
||
|
|
"""
|
||
|
|
Instantiate AntaCatalog from a dict
|
||
|
|
"""
|
||
|
|
with open(file=str(DATA_DIR / catalog_data["filename"]), mode="r", encoding="UTF-8") as file:
|
||
|
|
data = safe_load(file)
|
||
|
|
catalog: AntaCatalog = AntaCatalog.from_dict(data)
|
||
|
|
|
||
|
|
assert len(catalog.tests) == len(catalog_data["tests"])
|
||
|
|
for test_id, (test, inputs) in enumerate(catalog_data["tests"]):
|
||
|
|
assert catalog.tests[test_id].test == test
|
||
|
|
if inputs is not None:
|
||
|
|
if isinstance(inputs, dict):
|
||
|
|
inputs = test.Input(**inputs)
|
||
|
|
assert inputs == catalog.tests[test_id].inputs
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("catalog_data", CATALOG_PARSE_FAIL_DATA, ids=generate_test_ids_list(CATALOG_PARSE_FAIL_DATA))
|
||
|
|
def test_parse_fail(self, catalog_data: dict[str, Any]) -> None:
|
||
|
|
"""
|
||
|
|
Errors when instantiating AntaCatalog from a file
|
||
|
|
"""
|
||
|
|
with pytest.raises((ValidationError, ValueError)) as exec_info:
|
||
|
|
AntaCatalog.parse(str(DATA_DIR / catalog_data["filename"]))
|
||
|
|
if isinstance(exec_info.value, ValidationError):
|
||
|
|
assert catalog_data["error"] in exec_info.value.errors()[0]["msg"]
|
||
|
|
else:
|
||
|
|
assert catalog_data["error"] in str(exec_info)
|
||
|
|
|
||
|
|
def test_parse_fail_parsing(self, caplog: pytest.LogCaptureFixture) -> None:
|
||
|
|
"""
|
||
|
|
Errors when instantiating AntaCatalog from a file
|
||
|
|
"""
|
||
|
|
with pytest.raises(Exception) as exec_info:
|
||
|
|
AntaCatalog.parse(str(DATA_DIR / "catalog_does_not_exist.yml"))
|
||
|
|
assert "No such file or directory" in str(exec_info)
|
||
|
|
assert len(caplog.record_tuples) >= 1
|
||
|
|
_, _, message = caplog.record_tuples[0]
|
||
|
|
assert "Unable to parse ANTA Test Catalog file" in message
|
||
|
|
assert "FileNotFoundError ([Errno 2] No such file or directory" in message
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("catalog_data", CATALOG_FROM_LIST_FAIL_DATA, ids=generate_test_ids_list(CATALOG_FROM_LIST_FAIL_DATA))
|
||
|
|
def test_from_list_fail(self, catalog_data: dict[str, Any]) -> None:
|
||
|
|
"""
|
||
|
|
Errors when instantiating AntaCatalog from a list of tuples
|
||
|
|
"""
|
||
|
|
with pytest.raises(ValidationError) as exec_info:
|
||
|
|
AntaCatalog.from_list(catalog_data["tests"])
|
||
|
|
assert catalog_data["error"] in exec_info.value.errors()[0]["msg"]
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("catalog_data", CATALOG_FROM_DICT_FAIL_DATA, ids=generate_test_ids_list(CATALOG_FROM_DICT_FAIL_DATA))
|
||
|
|
def test_from_dict_fail(self, catalog_data: dict[str, Any]) -> None:
|
||
|
|
"""
|
||
|
|
Errors when instantiating AntaCatalog from a list of tuples
|
||
|
|
"""
|
||
|
|
with open(file=str(DATA_DIR / catalog_data["filename"]), mode="r", encoding="UTF-8") as file:
|
||
|
|
data = safe_load(file)
|
||
|
|
with pytest.raises((ValidationError, ValueError)) as exec_info:
|
||
|
|
AntaCatalog.from_dict(data)
|
||
|
|
if isinstance(exec_info.value, ValidationError):
|
||
|
|
assert catalog_data["error"] in exec_info.value.errors()[0]["msg"]
|
||
|
|
else:
|
||
|
|
assert catalog_data["error"] in str(exec_info)
|
||
|
|
|
||
|
|
def test_filename(self) -> None:
|
||
|
|
"""
|
||
|
|
Test filename
|
||
|
|
"""
|
||
|
|
catalog = AntaCatalog(filename="test")
|
||
|
|
assert catalog.filename == Path("test")
|
||
|
|
catalog = AntaCatalog(filename=Path("test"))
|
||
|
|
assert catalog.filename == Path("test")
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("catalog_data", INIT_CATALOG_DATA, ids=generate_test_ids_list(INIT_CATALOG_DATA))
|
||
|
|
def test__tests_setter_success(self, catalog_data: dict[str, Any]) -> None:
|
||
|
|
"""
|
||
|
|
Success when setting AntaCatalog.tests from a list of tuples
|
||
|
|
"""
|
||
|
|
catalog = AntaCatalog()
|
||
|
|
catalog.tests = [AntaTestDefinition(test=test, inputs=inputs) for test, inputs in catalog_data["tests"]]
|
||
|
|
assert len(catalog.tests) == len(catalog_data["tests"])
|
||
|
|
for test_id, (test, inputs) in enumerate(catalog_data["tests"]):
|
||
|
|
assert catalog.tests[test_id].test == test
|
||
|
|
if inputs is not None:
|
||
|
|
if isinstance(inputs, dict):
|
||
|
|
inputs = test.Input(**inputs)
|
||
|
|
assert inputs == catalog.tests[test_id].inputs
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("catalog_data", TESTS_SETTER_FAIL_DATA, ids=generate_test_ids_list(TESTS_SETTER_FAIL_DATA))
|
||
|
|
def test__tests_setter_fail(self, catalog_data: dict[str, Any]) -> None:
|
||
|
|
"""
|
||
|
|
Errors when setting AntaCatalog.tests from a list of tuples
|
||
|
|
"""
|
||
|
|
catalog = AntaCatalog()
|
||
|
|
with pytest.raises(ValueError) as exec_info:
|
||
|
|
catalog.tests = catalog_data["tests"]
|
||
|
|
assert catalog_data["error"] in str(exec_info)
|
||
|
|
|
||
|
|
def test_get_tests_by_tags(self) -> None:
|
||
|
|
"""
|
||
|
|
Test AntaCatalog.test_get_tests_by_tags()
|
||
|
|
"""
|
||
|
|
catalog: AntaCatalog = AntaCatalog.parse(str(DATA_DIR / "test_catalog_with_tags.yml"))
|
||
|
|
tests: list[AntaTestDefinition] = catalog.get_tests_by_tags(tags=["leaf"])
|
||
|
|
assert len(tests) == 2
|