Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions tests/python_test_cases/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ score_virtualenv(
# Tests targets
score_py_pytest(
name = "cit_cpp",
srcs = glob(["tests/**/*.py"]),
srcs = glob(["tests/**/*.py"]) + ["test_properties.py"],
args = [
"-m cpp",
"--cpp-target-path=$(rootpath //tests/cpp_test_scenarios)",
Expand All @@ -58,7 +58,7 @@ score_py_pytest(

score_py_pytest(
name = "cit_rust",
srcs = glob(["tests/**/*.py"]),
srcs = glob(["tests/**/*.py"]) + ["test_properties.py"],
args = [
"-m rust",
"--rust-target-path=$(rootpath //tests/rust_test_scenarios)",
Expand Down
13 changes: 8 additions & 5 deletions tests/python_test_cases/pytest.ini
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,17 @@
[pytest]
addopts = -v
testpaths = tests
pythonpath =
.
tests
markers =
cpp
rust
PartiallyVerifies
FullyVerifies
Description
TestType
DerivationTechnique
test_properties(dict): Add custom properties to test XML output

; Additional environment variables
env = D:RUST_BACKTRACE = 1
junit_family = xunit1
filterwarnings =
ignore:record_property is incompatible with junit_family:pytest.PytestWarning
ignore:record_xml_attribute is an experimental feature:pytest.PytestExperimentalApiWarning
22 changes: 22 additions & 0 deletions tests/python_test_cases/test_properties.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# *******************************************************************************
# Copyright (c) 2025 Contributors to the Eclipse Foundation
#
# See the NOTICE file(s) distributed with this work for additional
# information regarding copyright ownership.
#
# This program and the accompanying materials are made available under the
# terms of the Apache License Version 2.0 which is available at
# https://www.apache.org/licenses/LICENSE-2.0
#
# SPDX-License-Identifier: Apache-2.0
# *******************************************************************************
try:
from attribute_plugin import add_test_properties # type: ignore[import-untyped]
except ImportError:
# Define no-op decorator if attribute_plugin is not available (outside bazel)
# Keeps IDE debugging functionality
def add_test_properties(*args, **kwargs):
def decorator(func):
return func # No-op decorator

return decorator
115 changes: 52 additions & 63 deletions tests/python_test_cases/tests/test_cit_default_values.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import pytest
from testing_utils import LogContainer, ScenarioResult
from test_properties import add_test_properties

from .common import CommonScenario, ResultCode, temp_dir_common

Expand Down Expand Up @@ -89,22 +90,20 @@ def temp_dir(
)


@pytest.mark.PartiallyVerifies(
[
@add_test_properties(
partially_verifies=[
"comp_req__persistency__value_default_v2",
"comp_req__persistency__default_value_cfg_v2",
"comp_req__persistency__default_value_types_v2",
"comp_req__persistency__default_value_query_v2",
]
],
test_type="requirements-based",
derivation_technique="requirements-based",
)
@pytest.mark.FullyVerifies([])
@pytest.mark.Description(
"Verifies default value loading, querying, and override behavior for KVS instances with and without defaults."
)
@pytest.mark.TestType("requirements-based")
@pytest.mark.DerivationTechnique("requirements-based")
@pytest.mark.parametrize("defaults", ["optional", "required", "without"], scope="class")
class TestDefaultValues(DefaultValuesScenario):
"""Verifies default value loading, querying, and override behavior for KVS instances with and without defaults."""

KEY = "test_number"
VALUE = 111.1

Expand Down Expand Up @@ -174,21 +173,19 @@ def test_valid(
assert logs[1].current_value == "Ok(F64(432.1))"


@pytest.mark.PartiallyVerifies(
[
@add_test_properties(
partially_verifies=[
"comp_req__persistency__value_default_v2",
"comp_req__persistency__default_value_cfg_v2",
"comp_req__persistency__default_value_types_v2",
]
],
test_type="requirements-based",
derivation_technique="requirements-based",
)
@pytest.mark.FullyVerifies([])
@pytest.mark.Description(
"Tests removal of values in KVS with defaults enabled, ensuring keys revert to their default values."
)
@pytest.mark.TestType("requirements-based")
@pytest.mark.DerivationTechnique("requirements-based")
@pytest.mark.parametrize("defaults", ["optional", "required", "without"], scope="class")
class TestRemoveKey(DefaultValuesScenario):
"""Tests removal of values in KVS with defaults enabled, ensuring keys revert to their default values."""

KEY = "test_number"
VALUE = 111.1

Expand Down Expand Up @@ -265,21 +262,19 @@ def test_valid(
assert logs[2].current_value == "Err(KeyNotFound)"


@pytest.mark.PartiallyVerifies(
[
@add_test_properties(
partially_verifies=[
"comp_req__persistency__value_default_v2",
"comp_req__persistency__default_value_cfg_v2",
"comp_req__persistency__default_value_types_v2",
]
],
test_type="requirements-based",
derivation_technique="requirements-based",
)
@pytest.mark.FullyVerifies([])
@pytest.mark.Description(
"Verifies that KVS fails to open when the defaults file contains invalid JSON."
)
@pytest.mark.TestType("requirements-based")
@pytest.mark.DerivationTechnique("requirements-based")
@pytest.mark.parametrize("defaults", ["optional", "required"], scope="class")
class TestMalformedDefaultsFile(DefaultValuesScenario):
"""Verifies that KVS fails to open when the defaults file contains invalid JSON."""

@pytest.fixture(scope="class")
def scenario_name(self) -> str:
return "cit.default_values.default_values"
Expand Down Expand Up @@ -336,21 +331,19 @@ def test_invalid(
assert re.findall(pattern, results.stderr) is not None


@pytest.mark.PartiallyVerifies(
[
@add_test_properties(
partially_verifies=[
"comp_req__persistency__value_default_v2",
"comp_req__persistency__default_value_cfg_v2",
"comp_req__persistency__default_value_types_v2",
]
)
@pytest.mark.FullyVerifies([])
@pytest.mark.Description(
"Verifies that KVS fails to open when the defaults file is missing."
],
test_type="requirements-based",
derivation_technique="requirements-based",
)
@pytest.mark.TestType("requirements-based")
@pytest.mark.DerivationTechnique("requirements-based")
@pytest.mark.parametrize("defaults", ["required"], scope="class")
class TestMissingDefaultsFile(DefaultValuesScenario):
"""Verifies that KVS fails to open when the defaults file is missing."""

@pytest.fixture(scope="class")
def scenario_name(self) -> str:
return "cit.default_values.default_values"
Expand All @@ -375,21 +368,20 @@ def test_invalid(self, results: ScenarioResult) -> None:
assert re.findall(pattern, results.stderr) is not None


@pytest.mark.PartiallyVerifies(
[
@add_test_properties(
fully_verifies=["comp_req__persistency__value_reset_v2"],
partially_verifies=[
"comp_req__persistency__value_default_v2",
"comp_req__persistency__default_value_cfg_v2",
"comp_req__persistency__default_value_types_v2",
]
)
@pytest.mark.FullyVerifies(["comp_req__persistency__value_reset_v2"])
@pytest.mark.Description(
"Checks that resetting KVS restores all keys to their default values."
],
test_type="requirements-based",
derivation_technique="requirements-based",
)
@pytest.mark.TestType("requirements-based")
@pytest.mark.DerivationTechnique("requirements-based")
@pytest.mark.parametrize("defaults", ["optional", "required"], scope="class")
class TestResetAllKeys(DefaultValuesScenario):
"""Checks that resetting KVS restores all keys to their default values."""

NUM_VALUES = 5

@pytest.fixture(scope="class")
Expand Down Expand Up @@ -446,20 +438,18 @@ def test_valid(
assert logs[2].current_value == 432.1 * i


@pytest.mark.PartiallyVerifies(
[
@add_test_properties(
partially_verifies=[
"comp_req__persistency__value_default_v2",
"comp_req__persistency__default_value_cfg_v2",
]
)
@pytest.mark.FullyVerifies([])
@pytest.mark.Description(
"Checks that resetting single key restores it to its default value."
],
test_type="requirements-based",
derivation_technique="requirements-based",
)
@pytest.mark.TestType("requirements-based")
@pytest.mark.DerivationTechnique("requirements-based")
@pytest.mark.parametrize("defaults", ["optional", "required"], scope="class")
class TestResetSingleKey(DefaultValuesScenario):
"""Checks that resetting single key restores it to its default value."""

NUM_VALUES = 5
RESET_INDEX = 2

Expand Down Expand Up @@ -531,20 +521,19 @@ def test_valid(
assert logs[2].current_value == 123.4 * i


@pytest.mark.PartiallyVerifies(
[
@add_test_properties(
fully_verifies=["comp_req__persistency__default_val_chksum_v2"],
partially_verifies=[
"comp_req__persistency__value_default_v2",
"comp_req__persistency__default_value_cfg_v2",
]
)
@pytest.mark.FullyVerifies(["comp_req__persistency__default_val_chksum_v2"])
@pytest.mark.Description(
"Ensures that a checksum file is created when opening KVS with defaults."
],
test_type="requirements-based",
derivation_technique="requirements-based",
)
@pytest.mark.TestType("requirements-based")
@pytest.mark.DerivationTechnique("requirements-based")
@pytest.mark.parametrize("defaults", ["optional", "required"], scope="class")
class TestChecksumOnProvidedDefaults(DefaultValuesScenario):
"""Ensures that a checksum file is created when opening KVS with defaults."""

KEY = "test_number"
VALUE = 111.1

Expand Down
49 changes: 22 additions & 27 deletions tests/python_test_cases/tests/test_cit_multiple_kvs.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,25 +15,24 @@

import pytest
from testing_utils import LogContainer, ScenarioResult
from test_properties import add_test_properties

from .common import CommonScenario, ResultCode

pytestmark = pytest.mark.parametrize("version", ["rust", "cpp"], scope="class")


@pytest.mark.PartiallyVerifies(
[
@add_test_properties(
partially_verifies=[
"comp_req__persistency__multi_instance_v2",
"comp_req__persistency__concurrency_v2",
]
],
test_type="requirements-based",
derivation_technique="requirements-based",
)
@pytest.mark.FullyVerifies([])
@pytest.mark.Description(
"Verifies that multiple KVS instances with different IDs store and retrieve independent values without interference."
)
@pytest.mark.TestType("requirements-based")
@pytest.mark.DerivationTechnique("requirements-based")
class TestMultipleInstanceIds(CommonScenario):
"""Verifies that multiple KVS instances with different IDs store and retrieve independent values without interference."""

@pytest.fixture(scope="class")
def scenario_name(self) -> str:
return "cit.multiple_kvs.multiple_instance_ids"
Expand Down Expand Up @@ -64,19 +63,17 @@ def test_ok(self, results: ScenarioResult, logs_info_level: LogContainer):
assert round(log2.value, 1) == 222.2


@pytest.mark.PartiallyVerifies(
[
@add_test_properties(
partially_verifies=[
"comp_req__persistency__multi_instance_v2",
"comp_req__persistency__concurrency_v2",
]
)
@pytest.mark.FullyVerifies([])
@pytest.mark.Description(
"Checks that multiple KVS instances with the same ID and key maintain consistent values across instances."
],
test_type="requirements-based",
derivation_technique="requirements-based",
)
@pytest.mark.TestType("requirements-based")
@pytest.mark.DerivationTechnique("requirements-based")
class TestSameInstanceIdSameValue(CommonScenario):
"""Checks that multiple KVS instances with the same ID and key maintain consistent values across instances."""

@pytest.fixture(scope="class")
def scenario_name(self) -> str:
return "cit.multiple_kvs.same_instance_id_same_value"
Expand All @@ -102,19 +99,17 @@ def test_ok(self, results: ScenarioResult, logs_info_level: LogContainer):
assert log1.value == log2.value


@pytest.mark.PartiallyVerifies(
[
@add_test_properties(
partially_verifies=[
"comp_req__persistency__multi_instance_v2",
"comp_req__persistency__concurrency_v2",
]
)
@pytest.mark.FullyVerifies([])
@pytest.mark.Description(
"Verifies that changes in one KVS instance with a shared ID and key are reflected in another instance, demonstrating interference."
],
test_type="requirements-based",
derivation_technique="requirements-based",
)
@pytest.mark.TestType("requirements-based")
@pytest.mark.DerivationTechnique("requirements-based")
class TestSameInstanceIdDifferentValue(CommonScenario):
"""Verifies that changes in one KVS instance with a shared ID and key are reflected in another instance, demonstrating interference."""

@pytest.fixture(scope="class")
def scenario_name(self) -> str:
return "cit.multiple_kvs.same_instance_id_diff_value"
Expand Down
13 changes: 7 additions & 6 deletions tests/python_test_cases/tests/test_cit_persistency.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,20 +15,21 @@

import pytest
from testing_utils import LogContainer, ScenarioResult
from test_properties import add_test_properties

from .common import CommonScenario, ResultCode

pytestmark = pytest.mark.parametrize("version", ["rust"], scope="class")


@pytest.mark.PartiallyVerifies([])
@pytest.mark.FullyVerifies(["comp_req__persistency__persist_data_com_v2"])
@pytest.mark.Description(
"Verifies that disabling flush on exit but manually flushing ensures data is persisted correctly."
@add_test_properties(
fully_verifies=["comp_req__persistency__persist_data_com_v2"],
test_type="requirements-based",
derivation_technique="requirements-based",
)
@pytest.mark.TestType("requirements-based")
@pytest.mark.DerivationTechnique("requirements-based")
class TestExplicitFlush(CommonScenario):
"""Verifies that disabling flush on exit but manually flushing ensures data is persisted correctly."""

NUM_VALUES = 5

@pytest.fixture(scope="class")
Expand Down
Loading
Loading