generator

CombinationGenerator(*discrete_domains)

Bases: TestcaseGenerator

A generator that produces tests based on combination of ranges.

This generator creates a test case for each combination of the provided value ranges. Each value range must be associated with exactly one input feature of the model that shall be tested.

Initialize the combination generator.

Parameters:

Name	Type	Description	Default
discrete_domains	Discrete	A list of discrete domains to generate test cases from. Each domain must be associated with exactly one input feature of the model that shall be tested.	()

Source code in src/flowcean/testing/generator/combination_generator.py

def __init__(self, *discrete_domains: Discrete) -> None:
    """Initialize the combination generator.

    Args:
        discrete_domains: A list of discrete domains to generate test cases
            from. Each domain must be associated with exactly one input
            feature of the model that shall be tested.
    """
    super().__init__()

    # Check if no duplicate feature names are present
    feature_names_count: dict[str, int] = {}
    for domain in discrete_domains:
        feature_names_count[domain.feature_name] = (
            feature_names_count.get(
                domain.feature_name,
                0,
            )
            + 1
        )
    if len(feature_names_count) != len(discrete_domains):
        msg = "Duplicate features found: "
        msg += ", ".join(
            f"{feature_name}: {count}"
            for feature_name, count in feature_names_count.items()
            if count > 1
        )
        raise ValueError(msg)

    self.domains = discrete_domains

    # Calculate the number of test cases
    self.number_test_cases = functools.reduce(
        lambda n, range_: n * len(range_),
        self.domains,
        1,
    )

    # Call reset to initialize the generator
    self.reset()

reset()

Reset the generator to the initial state.

Source code in src/flowcean/testing/generator/combination_generator.py

def reset(self) -> None:
    """Reset the generator to the initial state."""
    # Build the product iterator
    self.product_iterator = itertools.product(
        *self.domains,
    )
    # Perform a first step to initialize the data
    self.step()

DDTIGenerator(model, *, n_testinputs, test_coverage_criterium, dataset=None, specs_file=None, classification=False, inverse_alloc=False, epsilon=0.5, performance_threshold=0.3, sample_limit=50000, n_predictions=50, max_depth=5, hoeffding_tree_extra_params=None)

Bases: TestcaseGenerator

Generates test inputs considering decision boundaries.

Methods:

save_hoeffding_tree() Saves the generated Hoeffding tree to a file.

print_eqclasses() Prints the equivalence classes and their test input counts.

print_testplans() Prints the test plans (intervals used to sample test inputs).

print_hoeffding_tree() Prints the Hoeffding tree structure as a PNG.

Initialize the stochastic generator.

Parameters:

Name	Type	Description	Default
model	Model	The trained Flowcean model.	required
n_testinputs	int	Number of test inputs to generate.	required
test_coverage_criterium	str	Test coverage strategy identifier.	required
dataset	DataFrame | None	Polars DataFrame containing the original dataset. Either the dataset or the specs_file must be provided.	None
specs_file	Path | None	Path to a file containing feature specifications. If you provide a dataset containing system inputs and outputs that already encodes the necessary specifications, then you do not need to supply a separate system specification file.	None
classification	bool	Whether the task is a classification problem.	False
inverse_alloc	bool	If True, allocate more tests to lower-priority equivalence classes.	False
epsilon	float	Interval offset used for boundary value analysis.	0.5
performance_threshold	float	Minimum performance needed before exporting the Hoeffding Tree.	0.3
sample_limit	int	Maximum number of samples used to train the Hoeffding Tree.	50000
n_predictions	int	Number of consecutive correct predictions needed before exporting the Hoeffding Tree.	50
max_depth	int	Maximum depth of the Hoeffding Tree.	5
hoeffding_tree_extra_params	dict[str, Any] | None	Extra keyword arguments forwarded to the Hoeffding Tree trainer.	None

Source code in src/flowcean/testing/generator/ddti_generator.py

def __init__(
    self,
    model: Model,
    *,
    n_testinputs: int,
    test_coverage_criterium: str,
    dataset: pl.DataFrame | None = None,
    specs_file: Path | None = None,
    classification: bool = False,
    inverse_alloc: bool = False,
    epsilon: float = 0.5,
    performance_threshold: float = 0.3,
    sample_limit: int = 50000,
    n_predictions: int = 50,
    max_depth: int = 5,
    hoeffding_tree_extra_params: dict[str, Any] | None = None,
) -> None:
    """Initialize the stochastic generator.

    Args:
        model: The trained Flowcean model.
        n_testinputs: Number of test inputs to generate.
        test_coverage_criterium: Test coverage strategy identifier.
        dataset: Polars DataFrame containing the original dataset.
            Either the dataset or the specs_file must be provided.
        specs_file: Path to a file containing feature specifications.
            If you provide a dataset containing system inputs and
            outputs that already encodes the necessary specifications,
            then you do not need to supply a separate system
            specification file.
        classification: Whether the task is a classification problem.
        inverse_alloc: If True, allocate more tests to lower-priority
            equivalence classes.
        epsilon: Interval offset used for boundary value analysis.
        performance_threshold: Minimum performance needed before
            exporting the Hoeffding Tree.
        sample_limit: Maximum number of samples used to train the
            Hoeffding Tree.
        n_predictions: Number of consecutive correct predictions needed
            before exporting the Hoeffding Tree.
        max_depth: Maximum depth of the Hoeffding Tree.
        hoeffding_tree_extra_params: Extra keyword arguments forwarded
            to the Hoeffding Tree trainer.
    """
    super().__init__()
    self.n_testinputs = n_testinputs
    self.seed = get_seed()

    self.test_pipeline = TestPipeline(
        model,
        dataset=dataset,
        specs_file=specs_file,
        classification=classification,
        n_testinputs=self.n_testinputs,
        test_coverage_criterium=test_coverage_criterium,
        inverse_alloc=inverse_alloc,
        epsilon=epsilon,
        seed=self.seed,
        performance_threshold=performance_threshold,
        sample_limit=sample_limit,
        n_predictions=n_predictions,
        max_depth=max_depth,
        hoeffding_tree_extra_params=hoeffding_tree_extra_params,
    )
    self.data = DataFrame(self.test_pipeline.execute())
    self.reset()

TestcaseGenerator

Bases: IncrementalEnvironment

A generator that produces test cases for a model.

reset() abstractmethod

Reset the generator to its initial state.

Source code in src/flowcean/testing/generator/generator.py

@abstractmethod
def reset(self) -> None:
    """Reset the generator to its initial state."""

save_csv(path, *, test_case_count=None, separator=',')

Save the generated test cases to a CSV file.

Parameters:

Name	Type	Description	Default
path	str | Path	The path where the CSV file should be saved. If the path does not have a suffix, '.csv' will be added.	required
test_case_count	int | None	The number of test cases to save. If None, all available test cases will be saved. If the number of test cases is not defined, a ValueError will be raised.	None
separator	str	The value separator to use in the CSV file. Defaults to ','.	','

Source code in src/flowcean/testing/generator/generator.py

def save_csv(
    self,
    path: str | Path,
    *,
    test_case_count: int | None = None,
    separator: str = ",",
) -> None:
    """Save the generated test cases to a CSV file.

    Args:
        path: The path where the CSV file should be saved.
            If the path does not have a suffix, '.csv' will be added.
        test_case_count: The number of test cases to save. If None, all
            available test cases will be saved. If the number of test cases
            is not defined, a ValueError will be raised.
        separator: The value separator to use in the CSV file.
            Defaults to ','.
    """
    path = Path(path)
    if not path.suffix:
        path = path.with_suffix(".csv")

    # Collect test cases and save them to a CSV file
    df = self.__collect_to_df(test_case_count)

    df.data.sink_csv(
        path,
        separator=separator,
        engine="streaming",
        include_header=True,
    )

save_excel(path, *, test_case_count=None, worksheet_name='Test Cases')

Save the generated test cases to an Excel file.

Parameters:

Name	Type	Description	Default
path	str | Path	The path where the Excel file should be saved. If the path does not have a suffix, '.xlsx' will be added.	required
test_case_count	int | None	The number of test cases to save. If None, all available test cases will be saved. If the number of test cases is not defined, a ValueError will be raised.	None
worksheet_name	str	The name of the worksheet in the Excel file. Defaults to 'Test Cases'.	'Test Cases'

Source code in src/flowcean/testing/generator/generator.py

def save_excel(
    self,
    path: str | Path,
    *,
    test_case_count: int | None = None,
    worksheet_name: str = "Test Cases",
) -> None:
    """Save the generated test cases to an Excel file.

    Args:
        path: The path where the Excel file should be saved.
            If the path does not have a suffix, '.xlsx' will be added.
        test_case_count: The number of test cases to save. If None, all
            available test cases will be saved. If the number of test cases
            is not defined, a ValueError will be raised.
        worksheet_name: The name of the worksheet in the Excel file.
            Defaults to 'Test Cases'.
    """
    path = Path(path)
    if not path.suffix:
        path = path.with_suffix(".xlsx")

    # Collect test cases and save them to a XLSX file
    df = self.__collect_to_df(test_case_count)

    df.data.collect(engine="streaming").write_excel(
        workbook=path,
        worksheet=worksheet_name,
        include_header=True,
    )

__collect_to_df(n)

Collect the first n test cases and return them as a DataFrame.

Source code in src/flowcean/testing/generator/generator.py

def __collect_to_df(self, n: int | None) -> DataFrame:
    """Collect the first n test cases and return them as a DataFrame."""
    # Make sure the number of test cases is defined
    if n is None and self.num_steps() is None:
        msg = (
            "Cannot save test cases to file without a defined "
            "number of cases."
        )
        raise ValueError(msg)

    n = min(
        n or sys.maxsize,
        self.num_steps() or sys.maxsize,
    )

    # Collect the data from the generator
    return collect(self, n)

StochasticGenerator(domains, *, test_case_count=None, seed=0)

Bases: TestcaseGenerator

A generator that produces random tests based on given domains.

Initialize the stochastic generator.

Parameters:

Name	Type	Description	Default
domains	list[Domain]	A list of domains to generate random values for. Each domain must be associated with exactly one input feature of the model that shall be tested.	required
test_case_count	int | None	The number of test cases to generate. If None, the generator will run indefinitely.	None
seed	int	The seed for the random number generator. The default is 0, which means a random seed will be used.	0

Source code in src/flowcean/testing/generator/stochastic_generator.py

def __init__(
    self,
    domains: list[Domain],
    *,
    test_case_count: int | None = None,
    seed: int = 0,
) -> None:
    """Initialize the stochastic generator.

    Args:
        domains: A list of domains to generate random values for.
            Each domain must be associated with exactly one input
            feature of the model that shall be tested.
        test_case_count: The number of test cases to generate. If None,
            the generator will run indefinitely.
        seed: The seed for the random number generator. The default is 0,
            which means a random seed will be used.
    """
    super().__init__()

    # Check if no duplicate feature names are present
    feature_names_count: dict[str, int] = {}
    for domain in domains:
        feature_names_count[domain.feature_name] = (
            feature_names_count.get(
                domain.feature_name,
                0,
            )
            + 1
        )
    if len(feature_names_count) != len(domains):
        msg = "Duplicate features found: "
        msg += ", ".join(
            f"{feature_name}: {count}"
            for feature_name, count in feature_names_count.items()
            if count > 1
        )
        raise ValueError(msg)

    # Seed all domains
    rng = random.Random(seed) if seed != 0 else random.Random()
    for domain in domains:
        domain.set_seed(rng.randint(0, 2**32 - 1))

    self.domains = domains
    self.count = 0
    self.number_test_cases = test_case_count
    # Perform the first step to initialize the generator
    self.step()

reset()

Reset the generator to its initial state.

Source code in src/flowcean/testing/generator/stochastic_generator.py

def reset(self) -> None:
    """Reset the generator to its initial state."""
    self.count = 0
    self.step()