Template

Module for TIM templating logic and classes.

BaseFileObject pydantic-model

Base model for file/directory type template objects

delete: bool pydantic-field

If the template object should be deleted after instantiating the TSM

dest: str pydantic-field required

The templates destination path (relative to the containing directories destination path)

src: str pydantic-field required

The template source path (relative to the containing directory)

BaseObject pydantic-model

Base model for template configuration models

extra: Any pydantic-field

Dict that can be used to bind additional context info to a template object

name: str pydantic-field required

A descriptive name for the file template directive

dict(self, **kwargs)

Generate a dictionary representation of the model, optionally specifying which fields to include or exclude.

Source code in generator/template.py

def dict(self, **kwargs):
    # set by alias to true by default
    # to ensure that template object fields are always serialized based
    # on their alias names
    kwargs.setdefault("by_alias", True)
    return super().dict(**kwargs)

Directory pydantic-model

Directory template model for configuring TIM template directories

contents: Union[cr_kyoushi.generator.template.File, Directory] pydantic-field

List of sub directories or files

copy_: str pydantic-field

List of file globs (relative to the dir) to copy as is

File pydantic-model

File template model for configuring TIM template files

alpha(string)

Returns a string only containing letters.

Parameters:

Name	Type	Description	Default
string	str	String containing all kinds of characters.	required

Returns:

Type	Description
str	The string without any non-alpha characters.

Source code in generator/template.py

def alpha(string: str) -> str:
    """Returns a string only containing letters.

    Args:
        string: String containing all kinds of characters.

    Returns:
        The string without any non-alpha characters.
    """
    return "".join(
        [
            x
            for x in string
            if x in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
        ]
    )

create_context_environment(seed_store, generators=[], template_dirs=PosixPath('.'))

Creates the Jinja2 context for rendering the TIM context configuration.

Parameters:

Name	Type	Description	Default
seed_store	SeedStore	The seed store to use for generating PRNG seeds.	required
generators	List[cr_kyoushi.generator.plugin.Generator]	The random data generators to make available in the Jinja2 context	[]
template_dirs	Union[str, pathlib.Path, List[Union[str, pathlib.Path]]]	The Jinja2 template directory.	PosixPath('.')

Returns:

Type	Description
NativeEnvironment	Jinja2 NativeEnvironment for rendering the TIM context.

Source code in generator/template.py

def create_context_environment(
    seed_store: SeedStore,
    generators: List[Generator] = [],
    template_dirs: Union[Text, Path, List[Union[Text, Path]]] = Path("./"),
) -> NativeEnvironment:
    """Creates the Jinja2 context for rendering the TIM context configuration.

    Args:
        seed_store: The seed store to use for generating PRNG seeds.
        generators: The random data generators to make available in the Jinja2 context
        template_dirs: The Jinja2 template directory.

    Returns:
        Jinja2 NativeEnvironment for rendering the TIM context.
    """
    env = create_template_object_environment(template_dirs)
    _env_add_generators(env, seed_store, generators)

    return env

create_environment(seed_store, config, template_dirs=PosixPath('.'), generators=[])

Creates the Jinja2 context for rendering the TIM templates.

Parameters:

Name	Type	Description	Default
config	JinjaConfig	The Jinja2 configuration.	required
template_dirs	Union[str, pathlib.Path, List[Union[str, pathlib.Path]]]	The Jinja2 template directory.	PosixPath('.')
generators	List[cr_kyoushi.generator.plugin.Generator]	The random data generators to make available in the Jinja2 context	[]

Returns:

Type	Description
NativeEnvironment	Jinja2 NativeEnvironment for rendering the TIM.

Source code in generator/template.py

def create_environment(
    seed_store: SeedStore,
    config: JinjaConfig,
    template_dirs: Union[Text, Path, List[Union[Text, Path]]] = Path("./"),
    generators: List[Generator] = [],
) -> NativeEnvironment:
    """Creates the Jinja2 context for rendering the TIM templates.

    Args:
        config: The Jinja2 configuration.
        template_dirs: The Jinja2 template directory.
        generators: The random data generators to make available in the Jinja2 context

    Returns:
        Jinja2 NativeEnvironment for rendering the TIM.
    """
    env = NativeEnvironment(
        loader=FileSystemLoader(template_dirs),
        block_start_string=config.block_start,
        block_end_string=config.block_end,
        variable_start_string=config.variable_start,
        variable_end_string=config.variable_end,
        comment_start_string=config.comment_start,
        comment_end_string=config.comment_end,
        line_statement_prefix=config.line_statement,
        line_comment_prefix=config.line_comment,
        keep_trailing_newline=True,
        undefined=StrictUndefined,
        extensions=["jinja2.ext.do", "jinja2.ext.loopcontrols"],
    )
    _add_env_options(env)
    _env_add_generators(env, seed_store, generators)

    return env

create_template_object_environment(template_dirs=PosixPath('.'))

Creates the Jinja2 context for rendering the templates.yml.j2 configuration.

Parameters:

Name	Type	Description	Default
template_dirs	Union[str, pathlib.Path, List[Union[str, pathlib.Path]]]	The Jinja2 template directory.	PosixPath('.')

Returns:

Type	Description
NativeEnvironment	Jinja2 NativeEnvironment for rendering the templates configuration.

Source code in generator/template.py

def create_template_object_environment(
    template_dirs: Union[Text, Path, List[Union[Text, Path]]] = Path("./")
) -> NativeEnvironment:
    """Creates the Jinja2 context for rendering the `templates.yml.j2` configuration.

    Args:
        template_dirs: The Jinja2 template directory.

    Returns:
        Jinja2 NativeEnvironment for rendering the templates configuration.
    """
    env = NativeEnvironment(
        loader=FileSystemLoader(template_dirs),
        undefined=StrictUndefined,
        extensions=["jinja2.ext.do", "jinja2.ext.loopcontrols"],
    )
    _add_env_options(env)
    return env

get_max_hosts(mask)

Returns the number of maximum hosts to use.

Parameters:

Name	Type	Description	Default
mask	int	network mask.	required

Returns:

Type	Description
int	An integer representing the maximum amount of hosts.

Source code in generator/template.py

def get_max_hosts(mask: int) -> int:
    """Returns the number of maximum hosts to use.

    Args:
        mask: network mask.

    Returns:
        An integer representing the maximum amount of hosts.
    """
    host_bits = 32 - mask
    return 2**host_bits - 2

get_yaml()

Utility function for creating a YAML parser and serializer.

Returns:

Type	Description
YAML	A ruamel.yaml.YAML object

Source code in generator/template.py

def get_yaml() -> YAML:
    """Utility function for creating a YAML parser and serializer.

    Returns:
        A ruamel.yaml.YAML object
    """
    yaml = YAML(typ="safe")
    for g in [File, Directory]:
        yaml.register_class(g)
    return yaml

normalize_probabilities_map(container, ignore=['extra'], skip=[])

Normalizes a dictionary of distributions.

Parameters:

Name	Type	Description	Default
container	Dict[str, Union[Dict[str, float], List[float]]]	Dictionary containing multiple distributions	required
ignore	List[str]	Distribution dictionary keys to ignore during normalization	['extra']
skip	List[str]	Dictionary keys to ignore (i.e., incase the given dict contains keys which are not distributions.)	[]

Exceptions:

Type	Description
Exception	If the given distributions have an invalid container type, sum to 0 or contain negative numbers.

Returns:

Type	Description
Dict[str, Union[Dict[str, float], List[float]]]	The dictionary with all its distributions normalized.

Source code in generator/template.py

def normalize_probabilities_map(
    container: Dict[str, Union[Dict[str, float], List[float]]],
    ignore: List[str] = ["extra"],
    skip: List[str] = [],
) -> Dict[str, Union[Dict[str, float], List[float]]]:
    """Normalizes a dictionary of distributions.

    Args:
        container: Dictionary containing multiple distributions
        ignore: Distribution dictionary keys to ignore during normalization
        skip: Dictionary keys to ignore (i.e., incase the given dict contains keys
                                    which are not distributions.)

    Raises:
        Exception: If the given distributions have an invalid container type, sum to 0
                   or contain negative numbers.

    Returns:
        The dictionary with all its distributions normalized.
    """
    if isinstance(container, dict):
        # normalize each distribution
        for k, v in container.items():
            # but do nothing with those in the skip list
            if k not in skip:
                container[k] = normalize_propabilities(v, ignore=ignore)
        return container
    raise Exception(f"Normalize map target must be a dictionary, but got '{container}'")

normalize_propabilities(propabilities, ignore=['extra'])

Accepts probability distribution dicts and lists and normalizes them.

Parameters:

Name	Type	Description	Default
propabilities	Union[Dict[str, float], List[float]]	A probability distribution as dict or list	required
ignore	List[str]	Additional dict keys to ignore during normalization.	['extra']

Exceptions:

Type	Description
Exception	If the given distribution has an invalid container type, sums to 0 or contains negative numbers.

Returns:

Type	Description
Union[Dict[str, float], List[float]]	The normalized distribution.

Source code in generator/template.py

def normalize_propabilities(
    propabilities: Union[Dict[str, float], List[float]], ignore: List[str] = ["extra"]
) -> Union[Dict[str, float], List[float]]:
    """Accepts probability distribution dicts and lists and normalizes them.

    Args:
        propabilities: A probability distribution as dict or list
        ignore: Additional dict keys to ignore during normalization.

    Raises:
        Exception: If the given distribution has an invalid container type, sums to 0
                   or contains negative numbers.

    Returns:
        The normalized distribution.
    """
    if isinstance(propabilities, dict):
        keys = []
        values = []

        # only consider non ignored fields as part of the distribution
        for k, p in propabilities.items():
            if k not in ignore:
                keys.append(k)
                values.append(p)

        # normalize the probabilities
        values = _normalize_propabilities(values)

        # the order is preserved so we can simply zip them
        # back together
        norm = dict(zip(keys, values))

        # add the ignored fields back
        for e in ignore:
            if e in propabilities:
                norm[e] = propabilities[e]

        return norm

    elif isinstance(propabilities, list):
        # ensure we work on a copy (we don't want to modify the original)
        propabilities = list(propabilities)
        return _normalize_propabilities(propabilities)

    raise Exception(
        f"Normalize target must be a dictionary or list, but got '{propabilities}'"
    )

render_template(env, template, context)

Utility function for rendering Jinja2 text or file templates.

Parameters:

Name	Type	Description	Default
env	NativeEnvironment	The Jinja2 environment to use for rendering	required
template	Union[str, pathlib.Path]	The template string or file to render	required
context	Any	The context variables to use for rendering	required

Returns:

Type	Description
Any	The rendered template string or data structure

Source code in generator/template.py

def render_template(
    env: NativeEnvironment,
    template: Union[Text, Path],
    context: Any,
) -> Any:
    """Utility function for rendering Jinja2 text or file templates.

    Args:
        env: The Jinja2 environment to use for rendering
        template: The template string or file to render
        context: The context variables to use for rendering

    Returns:
        The rendered template string or data structure
    """
    # convert strings to template
    if isinstance(template, Path):
        _template = env.get_template(str(template))
    else:
        _template = env.from_string(template)

    value = _template.render(**context)
    if isinstance(value, Undefined):
        value._fail_with_undefined_error()
    return value

render_tim(env, object_list, src_dir, dest_dir, inputs, global_context, parent_context={}, delete_dirs=None, delete_files=None)

Function for rendering a TIM using the context and template configuration.

Parameters:

Name	Type	Description	Default
env	NativeEnvironment	The TIM Jinja2 environment.	required
object_list	List[Union[cr_kyoushi.generator.template.File, cr_kyoushi.generator.template.Directory]]	The list of template objects to render for the TIM.	required
src_dir	Path	The current source directory for template objects.	required
dest_dir	Path	The current destination directory for rendered templates.	required
global_context	Dict[str, Any]	The global TIM context.	required
parent_context	Dict[str, Any]	The TIM context specific to the parent template objects.	{}
delete_dirs	Optional[Deque[pathlib.Path]]	LiFo queue of directories to delete at the end of render process.	None
delete_files	Optional[Deque[pathlib.Path]]	LiFo queue of files to delete at the end of render process.	None

Exceptions:

Type	Description
NotImplementedError	If the raw object_list contains an unknown template object type

Returns:

Type	Description
Tuple[Deque[pathlib.Path], Deque[pathlib.Path]]	The final delete_dirs and delete_files queues as tuple (delete_dirs, delete_files)

Source code in generator/template.py

def render_tim(
    env: NativeEnvironment,
    object_list: List[Union[File, Directory]],
    src_dir: Path,
    dest_dir: Path,
    inputs: Dict[str, Any],
    global_context: Dict[str, Any],
    parent_context: Dict[str, Any] = {},
    delete_dirs: Optional[Deque[Path]] = None,
    delete_files: Optional[Deque[Path]] = None,
) -> Tuple[Deque[Path], Deque[Path]]:
    """Function for rendering a TIM using the context and template configuration.

    Args:
        env: The TIM Jinja2 environment.
        object_list: The list of template objects to render for the TIM.
        src_dir: The current source directory for template objects.
        dest_dir: The current destination directory for rendered templates.
        global_context: The global TIM context.
        parent_context: The TIM context specific to the parent template objects.
        delete_dirs: LiFo queue of directories to delete at the end of render process.
        delete_files: LiFo queue of files to delete at the end of render process.

    Raises:
        NotImplementedError: If the raw object_list contains an unknown template object type

    Returns:
        The final `delete_dirs` and `delete_files` queues as tuple `(delete_dirs, delete_files)`
    """

    if delete_dirs is None:
        delete_dirs = deque()
    if delete_files is None:
        delete_files = deque()

    for obj in object_list:
        src: Path = src_dir.joinpath(obj.src)
        dest: Path = dest_dir.joinpath(obj.dest)
        if isinstance(obj, Directory):
            # create the directory
            dest.mkdir(parents=True, exist_ok=True)

            # handle the copy configuration
            # i.e., resolve all globs and copy them to the destination
            for glb in obj.copy_:
                for copy_src in src.glob(glb):
                    copy_src_relative = copy_src.relative_to(src)
                    copy_dest = dest.joinpath(copy_src_relative)
                    # make sure the containing directory exists (might be needed if glob is for some sub dir)
                    copy_dest.parent.mkdir(parents=True, exist_ok=True)
                    shutil.copy(copy_src, copy_dest)

            # handle all sub template objects (dirs and files)
            new_parent_context = parent_context.copy()
            new_parent_context.update(obj.extra)
            if obj.delete and src not in delete_files:
                delete_dirs.append(src)
            render_tim(
                env,
                obj.contents,
                src,
                dest,
                inputs,
                global_context,
                new_parent_context,
                delete_dirs,
                delete_files,
            )

        elif isinstance(obj, File):
            context = {
                "inputs": inputs,
                "context": global_context,
                "parent_context": parent_context,
                "local_context": obj.extra,
            }
            if obj.delete and src not in delete_files:
                delete_files.append(src)
            write_template(env, src, dest, context)
        else:
            raise NotImplementedError()

    return (delete_dirs, delete_files)

validate_object_list(object_list)

Utility function for validating template object model lists.

Parameters:

Name	Type	Description	Default
object_list		List of unvalidated template object models.	required

Returns:

Type	Description
List[Union[cr_kyoushi.generator.template.File, cr_kyoushi.generator.template.Directory]]	A validated list of template object models converted to the correct Python classes.

Source code in generator/template.py

def validate_object_list(object_list) -> List[Union[File, Directory]]:
    """Utility function for validating template object model lists.

    Args:
        object_list: List of unvalidated template object models.

    Returns:
        A validated list of template object models converted to the correct
        Python classes.
    """
    return parse_obj_as(
        Annotated[List[Union[File, Directory]], Field()],  # type: ignore[arg-type]
        object_list,
    )

write_template(env, src, dest, context)

Utility function for rendering and writing a template file.

Parameters:

Name	Type	Description	Default
env	NativeEnvironment	The Jinja2 environment to use for rendering	required
src	Path	The template file path	required
dest	Path	The path to write the rendered template to	required
context	Any	The context variables to use for rendering	required

Source code in generator/template.py

def write_template(env: NativeEnvironment, src: Path, dest: Path, context: Any):
    """Utility function for rendering and writing a template file.

    Args:
        env: The Jinja2 environment to use for rendering
        src: The template file path
        dest: The path to write the rendered template to
        context: The context variables to use for rendering
    """
    template_rendered = render_template(env, src, context)
    with open(dest, "w") as f:
        # mappings and lists are output as yaml files
        if isinstance(template_rendered, Mapping) or (
            # need to exclude str types since they are also sequences
            not isinstance(template_rendered, Text)
            and isinstance(template_rendered, Sequence)
        ):
            yaml = YAML(typ="safe")
            yaml.dump(template_rendered, f)
        else:
            f.write(str(template_rendered))