golden hour

/opt/saltstack/salt/lib/python3.10/site-packages/salt/utils

File/Folder	Size	Actions
__init__.py	237 B
__pycache__	-
aggregation.py	5.17 KB
ansible.py	1.48 KB
args.py	18.33 KB
asynchronous.py	4.06 KB
atomicfile.py	5.33 KB
aws.py	20.37 KB
azurearm.py	11.42 KB
beacons.py	517 B
boto3mod.py	8.36 KB
boto_elb_tag.py	3.02 KB
botomod.py	7.98 KB
cache.py	11.49 KB
channel.py	489 B
cloud.py	116.3 KB
color.py	2.72 KB
compat.py	1.89 KB
configcomparer.py	3.88 KB
configparser.py	10.82 KB
context.py	6.8 KB
crypt.py	5 KB
ctx.py	1.42 KB
data.py	53.03 KB
dateutils.py	2.3 KB
debug.py	4.19 KB
decorators	-
dictdiffer.py	16.48 KB
dicttrim.py	3.9 KB
dictupdate.py	11.33 KB
dns.py	35.21 KB
doc.py	2.25 KB
dockermod	-
entrypoints.py	1.83 KB
environment.py	2.2 KB
error.py	1.18 KB
etcd_util.py	33.25 KB
event.py	52.45 KB
extend.py	8.87 KB
extmods.py	6.04 KB
filebuffer.py	3.15 KB
files.py	27.94 KB
find.py	22.08 KB
fsutils.py	3.29 KB
functools.py	6.02 KB
gitfs.py	130.41 KB
github.py	1.52 KB
gzip_util.py	2.86 KB
hashutils.py	5.91 KB
http.py	33.9 KB
iam.py	1.22 KB
icinga2.py	754 B
idem.py	1.22 KB
immutabletypes.py	2.46 KB
itertools.py	2.36 KB
jid.py	3 KB
jinja.py	33.92 KB
job.py	6.89 KB
json.py	3.78 KB
kickstart.py	41.04 KB
kinds.py	493 B
lazy.py	3.06 KB
listdiffer.py	10.9 KB
locales.py	2.06 KB
mac_utils.py	14.01 KB
mako.py	3.97 KB
master.py	29.7 KB
mattermost.py	1.77 KB
memcached.py	3.56 KB
migrations.py	1.46 KB
mine.py	3.68 KB
minion.py	4.13 KB
minions.py	43.38 KB
mount.py	1.15 KB
msazure.py	5.36 KB
msgpack.py	4.69 KB
nacl.py	13.65 KB
namecheap.py	4.32 KB
napalm.py	23.22 KB
nb_popen.py	7.24 KB
network.py	73.86 KB
nxos.py	12.94 KB
nxos_api.py	4 KB
odict.py	13.21 KB
openstack	-
oset.py	6.41 KB
pagerduty.py	3.03 KB
parsers.py	122.69 KB
path.py	11.24 KB
pbm.py	9.81 KB
pkg	-
platform.py	5.84 KB
powershell.py	4.15 KB
preseed.py	2.64 KB
process.py	40.76 KB
profile.py	3.21 KB
proxy.py	331 B
psutil_compat.py	3.63 KB
pushover.py	4.51 KB
pycrypto.py	5.41 KB
pydsl.py	13.74 KB
pyobjects.py	10.75 KB
reactor.py	18.99 KB
reclass.py	752 B
roster_matcher.py	3.55 KB
rsax931.py	8.42 KB
s3.py	8.78 KB
saltclass.py	14.27 KB
sanitizers.py	2.51 KB
schedule.py	71.81 KB
schema.py	54.26 KB
sdb.py	4.04 KB
slack.py	3.58 KB
smb.py	11.16 KB
smtp.py	3.27 KB
ssdp.py	14.75 KB
ssh.py	769 B
state.py	8.43 KB
stringio.py	355 B
stringutils.py	16.95 KB
systemd.py	5.51 KB
templates.py	24.03 KB
textformat.py	5.03 KB
thin.py	31.91 KB
timed_subprocess.py	4.06 KB
timeout.py	1.53 KB
timeutil.py	2.4 KB
url.py	5 KB
user.py	11.86 KB
validate	-
value.py	247 B
vault.py	21.74 KB
verify.py	25.34 KB
versions.py	17.17 KB
virt.py	3.24 KB
virtualbox.py	22.43 KB
vmware.py	129.74 KB
vsan.py	17.18 KB
vt.py	31.47 KB
vt_helper.py	4.4 KB
win_chcp.py	3.7 KB
win_dacl.py	95.49 KB
win_dotnet.py	4.74 KB
win_functions.py	12.69 KB
win_lgpo_auditpol.py	8.48 KB
win_lgpo_netsh.py	17.87 KB
win_lgpo_reg.py	16.98 KB
win_network.py	16.35 KB
win_osinfo.py	2.83 KB
win_pdh.py	13.85 KB
win_reg.py	30.82 KB
win_runas.py	10.53 KB
win_service.py	5.2 KB
win_system.py	14.47 KB
win_update.py	40.36 KB
winapi.py	818 B
x509.py	73.19 KB
xdg.py	316 B
xmlutil.py	13.91 KB
yaml.py	349 B
yamldumper.py	3.37 KB
yamlencoding.py	1.55 KB
yamllint.py	1.61 KB
yamlloader.py	6.04 KB
yamlloader_old.py	8.15 KB
yast.py	619 B
zeromq.py	1.74 KB
zfs.py	19.15 KB

Edit: saltclass.py

import glob
import logging
import os
import re

from jinja2 import Environment, FileSystemLoader

import salt.utils.path
import salt.utils.yaml

log = logging.getLogger(__name__)

# Renders jinja from a template file
def render_jinja(_file, salt_data):
    j_env = Environment(loader=FileSystemLoader(os.path.dirname(_file)))
    j_env.globals.update(
        {
            "__opts__": salt_data["__opts__"],
            "__salt__": salt_data["__salt__"],
            "__grains__": salt_data["__grains__"],
            "__pillar__": salt_data["__pillar__"],
            "minion_id": salt_data["minion_id"],
        }
    )
    j_render = j_env.get_template(os.path.basename(_file)).render()
    return j_render

# Renders yaml from rendered jinja
def render_yaml(_file, salt_data):
    return salt.utils.yaml.safe_load(render_jinja(_file, salt_data))

# Returns a dict from a class yaml definition
def get_class(_class, salt_data):
    l_files = []
    saltclass_path = salt_data["path"]

straight, sub_init, sub_straight = get_class_paths(_class, saltclass_path)

for root, dirs, files in salt.utils.path.os_walk(
        os.path.join(saltclass_path, "classes"), followlinks=True
    ):
        for l_file in files:
            l_files.append(os.path.join(root, l_file))

if straight in l_files:
        return render_yaml(straight, salt_data)

if sub_straight in l_files:
        return render_yaml(sub_straight, salt_data)

if sub_init in l_files:
        return render_yaml(sub_init, salt_data)

log.warning("%s: Class definition not found", _class)
    return {}

def get_class_paths(_class, saltclass_path):
    """
    Converts the dotted notation of a saltclass class to its possible file counterparts.

:param str _class: Dotted notation of the class
    :param str saltclass_path: Root to saltclass storage
    :return: 3-tuple of possible file counterparts
    :rtype: tuple(str)
    """
    straight = os.path.join(saltclass_path, "classes", "{}.yml".format(_class))
    sub_straight = os.path.join(
        saltclass_path, "classes", "{}.yml".format(_class.replace(".", os.sep))
    )
    sub_init = os.path.join(
        saltclass_path, "classes", _class.replace(".", os.sep), "init.yml"
    )
    return straight, sub_init, sub_straight

def get_class_from_file(_file, saltclass_path):
    """
    Converts the absolute path to a saltclass file back to the dotted notation.

.. code-block:: python

print(get_class_from_file('/srv/saltclass/classes/services/nginx/init.yml', '/srv/saltclass'))
       # services.nginx

:param str _file: Absolute path to file
    :param str saltclass_path: Root to saltclass storage
    :return: class name in dotted notation
    :rtype: str
    """
    # remove classes path prefix
    _file = _file[len(os.path.join(saltclass_path, "classes")) + len(os.sep) :]
    # remove .yml extension
    _file = _file[:-4]
    # revert to dotted notation
    _file = _file.replace(os.sep, ".")
    # remove tailing init
    if _file.endswith(".init"):
        _file = _file[:-5]
    return _file

# Return environment
def get_env_from_dict(exp_dict_list):
    environment = ""
    for s_class in exp_dict_list:
        if "environment" in s_class:
            environment = s_class["environment"]
    return environment

# Merge dict b into a
def dict_merge(a, b, path=None):
    if path is None:
        path = []

for key in b:
        if key in a:
            if isinstance(a[key], list) and isinstance(b[key], list):
                if b[key][0] == "^":
                    b[key].pop(0)
                    a[key] = b[key]
                else:
                    a[key].extend(b[key])
            elif isinstance(a[key], dict) and isinstance(b[key], dict):
                dict_merge(a[key], b[key], path + [str(key)])
            elif a[key] == b[key]:
                pass
            else:
                a[key] = b[key]
        else:
            a[key] = b[key]
    return a

# Recursive search and replace in a dict
def dict_search_and_replace(d, old, new, expanded):
    for (k, v) in d.items():
        if isinstance(v, dict):
            dict_search_and_replace(d[k], old, new, expanded)

if isinstance(v, list):
            x = 0
            for i in v:
                if isinstance(i, dict):
                    dict_search_and_replace(v[x], old, new, expanded)
                if isinstance(i, str):
                    if i == old:
                        v[x] = new
                x = x + 1

if v == old:
            d[k] = new

return d

# Retrieve original value from ${xx:yy:zz} to be expanded
def find_value_to_expand(x, v):
    a = x
    for i in v[2:-1].split(":"):
        if a is None:
            return v
        if i in a:
            a = a.get(i)
        else:
            return v
    return a

# Look for regexes and expand them
def find_and_process_re(_str, v, k, b, expanded):
    vre = re.finditer(r"(^|.)\$\{.*?\}", _str)
    if vre:
        for re_v in vre:
            re_str = str(re_v.group())
            if re_str.startswith("\\"):
                v_new = _str.replace(re_str, re_str.lstrip("\\"))
                b = dict_search_and_replace(b, _str, v_new, expanded)
                expanded.append(k)
            elif not re_str.startswith("$"):
                v_expanded = find_value_to_expand(b, re_str[1:])
                v_new = _str.replace(re_str[1:], v_expanded)
                b = dict_search_and_replace(b, _str, v_new, expanded)
                _str = v_new
                expanded.append(k)
            else:
                v_expanded = find_value_to_expand(b, re_str)
                if isinstance(v, str):
                    v_new = v.replace(re_str, v_expanded)
                else:
                    v_new = _str.replace(re_str, v_expanded)
                b = dict_search_and_replace(b, _str, v_new, expanded)
                _str = v_new
                v = v_new
                expanded.append(k)
    return b

# Return a dict that contains expanded variables if found
def expand_variables(a, b, expanded, path=None):
    if path is None:
        b = a.copy()
        path = []

for (k, v) in a.items():
        if isinstance(v, dict):
            expand_variables(v, b, expanded, path + [str(k)])
        else:
            if isinstance(v, list):
                for i in v:
                    if isinstance(i, dict):
                        expand_variables(i, b, expanded, path + [str(k)])
                    if isinstance(i, str):
                        b = find_and_process_re(i, v, k, b, expanded)

if isinstance(v, str):
                b = find_and_process_re(v, v, k, b, expanded)
    return b

def match_class_glob(_class, saltclass_path):
    """
    Takes a class name possibly including `*` or `?` wildcards (or any other wildcards supportet by `glob.glob`) and
    returns a list of expanded class names without wildcards.

.. code-block:: python

classes = match_class_glob('services.*', '/srv/saltclass')
       print(classes)
       # services.mariadb
       # services.nginx...

:param str _class: dotted class name, globbing allowed.
    :param str saltclass_path: path to the saltclass root directory.

:return: The list of expanded class matches.
    :rtype: list(str)
    """
    straight, sub_init, sub_straight = get_class_paths(_class, saltclass_path)
    classes = []
    matches = []
    matches.extend(glob.glob(straight))
    matches.extend(glob.glob(sub_straight))
    matches.extend(glob.glob(sub_init))
    if not matches:
        log.warning("%s: Class globbing did not yield any results", _class)
    for match in matches:
        classes.append(get_class_from_file(match, saltclass_path))
    return classes

def expand_classes_glob(classes, salt_data):
    """
    Expand the list of `classes` to no longer include any globbing.

:param iterable(str) classes: Iterable of classes
    :param dict salt_data: configuration data
    :return: Expanded list of classes with resolved globbing
    :rtype: list(str)
    """
    all_classes = []
    expanded_classes = []
    saltclass_path = salt_data["path"]

for _class in classes:
        all_classes.extend(match_class_glob(_class, saltclass_path))

for _class in all_classes:
        if _class not in expanded_classes:
            expanded_classes.append(_class)

return expanded_classes

def expand_classes_in_order(
    minion_dict, salt_data, seen_classes, expanded_classes, classes_to_expand
):
    # Get classes to expand from minion dictionary
    if not classes_to_expand and "classes" in minion_dict:
        classes_to_expand = minion_dict["classes"]

classes_to_expand = expand_classes_glob(classes_to_expand, salt_data)

# Now loop on list to recursively expand them
    for klass in classes_to_expand:
        if klass not in seen_classes:
            seen_classes.append(klass)
            expanded_classes[klass] = get_class(klass, salt_data)
            # Fix corner case where class is loaded but doesn't contain anything
            if expanded_classes[klass] is None:
                expanded_classes[klass] = {}

# Merge newly found pillars into existing ones
            new_pillars = expanded_classes[klass].get("pillars", {})
            if new_pillars:
                dict_merge(salt_data["__pillar__"], new_pillars)

# Now replace class element in classes_to_expand by expansion
            if expanded_classes[klass].get("classes"):
                l_id = classes_to_expand.index(klass)
                classes_to_expand[l_id:l_id] = expanded_classes[klass]["classes"]
                expand_classes_in_order(
                    minion_dict,
                    salt_data,
                    seen_classes,
                    expanded_classes,
                    classes_to_expand,
                )
            else:
                expand_classes_in_order(
                    minion_dict,
                    salt_data,
                    seen_classes,
                    expanded_classes,
                    classes_to_expand,
                )

# We may have duplicates here and we want to remove them
    tmp = []
    for t_element in classes_to_expand:
        if t_element not in tmp:
            tmp.append(t_element)

classes_to_expand = tmp

# Now that we've retrieved every class in order,
    # let's return an ordered list of dicts
    ord_expanded_classes = []
    ord_expanded_states = []
    for ord_klass in classes_to_expand:
        ord_expanded_classes.append(expanded_classes[ord_klass])
        # And be smart and sort out states list
        # Address the corner case where states is empty in a class definition
        if (
            "states" in expanded_classes[ord_klass]
            and expanded_classes[ord_klass]["states"] is None
        ):
            expanded_classes[ord_klass]["states"] = {}

if "states" in expanded_classes[ord_klass]:
            ord_expanded_states.extend(expanded_classes[ord_klass]["states"])

# Add our minion dict as final element but check if we have states to process
    if "states" in minion_dict and minion_dict["states"] is None:
        minion_dict["states"] = []

if "states" in minion_dict:
        ord_expanded_states.extend(minion_dict["states"])

ord_expanded_classes.append(minion_dict)

return ord_expanded_classes, classes_to_expand, ord_expanded_states

def expanded_dict_from_minion(minion_id, salt_data):
    _file = ""
    saltclass_path = salt_data["path"]
    # Start
    for root, dirs, files in salt.utils.path.os_walk(
        os.path.join(saltclass_path, "nodes"), followlinks=True
    ):
        for minion_file in files:
            if minion_file == "{}.yml".format(minion_id):
                _file = os.path.join(root, minion_file)

# Load the minion_id definition if existing, else an empty dict
    node_dict = {}
    if _file:
        node_dict[minion_id] = render_yaml(_file, salt_data)
    else:
        log.warning("%s: Node definition not found", minion_id)
        node_dict[minion_id] = {}

# Merge newly found pillars into existing ones
    dict_merge(salt_data["__pillar__"], node_dict[minion_id].get("pillars", {}))

# Get 2 ordered lists:
    # expanded_classes: A list of all the dicts
    # classes_list: List of all the classes
    expanded_classes, classes_list, states_list = expand_classes_in_order(
        node_dict[minion_id], salt_data, [], {}, []
    )

# Here merge the pillars together
    pillars_dict = {}
    for exp_dict in expanded_classes:
        if "pillars" in exp_dict:
            dict_merge(pillars_dict, exp_dict)

return expanded_classes, pillars_dict, classes_list, states_list

def get_pillars(minion_id, salt_data):
    # Get 2 dicts and 2 lists
    # expanded_classes: Full list of expanded dicts
    # pillars_dict: dict containing merged pillars in order
    # classes_list: All classes processed in order
    # states_list: All states listed in order
    (
        expanded_classes,
        pillars_dict,
        classes_list,
        states_list,
    ) = expanded_dict_from_minion(minion_id, salt_data)

# Retrieve environment
    environment = get_env_from_dict(expanded_classes)

# Expand ${} variables in merged dict
    # pillars key shouldn't exist if we haven't found any minion_id ref
    if "pillars" in pillars_dict:
        pillars_dict_expanded = expand_variables(pillars_dict["pillars"], {}, [])
    else:
        pillars_dict_expanded = expand_variables({}, {}, [])

# Build the final pillars dict
    pillars_dict = {}
    pillars_dict["__saltclass__"] = {}
    pillars_dict["__saltclass__"]["states"] = states_list
    pillars_dict["__saltclass__"]["classes"] = classes_list
    pillars_dict["__saltclass__"]["environment"] = environment
    pillars_dict["__saltclass__"]["nodename"] = minion_id
    pillars_dict.update(pillars_dict_expanded)

return pillars_dict

def get_tops(minion_id, salt_data):
    # Get 2 dicts and 2 lists
    # expanded_classes: Full list of expanded dicts
    # pillars_dict: dict containing merged pillars in order
    # classes_list: All classes processed in order
    # states_list: All states listed in order
    (
        expanded_classes,
        pillars_dict,
        classes_list,
        states_list,
    ) = expanded_dict_from_minion(minion_id, salt_data)

# Retrieve environment
    environment = get_env_from_dict(expanded_classes)

# Build final top dict
    tops_dict = {}
    tops_dict[environment] = states_list

return tops_dict