golden hour

/opt/saltstack/salt/lib/python3.10/site-packages/salt/modules

File/Folder	Size	Actions
__init__.py	35 B
__pycache__	-
acme.py	12.74 KB
aix_group.py	4.12 KB
aix_shadow.py	1.93 KB
aixpkg.py	24.15 KB
aliases.py	5.07 KB
alternatives.py	5.71 KB
ansiblegate.py	19.51 KB
apache.py	12.47 KB
apcups.py	2.15 KB
apf.py	3.09 KB
apkpkg.py	16 KB
aptly.py	15.28 KB
aptpkg.py	116.64 KB
archive.py	48.63 KB
arista_pyeapi.py	22.06 KB
artifactory.py	24.78 KB
at.py	10.72 KB
at_solaris.py	8.51 KB
augeas_cfg.py	13.93 KB
aws_sqs.py	6.55 KB
azurearm_compute.py	20.8 KB
azurearm_dns.py	15.8 KB
azurearm_network.py	82.99 KB
azurearm_resource.py	35.75 KB
bamboohr.py	7.36 KB
baredoc.py	11.13 KB
bcache.py	28.97 KB
beacons.py	27.82 KB
bigip.py	69.11 KB
bluez_bluetooth.py	6.76 KB
boto3_elasticache.py	37.34 KB
boto3_elasticsearch.py	53.17 KB
boto3_route53.py	39.82 KB
boto3_sns.py	12.93 KB
boto_apigateway.py	61.86 KB
boto_asg.py	35.69 KB
boto_cfn.py	7.95 KB
boto_cloudfront.py	12.75 KB
boto_cloudtrail.py	14.45 KB
boto_cloudwatch.py	10.99 KB
boto_cloudwatch_event.py	9.48 KB
boto_cognitoidentity.py	14.63 KB
boto_datapipeline.py	6.94 KB
boto_dynamodb.py	14.98 KB
boto_ec2.py	79.27 KB
boto_efs.py	14.05 KB
boto_elasticache.py	23.69 KB
boto_elasticsearch_domain.py	15.85 KB
boto_elb.py	35.53 KB
boto_elbv2.py	10.78 KB
boto_iam.py	75.62 KB
boto_iot.py	26.2 KB
boto_kinesis.py	19.62 KB
boto_kms.py	17.29 KB
boto_lambda.py	35.05 KB
boto_rds.py	34.92 KB
boto_route53.py	32.55 KB
boto_s3.py	4.24 KB
boto_s3_bucket.py	31.8 KB
boto_secgroup.py	25.22 KB
boto_sns.py	7.22 KB
boto_sqs.py	6.43 KB
boto_ssm.py	3.63 KB
boto_vpc.py	113.08 KB
bower.py	5.85 KB
bridge.py	10.81 KB
bsd_shadow.py	6.25 KB
btrfs.py	33.66 KB
cabal.py	3.79 KB
capirca_acl.py	40.04 KB
cassandra_cql.py	54.16 KB
cassandra_mod.py	4.3 KB
celery.py	3.33 KB
ceph.py	15.82 KB
chassis.py	1.52 KB
chef.py	4.66 KB
chocolatey.py	41.55 KB
chronos.py	2.89 KB
chroot.py	11.73 KB
cimc.py	23.02 KB
ciscoconfparse_mod.py	14.79 KB
cisconso.py	3.83 KB
cloud.py	9.39 KB
cmdmod.py	163.73 KB
composer.py	10.31 KB
config.py	16.98 KB
consul.py	69.3 KB
container_resource.py	12.94 KB
cp.py	31.98 KB
cpan.py	5.54 KB
cron.py	28.09 KB
cryptdev.py	10.08 KB
csf.py	16.04 KB
cyg.py	8.32 KB
daemontools.py	5.41 KB
data.py	3.85 KB
datadog_api.py	7.76 KB
ddns.py	7.12 KB
deb_apache.py	7.41 KB
deb_postgres.py	4.18 KB
debconfmod.py	4.06 KB
debian_ip.py	64.91 KB
debian_service.py	6.55 KB
debuild_pkgbuild.py	34.68 KB
defaults.py	6.55 KB
devinfo.py	9.07 KB
devmap.py	627 B
dig.py	8.75 KB
disk.py	30.82 KB
djangomod.py	7.53 KB
dnsmasq.py	5.71 KB
dnsutil.py	11.51 KB
dockercompose.py	32.62 KB
dockermod.py	224.85 KB
dpkg_lowpkg.py	12.94 KB
drac.py	10.97 KB
dracr.py	38.53 KB
drbd.py	7.19 KB
dummyproxy_pkg.py	2.46 KB
dummyproxy_service.py	2.91 KB
ebuildpkg.py	38.74 KB
eix.py	1.58 KB
elasticsearch.py	51.44 KB
environ.py	8.96 KB
eselect.py	4.99 KB
esxcluster.py	1.66 KB
esxdatacenter.py	1.68 KB
esxi.py	2.79 KB
esxvm.py	1.63 KB
etcd_mod.py	8.56 KB
ethtool.py	11.12 KB
event.py	7.67 KB
extfs.py	8.78 KB
file.py	232.18 KB
firewalld.py	20.51 KB
freebsd_sysctl.py	4.99 KB
freebsd_update.py	6.19 KB
freebsdjail.py	7.16 KB
freebsdkmod.py	6.17 KB
freebsdpkg.py	17.04 KB
freebsdports.py	13.13 KB
freebsdservice.py	12.53 KB
freezer.py	10.2 KB
gcp_addon.py	4.07 KB
gem.py	10.6 KB
genesis.py	21.75 KB
gentoo_service.py	9.18 KB
gentoolkitmod.py	8.39 KB
git.py	172.01 KB
github.py	53.19 KB
glanceng.py	4.69 KB
glassfish.py	19.49 KB
glusterfs.py	19.55 KB
gnomedesktop.py	6.85 KB
google_chat.py	1.52 KB
gpg.py	39.09 KB
grafana4.py	30.27 KB
grains.py	21.81 KB
groupadd.py	11.87 KB
grub_legacy.py	3.08 KB
guestfs.py	2.37 KB
hadoop.py	3.76 KB
haproxyconn.py	10.17 KB
hashutil.py	6.77 KB
heat.py	25.25 KB
helm.py	39.27 KB
hg.py	7.16 KB
highstate_doc.py	22.76 KB
hosts.py	10.47 KB
http.py	3.76 KB
icinga2.py	4.46 KB
idem.py	1.75 KB
ifttt.py	2.28 KB
ilo.py	15.98 KB
incron.py	7.68 KB
influxdb08mod.py	15.07 KB
influxdbmod.py	16.13 KB
infoblox.py	17.53 KB
ini_manage.py	14.63 KB
inspectlib	-
inspector.py	8.19 KB
introspect.py	4.02 KB
iosconfig.py	14.78 KB
ipmi.py	25.47 KB
ipset.py	17.97 KB
iptables.py	57.44 KB
iwtools.py	3.99 KB
jboss7.py	20.51 KB
jboss7_cli.py	15.23 KB
jenkinsmod.py	11.9 KB
jinja.py	2.66 KB
jira_mod.py	7.07 KB
junos.py	73.96 KB
k8s.py	24.87 KB
kapacitor.py	5.37 KB
kerberos.py	5.42 KB
kernelpkg_linux_apt.py	6.71 KB
kernelpkg_linux_yum.py	7.26 KB
key.py	1007 B
keyboard.py	2.64 KB
keystone.py	43.14 KB
keystoneng.py	21.82 KB
keystore.py	7.18 KB
kmod.py	7.65 KB
kubeadm.py	34.64 KB
kubernetesmod.py	46.77 KB
launchctl_service.py	9.69 KB
layman.py	4.22 KB
ldap3.py	18.81 KB
ldapmod.py	5.9 KB
libcloud_compute.py	23.48 KB
libcloud_dns.py	9.73 KB
libcloud_loadbalancer.py	13.14 KB
libcloud_storage.py	12.16 KB
linux_acl.py	7.7 KB
linux_ip.py	5.55 KB
linux_lvm.py	17.86 KB
linux_service.py	4.64 KB
linux_shadow.py	12.96 KB
linux_sysctl.py	7.5 KB
localemod.py	11.84 KB
locate.py	2.58 KB
logadm.py	9.44 KB
logmod.py	1.25 KB
logrotate.py	7.72 KB
lvs.py	11.54 KB
lxc.py	147.27 KB
lxd.py	90.07 KB
mac_assistive.py	11.37 KB
mac_brew_pkg.py	19.91 KB
mac_desktop.py	2.77 KB
mac_group.py	6.62 KB
mac_keychain.py	6.39 KB
mac_pkgutil.py	2.84 KB
mac_portspkg.py	11.36 KB
mac_power.py	13.29 KB
mac_service.py	19.64 KB
mac_shadow.py	14.23 KB
mac_softwareupdate.py	14.52 KB
mac_sysctl.py	5.13 KB
mac_system.py	15.07 KB
mac_timezone.py	8.34 KB
mac_user.py	16.41 KB
mac_xattr.py	6.27 KB
macdefaults.py	2.33 KB
macpackage.py	6.66 KB
makeconf.py	17.31 KB
mandrill.py	6.31 KB
marathon.py	5.36 KB
match.py	13 KB
mattermost.py	3.4 KB
mdadm_raid.py	9.86 KB
mdata.py	3.38 KB
memcached.py	6.13 KB
mine.py	18.84 KB
minion.py	7.68 KB
mod_random.py	7.18 KB
modjk.py	12.48 KB
mongodb.py	29.75 KB
monit.py	5.51 KB
moosefs.py	3.87 KB
mount.py	58.44 KB
mssql.py	14.64 KB
msteams.py	2.11 KB
munin.py	2.4 KB
mysql.py	90.66 KB
nacl.py	9.72 KB
nagios.py	6.53 KB
nagios_rpc.py	5.09 KB
namecheap_domains.py	12.84 KB
namecheap_domains_dns.py	5.93 KB
namecheap_domains_ns.py	4.51 KB
namecheap_ssl.py	25.69 KB
namecheap_users.py	2.4 KB
napalm_bgp.py	9.72 KB
napalm_formula.py	11.33 KB
napalm_mod.py	61.37 KB
napalm_netacl.py	28.59 KB
napalm_network.py	93.22 KB
napalm_ntp.py	10.22 KB
napalm_probes.py	13.25 KB
napalm_route.py	5.09 KB
napalm_snmp.py	7.05 KB
napalm_users.py	6.49 KB
napalm_yang_mod.py	20.28 KB
netaddress.py	1.6 KB
netbox.py	32.22 KB
netbsd_sysctl.py	3.92 KB
netbsdservice.py	6.43 KB
netmiko_mod.py	19.61 KB
netscaler.py	27.02 KB
network.py	63.42 KB
neutron.py	44.92 KB
neutronng.py	15.02 KB
nexus.py	22.95 KB
nfs3.py	3.9 KB
nftables.py	33.58 KB
nginx.py	3.83 KB
nilrt_ip.py	36.18 KB
nix.py	8.03 KB
nova.py	19.6 KB
npm.py	10.4 KB
nspawn.py	41.35 KB
nxos.py	24.65 KB
nxos_api.py	14.72 KB
nxos_upgrade.py	14.74 KB
omapi.py	3.6 KB
openbsd_sysctl.py	3.74 KB
openbsdpkg.py	10.97 KB
openbsdrcctl_service.py	6.33 KB
openbsdservice.py	8.31 KB
openscap.py	2.81 KB
openstack_config.py	3.21 KB
openstack_mng.py	2.71 KB
openvswitch.py	17.19 KB
opkg.py	49.67 KB
opsgenie.py	3.29 KB
oracle.py	5.82 KB
osquery.py	24.93 KB
out.py	2.53 KB
pacmanpkg.py	31.92 KB
pagerduty.py	4.7 KB
pagerduty_util.py	13.48 KB
pam.py	2.01 KB
panos.py	61.05 KB
parallels.py	19.85 KB
parted_partition.py	21.53 KB
pcs.py	14.11 KB
pdbedit.py	10.79 KB
pecl.py	3.79 KB
peeringdb.py	8.39 KB
pf.py	9.51 KB
philips_hue.py	1.55 KB
pillar.py	21.37 KB
pip.py	53.42 KB
pkg_resource.py	12.3 KB
pkgin.py	17.29 KB
pkgng.py	61.07 KB
pkgutil.py	9.85 KB
portage_config.py	22.73 KB
postfix.py	16.24 KB
postgres.py	88.24 KB
poudriere.py	7.85 KB
powerpath.py	2.57 KB
proxy.py	11.49 KB
ps.py	20.89 KB
publish.py	10.25 KB
puppet.py	10.9 KB
purefa.py	33.59 KB
purefb.py	13.69 KB
pushbullet.py	1.88 KB
pushover_notify.py	3.48 KB
pw_group.py	4.62 KB
pw_user.py	12.47 KB
pyenv.py	6.93 KB
qemu_img.py	1.53 KB
qemu_nbd.py	3.28 KB
quota.py	6.43 KB
rabbitmq.py	38.4 KB
rallydev.py	6.09 KB
random_org.py	23.76 KB
rbac_solaris.py	16.05 KB
rbenv.py	10.75 KB
rdp.py	6.08 KB
rebootmgr.py	7.68 KB
redismod.py	16.36 KB
reg.py	16.36 KB
rest_pkg.py	2.26 KB
rest_sample_utils.py	558 B
rest_service.py	3.63 KB
restartcheck.py	24.1 KB
restconf.py	3.15 KB
ret.py	1.27 KB
rh_ip.py	38.55 KB
rh_service.py	16.61 KB
riak.py	5.19 KB
rpm_lowpkg.py	27.67 KB
rpmbuild_pkgbuild.py	24.53 KB
rsync.py	8.04 KB
runit.py	17.17 KB
rvm.py	11.1 KB
s3.py	9.93 KB
s6.py	3.62 KB
salt_proxy.py	4.48 KB
salt_version.py	4.58 KB
saltcheck.py	46.66 KB
saltcloudmod.py	954 B
saltutil.py	57.49 KB
schedule.py	50.81 KB
scp_mod.py	6.22 KB
scsi.py	2.66 KB
sdb.py	2.48 KB
seed.py	8.87 KB
selinux.py	24.2 KB
sensehat.py	7.79 KB
sensors.py	1.3 KB
serverdensity_device.py	8.1 KB
servicenow.py	4.36 KB
slack_notify.py	7.83 KB
slackware_service.py	6.84 KB
slsutil.py	19.04 KB
smartos_imgadm.py	12.04 KB
smartos_nictagadm.py	6.46 KB
smartos_virt.py	5.21 KB
smartos_vmadm.py	26.2 KB
smbios.py	10.05 KB
smf_service.py	8.52 KB
smtp.py	5.41 KB
snapper.py	27.14 KB
solaris_fmadm.py	11.2 KB
solaris_group.py	2.8 KB
solaris_shadow.py	7.98 KB
solaris_system.py	3.72 KB
solaris_user.py	11.06 KB
solarisipspkg.py	18.68 KB
solarispkg.py	15.4 KB
solr.py	45.54 KB
solrcloud.py	14.63 KB
splunk.py	8.14 KB
splunk_search.py	8.76 KB
sqlite3.py	2.54 KB
ssh.py	43.89 KB
ssh_pkg.py	1.08 KB
ssh_service.py	3.39 KB
state.py	82.34 KB
status.py	57.79 KB
statuspage.py	14.67 KB
supervisord.py	11.15 KB
suse_apache.py	2.45 KB
suse_ip.py	35.72 KB
svn.py	10.75 KB
swarm.py	13.5 KB
swift.py	5.53 KB
sysbench.py	6.62 KB
sysfs.py	6.61 KB
syslog_ng.py	31.52 KB
sysmod.py	22.59 KB
sysrc.py	3.38 KB
system.py	19.28 KB
system_profiler.py	3.54 KB
systemd_service.py	46.29 KB
telegram.py	3.28 KB
telemetry.py	12.87 KB
temp.py	831 B
test.py	15.4 KB
test_virtual.py	237 B
testinframod.py	9.92 KB
textfsm_mod.py	16.22 KB
timezone.py	19.98 KB
tls.py	58.63 KB
tomcat.py	18.59 KB
trafficserver.py	10.44 KB
transactional_update.py	35.83 KB
travisci.py	2.05 KB
tuned.py	2.34 KB
twilio_notify.py	2.95 KB
udev.py	3.72 KB
upstart_service.py	16.92 KB
uptime.py	3.23 KB
useradd.py	22.63 KB
uwsgi.py	996 B
vagrant.py	20.4 KB
varnish.py	3.08 KB
vault.py	15.61 KB
vbox_guest.py	10.55 KB
vboxmanage.py	14.71 KB
vcenter.py	1.61 KB
victorops.py	6.54 KB
virt.py	287.71 KB
virtualenv_mod.py	15.09 KB
vmctl.py	9.56 KB
vsphere.py	380.41 KB
webutil.py	3.66 KB
win_auditpol.py	4.74 KB
win_autoruns.py	2.29 KB
win_certutil.py	4.55 KB
win_dacl.py	32.27 KB
win_disk.py	1.8 KB
win_dism.py	20.7 KB
win_dns_client.py	4.19 KB
win_dsc.py	27.54 KB
win_event.py	22.32 KB
win_file.py	64.39 KB
win_firewall.py	20.15 KB
win_groupadd.py	11.27 KB
win_iis.py	68.78 KB
win_ip.py	11.43 KB
win_lgpo.py	491.76 KB
win_lgpo_reg.py	17.9 KB
win_license.py	2.72 KB
win_network.py	13.9 KB
win_ntp.py	1.8 KB
win_path.py	11.12 KB
win_pkg.py	86.43 KB
win_pki.py	15.8 KB
win_powercfg.py	9.85 KB
win_psget.py	8.97 KB
win_servermanager.py	14.21 KB
win_service.py	32.96 KB
win_shadow.py	3.03 KB
win_shortcut.py	16.49 KB
win_smtp_server.py	17.67 KB
win_snmp.py	13.38 KB
win_status.py	16.94 KB
win_system.py	40.61 KB
win_task.py	79.17 KB
win_timezone.py	13.3 KB
win_useradd.py	27.39 KB
win_wua.py	38.29 KB
win_wusa.py	5.88 KB
winrepo.py	6.09 KB
wordpress.py	4.71 KB
x509.py	63.1 KB
x509_v2.py	74.15 KB
xapi_virt.py	24.07 KB
xbpspkg.py	15.84 KB
xfs.py	15.35 KB
xml.py	2.14 KB
xmpp.py	5.28 KB
yaml.py	1.94 KB
yumpkg.py	116.5 KB
zabbix.py	97.55 KB
zcbuildout.py	28.16 KB
zenoss.py	5.64 KB
zfs.py	34.44 KB
zk_concurrency.py	11.16 KB
znc.py	2.26 KB
zoneadm.py	15.05 KB
zonecfg.py	21.85 KB
zookeeper.py	14.72 KB
zpool.py	44.02 KB
zypperpkg.py	94.87 KB

Edit: highstate_doc.py

# pylint: disable=W1401
'''
This module renders highstate configuration into a more human readable format.

How it works:

`highstate or lowstate` data is parsed with a `processor` this defaults to `highstate_doc.processor_markdown`.
The processed data is passed to a `jinja` template that builds up the document content.

configuration: Pillar

.. code-block:: none

# the following defaults can be overridden
    highstate_doc.config:

# list of regex of state names to ignore in `highstate_doc.process_lowstates`
        filter_id_regex:
            - '.*!doc_skip$'

# list of regex of state functions to ignore in `highstate_doc.process_lowstates`
        filter_state_function_regex:
            - 'file.accumulated'

# dict of regex to replace text after `highstate_doc.render`. (remove passwords)
        text_replace_regex:
            'password:.*^': '[PASSWORD]'

# limit size of files that can be included in doc (10000 bytes)
        max_render_file_size: 10000

# advanced option to set a custom lowstate processor
        processor: highstate_doc.processor_markdown

State example

.. code-block:: yaml

{{sls}} note:
        highstate_doc.note:
            - name: example
            - order: 0
            - contents: |
                example `highstate_doc.note`
                ------------------
                This state does not do anything to the system! It is only used by a `processor`
                you can use `requisites` and `order` to move your docs around the rendered file.

{{sls}} a file we don't want in the doc !doc_skip:
        file.managed:
            - name: /root/passwords
            - contents: 'password: sadefgq34y45h56q'
            # also could use `highstate_doc.config: text_replace_regex` to replace
            # password string. `password:.*^': '[PASSWORD]`

To create the help document build a State that uses `highstate_doc.render`.
For performance it's advised to not included this state in your `top.sls` file.

.. code-block:: yaml

# example `salt://makereadme.sls`
    make helpfile:
        file.managed:
            - name: /root/README.md
            - contents: {{salt.highstate_doc.render()|json}}
            - show_diff: {{opts['test']}}
            - mode: '0640'
            - order: last

Run our `makereadme.sls` state to create `/root/README.md`.

.. code-block:: bash

# first ensure `highstate` return without errors or changes
    salt-call state.highstate
    salt-call state.apply makereadme
    # or if you don't want the extra `make helpfile` state
    salt-call --out=newline_values_only salt.highstate_doc.render > /root/README.md ; chmod 0600 /root/README.md

Creating a document collection
------------------------------

From the master we can run the following script to
creates a collection of all your minion documents.

.. code-block:: bash

salt '*' state.apply makereadme

.. code-block:: python

#!/bin/python
    import os
    import salt.client
    s = salt.client.LocalClient()
    # NOTE: because of issues with `cp.push` use `highstate_doc.read_file`
    o = s.cmd('*', 'highstate_doc.read_file', ['/root/README.md'])
    for m in o:
        d = o.get(m)
        if d and not d.endswith('is not available.'):
            # mkdir m
            #directory = os.path.dirname(file_path)
            if not os.path.exists(m):
                os.makedirs(m)
            with open(m + '/README.md','wb') as f:
                f.write(d)
            print('ADDED: ' + m + '/README.md')

Once the master has a collection of all the README files.
You can use pandoc to create HTML versions of the markdown.

.. code-block:: bash

# process all the readme.md files to readme.html
    if which pandoc; then echo "Found pandoc"; else echo "** Missing pandoc"; exit 1; fi
    if which gs; then echo "Found gs"; else echo "** Missing gs(ghostscript)"; exit 1; fi
    readme_files=$(find $dest -type f -path "*/README.md" -print)
    for f in $readme_files ; do
        ff=${f#$dest/}
        minion=${ff%%/*}
        echo "process: $dest/${minion}/$(basename $f)"
        cat $dest/${minion}/$(basename $f) | \
            pandoc --standalone --from markdown_github --to html \
            --include-in-header $dest/style.html \
            > $dest/${minion}/$(basename $f).html
    done

It is also nice to put the help files in source control.

# git init
    git add -A
    git commit -am 'updated docs'
    git push -f

Other hints
-----------

If you wish to customize the document format:

.. code-block:: none

# you could also create a new `processor` for perhaps reStructuredText
    # highstate_doc.config:
    #     processor: doc_custom.processor_rst

# example `salt://makereadme.jinja`
    """
    {{opts['id']}}
    ==========================================

{# lowstates is set from highstate_doc.render() #}
    {# if lowstates is missing use salt.highstate_doc.process_lowstates() #}
    {% for s in lowstates %}
    {{s.id}}
    -----------------------------------------------------------------
    {{s.function}}

{%- endfor %}
    """

# example `salt://makereadme.sls`
    {% import_text "makereadme.jinja" as makereadme %}
    {{sls}} or:
        file.managed:
            - name: /root/README_other.md
            - contents: {{salt.highstate_doc.render(jinja_template_text=makereadme)|json}}
            - mode: '0640'

Some `replace_text_regex` values that might be helpful::

CERTS
    -----

``'-----BEGIN RSA PRIVATE KEY-----[\\r\\n\\t\\f\\S]{0,2200}': 'XXXXXXX'``
    ``'-----BEGIN CERTIFICATE-----[\\r\\n\\t\\f\\S]{0,2200}': 'XXXXXXX'``
    ``'-----BEGIN DH PARAMETERS-----[\\r\\n\\t\\f\\S]{0,2200}': 'XXXXXXX'``
    ``'-----BEGIN PRIVATE KEY-----[\\r\\n\\t\\f\\S]{0,2200}': 'XXXXXXX'``
    ``'-----BEGIN OPENSSH PRIVATE KEY-----[\\r\\n\\t\\f\\S]{0,2200}': 'XXXXXXX'``
    ``'ssh-rsa .* ': 'ssh-rsa XXXXXXX '``
    ``'ssh-dss .* ': 'ssh-dss XXXXXXX '``

DB
    --

``'DB_PASS.*': 'DB_PASS = XXXXXXX'``
    ``'5432:*:*:.*': '5432:*:XXXXXXX'``
    ``"'PASSWORD': .*": "'PASSWORD': 'XXXXXXX',"``
    ``" PASSWORD '.*'": " PASSWORD 'XXXXXXX'"``
    ``'PGPASSWORD=.* ': 'PGPASSWORD=XXXXXXX'``
    ``"_replication password '.*'":  "_replication password 'XXXXXXX'"``

OTHER
    -----

``'EMAIL_HOST_PASSWORD =.*': 'EMAIL_HOST_PASSWORD =XXXXXXX'``
    ``"net ads join -U '.*@MFCFADS.MATH.EXAMPLE.CA.* ": "net ads join -U '.*@MFCFADS.MATH.EXAMPLE.CA%XXXXXXX "``
    ``"net ads join -U '.*@NEXUS.EXAMPLE.CA.* ": "net ads join -U '.*@NEXUS.EXAMPLE.CA%XXXXXXX "``
    ``'install-uptrack .* --autoinstall': 'install-uptrack XXXXXXX --autoinstall'``
    ``'accesskey = .*': 'accesskey = XXXXXXX'``
    ``'auth_pass .*': 'auth_pass XXXXXXX'``
    ``'PSK "0x.*': 'PSK "0xXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'``
    ``'SECRET_KEY.*': 'SECRET_KEY = XXXXXXX'``
    ``"password=.*": "password=XXXXXXX"``
    ``'<password>.*</password>': '<password>XXXXXXX</password>'``
    ``'<salt>.*</salt>': '<salt>XXXXXXX</salt>'``
    ``'application.secret = ".*"': 'application.secret = "XXXXXXX"'``
    ``'url = "postgres://.*"': 'url = "postgres://XXXXXXX"'``
    ``'PASS_.*_PASS': 'PASS_XXXXXXX_PASS'``

HTACCESS
    --------

``':{PLAIN}.*': ':{PLAIN}XXXXXXX'``

'''

import logging
import re

import salt.utils.files
import salt.utils.stringutils
import salt.utils.templates as tpl
import salt.utils.yaml

__virtualname__ = "highstate_doc"

log = logging.getLogger(__name__)

markdown_basic_jinja_template_txt = """
{% for s in lowstates %}
`{{s.id_full}}`
-----------------------------------------------------------------
 * state: {{s.state_function}}
 * name: `{{s.name}}`

{%- endfor %}
"""

markdown_default_jinja_template_txt = (
    """
Configuration Managment
===============================================================================

```
####################################################
fqdn: {{grains.get('fqdn')}}
os: {{grains.get('os')}}
osfinger: {{grains.get('osfinger')}}
mem_total: {{grains.get('mem_total')}}MB
num_cpus: {{grains.get('num_cpus')}}
ipv4: {{grains.get('ipv4')}}
master: {{opts.get('master')}}
####################################################
```

This system is fully or partly managed using Salt.

The following sections are a rendered view of what the configuration management system
controlled on this system. Each item is handled in order from top to bottom unless some
requisites like `require` force other ordering.

"""
    + markdown_basic_jinja_template_txt
)

markdown_advanced_jinja_template_txt = (
    markdown_default_jinja_template_txt
    + r"""

{% if vars.get('doc_other', True) -%}
Other information
=====================================================================================

```

salt grain: ip_interfaces
-----------------------------------------------------------------
{{grains['ip_interfaces']|dictsort}}

salt grain: hwaddr_interfaces
-----------------------------------------------------------------
{{grains['hwaddr_interfaces']|dictsort}}

{% if not grains['os'] == 'Windows' %}

{% if salt['cmd.has_exec']('ip') -%}
# ip address show
-----------------------------------------------------------------
{{salt['cmd.run']('ip address show | sed "/valid_lft/d"')}}

# ip route list table all
-----------------------------------------------------------------
{{salt['cmd.run']('ip route list table all')}}
{% endif %}

{% if salt['cmd.has_exec']('iptables') %}
{%- if salt['cmd.has_exec']('iptables-save') -%}
# iptables-save
-----------------------------------------------------------------
{{salt['cmd.run']("iptables --list > /dev/null; iptables-save | \grep -v -F '#' | sed '/^:/s@\[[0-9]\{1,\}:[0-9]\{1,\}\]@[0:0]@g'")}}

# ip6tables-save
-----------------------------------------------------------------
{{salt['cmd.run']("ip6tables --list > /dev/null; ip6tables-save | \grep -v -F '#' | sed '/^:/s@\[[0-9]\{1,\}:[0-9]\{1,\}\]@[0:0]@g'")}}
{%- else -%}
# iptables --list-rules
-----------------------------------------------------------------
{{salt['cmd.run']('iptables --list-rules')}}

# ip6tables --list-rules
-----------------------------------------------------------------
{{salt['cmd.run']('ip6tables --list-rules')}}
{% endif %}
{% endif %}

{% if salt['cmd.has_exec']('firewall-cmd') -%}
# firewall-cmd --list-all
-----------------------------------------------------------------
{{salt['cmd.run']('firewall-cmd --list-all')}}
{% endif %}

# mount
-----------------------------------------------------------------
{{salt['cmd.run']('mount')}}

{% endif %}
"""
)

def markdown_basic_jinja_template(**kwargs):
    """
    Return text for a simple markdown jinja template

This function can be used from the `highstate_doc.render` modules `jinja_template_function` option.
    """
    return markdown_basic_jinja_template_txt

def markdown_default_jinja_template(**kwargs):
    """
    Return text for a markdown jinja template that included a header

This function can be used from the `highstate_doc.render` modules `jinja_template_function` option.
    """
    return markdown_default_jinja_template_txt

def markdown_full_jinja_template(**kwargs):
    """
    Return text for an advanced markdown jinja template

This function can be used from the `highstate_doc.render` modules `jinja_template_function` option.
    """
    return markdown_advanced_jinja_template_txt

def _get_config(**kwargs):
    """
    Return configuration
    """
    config = {
        "filter_id_regex": [".*!doc_skip"],
        "filter_function_regex": [],
        "replace_text_regex": {},
        "processor": "highstate_doc.processor_markdown",
        "max_render_file_size": 10000,
        "note": None,
    }
    if "__salt__" in globals():
        config_key = "{}.config".format(__virtualname__)
        config.update(__salt__["config.get"](config_key, {}))
    # pylint: disable=C0201
    for k in set(config.keys()) & set(kwargs.keys()):
        config[k] = kwargs[k]
    return config

def read_file(name):
    """
    output the contents of a file:

this is a workaround if the cp.push module does not work.
    https://github.com/saltstack/salt/issues/37133

help the master output the contents of a document
    that might be saved on the minions filesystem.

.. code-block:: python

#!/bin/python
        import os
        import salt.client
        s = salt.client.LocalClient()
        o = s.cmd('*', 'highstate_doc.read_file', ['/root/README.md'])
        for m in o:
            d = o.get(m)
            if d and not d.endswith('is not available.'):
                # mkdir m
                #directory = os.path.dirname(file_path)
                if not os.path.exists(m):
                    os.makedirs(m)
                with open(m + '/README.md','wb') as fin:
                    fin.write(d)
                print('ADDED: ' + m + '/README.md')
    """
    out = ""
    try:
        with salt.utils.files.fopen(name, "r") as f:
            out = salt.utils.stringutils.to_unicode(f.read())
    except Exception as ex:  # pylint: disable=broad-except
        log.error(ex)
        return None
    return out

def render(
    jinja_template_text=None,
    jinja_template_function="highstate_doc.markdown_default_jinja_template",
    **kwargs
):
    """
    Render highstate to a text format (default Markdown)

if `jinja_template_text` is not set, `jinja_template_function` is used.

jinja_template_text: jinja text that the render uses to create the document.
    jinja_template_function: a salt module call that returns template text.

:options:
        highstate_doc.markdown_basic_jinja_template
        highstate_doc.markdown_default_jinja_template
        highstate_doc.markdown_full_jinja_template

"""
    config = _get_config(**kwargs)
    lowstates = process_lowstates(**kwargs)
    # TODO: __env__,
    context = {
        "saltenv": None,
        "config": config,
        "lowstates": lowstates,
        "salt": __salt__,
        "pillar": __pillar__,
        "grains": __grains__,
        "opts": __opts__,
        "kwargs": kwargs,
    }
    template_text = jinja_template_text
    if template_text is None and jinja_template_function:
        template_text = __salt__[jinja_template_function](**kwargs)
    if template_text is None:
        raise Exception("No jinja template text")

txt = tpl.render_jinja_tmpl(template_text, context, tmplpath=None)
    # after processing the template replace passwords or other data.
    rt = config.get("replace_text_regex")
    for r in rt:
        txt = re.sub(r, rt[r], txt)
    return txt

def _blacklist_filter(s, config):
    ss = s["state"]
    sf = s["fun"]
    state_function = "{}.{}".format(s["state"], s["fun"])
    for b in config["filter_function_regex"]:
        if re.match(b, state_function):
            return True
    for b in config["filter_id_regex"]:
        if re.match(b, s["__id__"]):
            return True
    return False

def process_lowstates(**kwargs):
    """
    return processed lowstate data that was not blacklisted

render_module_function is used to provide your own.
    defaults to from_lowstate
    """
    states = []
    config = _get_config(**kwargs)
    processor = config.get("processor")
    ls = __salt__["state.show_lowstate"]()

if not isinstance(ls, list):
        raise Exception(
            "ERROR: to see details run: [salt-call state.show_lowstate]"
            " <-----***-SEE-***"
        )
    else:
        if ls:
            if not isinstance(ls[0], dict):
                raise Exception(
                    "ERROR: to see details run: [salt-call state.show_lowstate]"
                    " <-----***-SEE-***"
                )

for s in ls:
        if _blacklist_filter(s, config):
            continue
        doc = __salt__[processor](s, config, **kwargs)
        states.append(doc)
    return states

def _state_data_to_yaml_string(data, whitelist=None, blacklist=None):
    """
    return a data dict in yaml string format.
    """
    y = {}
    if blacklist is None:
        # TODO: use salt defined STATE_REQUISITE_IN_KEYWORDS  STATE_RUNTIME_KEYWORDS  STATE_INTERNAL_KEYWORDS
        blacklist = [
            "__env__",
            "__id__",
            "__sls__",
            "fun",
            "name",
            "context",
            "order",
            "state",
            "require",
            "require_in",
            "watch",
            "watch_in",
        ]
    kset = set(data.keys())
    if blacklist:
        kset -= set(blacklist)
    if whitelist:
        kset &= set(whitelist)
    for k in kset:
        y[k] = data[k]
    if not y:
        return None
    return salt.utils.yaml.safe_dump(y, default_flow_style=False)

def _md_fix(text):
    """
    sanitize text data that is to be displayed in a markdown code block
    """
    return text.replace("```", "``[`][markdown parse fix]")

def _format_markdown_system_file(filename, config):
    ret = ""
    file_stats = __salt__["file.stats"](filename)
    y = _state_data_to_yaml_string(
        file_stats, whitelist=["user", "group", "mode", "uid", "gid", "size"]
    )
    if y:
        ret += "file stat {1}\n```\n{0}```\n".format(y, filename)
    file_size = file_stats.get("size")
    if file_size <= config.get("max_render_file_size"):
        is_binary = True
        try:
            # TODO: this is linux only should find somthing portable
            file_type = __salt__["cmd.shell"]("\\file -i '{}'".format(filename))
            if "charset=binary" not in file_type:
                is_binary = False
        except Exception as ex:  # pylint: disable=broad-except
            # likely on a windows system, set as not binary for now.
            is_binary = False
        if is_binary:
            file_data = "[[skipped binary data]]"
        else:
            with salt.utils.files.fopen(filename, "r") as f:
                file_data = salt.utils.stringutils.to_unicode(f.read())
        file_data = _md_fix(file_data)
        ret += "file data {1}\n```\n{0}\n```\n".format(file_data, filename)
    else:
        ret += "```\n{}\n```\n".format(
            "SKIPPED LARGE FILE!\nSet {}:max_render_file_size > {} to render.".format(
                "{}.config".format(__virtualname__), file_size
            )
        )
    return ret

def _format_markdown_link(name):
    link = name
    symbals = "~`!@#$%^&*()+={}[]:;\"<>,.?/|'\\"
    for s in symbals:
        link = link.replace(s, "")
    link = link.replace(" ", "-")
    return link

def _format_markdown_requisite(state, stateid, makelink=True):
    """
    format requisite as a link users can click
    """
    fmt_id = "{}: {}".format(state, stateid)
    if makelink:
        return " * [{}](#{})\n".format(fmt_id, _format_markdown_link(fmt_id))
    else:
        return " * `{}`\n".format(fmt_id)

def processor_markdown(lowstate_item, config, **kwargs):
    """
    Takes low state data and returns a dict of processed data
    that is by default used in a jinja template when rendering a markdown highstate_doc.

This `lowstate_item_markdown` given a lowstate item, returns a dict like:

.. code-block:: none

vars:       # the raw lowstate_item that was processed
        id:         # the 'id' of the state.
        id_full:    # combo of the state type and id "state: id"
        state:      # name of the salt state module
        function:   # name of the state function
        name:       # value of 'name:' passed to the salt state module
        state_function:    # the state name and function name
        markdown:          # text data to describe a state
            requisites:    # requisite like [watch_in, require_in]
            details:       # state name, parameters and other details like file contents

"""
    # TODO: switch or ... ext call.
    s = lowstate_item
    state_function = "{}.{}".format(s["state"], s["fun"])
    id_full = "{}: {}".format(s["state"], s["__id__"])

# TODO: use salt defined STATE_REQUISITE_IN_KEYWORDS
    requisites = ""
    for comment, key in (
        ("run or update after changes in:\n", "watch"),
        ("after changes, run or update:\n", "watch_in"),
        ("require:\n", "require"),
        ("required in:\n", "require_in"),
    ):
        reqs = s.get(key, [])
        if reqs:
            requisites += comment
            for w in reqs:
                requisites += _format_markdown_requisite(*next(iter(w.items())))

details = ""

if state_function == "highstate_doc.note":
        if "contents" in s:
            details += "\n{}\n".format(s["contents"])
        if "source" in s:
            text = __salt__["cp.get_file_str"](s["source"])
            if text:
                details += "\n{}\n".format(text)
            else:
                details += "\n{}\n".format("ERROR: opening {}".format(s["source"]))

if state_function == "pkg.installed":
        pkgs = s.get("pkgs", s.get("name"))
        details += "\n```\ninstall: {}\n```\n".format(pkgs)

if state_function == "file.recurse":
        details += """recurse copy of files\n"""
        y = _state_data_to_yaml_string(s)
        if y:
            details += "```\n{}\n```\n".format(y)
        if "!doc_recurse" in id_full:
            findfiles = __salt__["file.find"](path=s.get("name"), type="f")
            if len(findfiles) < 10 or "!doc_recurse_force" in id_full:
                for f in findfiles:
                    details += _format_markdown_system_file(f, config)
            else:
                details += """ > Skipping because more than 10 files to display.\n"""
                details += (
                    """ > HINT: to force include !doc_recurse_force in state id.\n"""
                )
        else:
            details += """ > For more details review logs and Salt state files.\n\n"""
            details += """ > HINT: for improved docs use multiple file.managed states or file.archive, git.latest. etc.\n"""
            details += """ > HINT: to force doc to show all files in path add !doc_recurse .\n"""

if state_function == "file.blockreplace":
        if s.get("content"):
            details += "ensure block of content is in file\n```\n{}\n```\n".format(
                _md_fix(s["content"])
            )
        if s.get("source"):
            text = "** source: " + s.get("source")
            details += "ensure block of content is in file\n```\n{}\n```\n".format(
                _md_fix(text)
            )

if state_function == "file.managed":
        details += _format_markdown_system_file(s["name"], config)

# if no state doc is created use default state as yaml
    if not details:
        y = _state_data_to_yaml_string(s)
        if y:
            details += "```\n{}```\n".format(y)

r = {
        "vars": lowstate_item,
        "state": s["state"],
        "name": s["name"],
        "function": s["fun"],
        "id": s["__id__"],
        "id_full": id_full,
        "state_function": state_function,
        "markdown": {
            "requisites": requisites.decode("utf-8"),
            "details": details.decode("utf-8"),
        },
    }
    return r