golden hour

/opt/saltstack/salt/lib/python3.10/site-packages/salt/states

File/Folder	Size	Actions
__init__.py	25 B
__pycache__	-
acme.py	5.08 KB
alias.py	2.49 KB
alternatives.py	6.75 KB
ansiblegate.py	7.93 KB
apache.py	3.95 KB
apache_conf.py	2.72 KB
apache_module.py	2.73 KB
apache_site.py	2.66 KB
aptpkg.py	1.42 KB
archive.py	68.24 KB
artifactory.py	6.84 KB
at.py	7.48 KB
augeas.py	10.57 KB
aws_sqs.py	2.59 KB
azurearm_compute.py	11.78 KB
azurearm_dns.py	26.05 KB
azurearm_network.py	89.12 KB
azurearm_resource.py	28.23 KB
beacon.py	7.58 KB
bigip.py	96.63 KB
blockdev.py	5.13 KB
boto3_elasticache.py	48.01 KB
boto3_elasticsearch.py	32.58 KB
boto3_route53.py	37.54 KB
boto3_sns.py	12.69 KB
boto_apigateway.py	82.83 KB
boto_asg.py	31.93 KB
boto_cfn.py	11.53 KB
boto_cloudfront.py	6.01 KB
boto_cloudtrail.py	13.18 KB
boto_cloudwatch_alarm.py	6.4 KB
boto_cloudwatch_event.py	12.33 KB
boto_cognitoidentity.py	13.69 KB
boto_datapipeline.py	18.5 KB
boto_dynamodb.py	29.32 KB
boto_ec2.py	71.98 KB
boto_elasticache.py	16.75 KB
boto_elasticsearch_domain.py	12.27 KB
boto_elb.py	55.1 KB
boto_elbv2.py	12.19 KB
boto_iam.py	69.16 KB
boto_iam_role.py	27.12 KB
boto_iot.py	25.33 KB
boto_kinesis.py	16.69 KB
boto_kms.py	12.11 KB
boto_lambda.py	35.52 KB
boto_lc.py	11.04 KB
boto_rds.py	26 KB
boto_route53.py	19.49 KB
boto_s3.py	9.32 KB
boto_s3_bucket.py	24.67 KB
boto_secgroup.py	32.62 KB
boto_sns.py	8.92 KB
boto_sqs.py	7.97 KB
boto_vpc.py	62.23 KB
bower.py	8.26 KB
btrfs.py	10.34 KB
cabal.py	5.73 KB
ceph.py	1.9 KB
chef.py	3.76 KB
chocolatey.py	16.15 KB
chronos_job.py	4.6 KB
cimc.py	14.32 KB
cisconso.py	3.14 KB
cloud.py	14.4 KB
cmd.py	40.92 KB
composer.py	8.38 KB
consul.py	5.4 KB
cron.py	23.39 KB
cryptdev.py	6.17 KB
csf.py	9.98 KB
cyg.py	7.05 KB
ddns.py	4.2 KB
debconfmod.py	6.33 KB
dellchassis.py	24.49 KB
disk.py	6.49 KB
docker_container.py	85.27 KB
docker_image.py	16.7 KB
docker_network.py	36.78 KB
docker_volume.py	6.72 KB
drac.py	4.17 KB
dvs.py	26.29 KB
elasticsearch.py	20.38 KB
elasticsearch_index.py	3.25 KB
elasticsearch_index_template.py	3.67 KB
environ.py	5.81 KB
eselect.py	2.27 KB
esxcluster.py	22.4 KB
esxdatacenter.py	4.44 KB
esxi.py	63.07 KB
esxvm.py	20.11 KB
etcd_mod.py	11 KB
ethtool.py	9.88 KB
event.py	2.48 KB
file.py	316.7 KB
firewall.py	1.33 KB
firewalld.py	26.08 KB
gem.py	7.13 KB
git.py	123.85 KB
github.py	27.25 KB
glance_image.py	2.26 KB
glassfish.py	21.47 KB
glusterfs.py	12.21 KB
gnomedesktop.py	7.47 KB
gpg.py	5.28 KB
grafana.py	12.11 KB
grafana4_dashboard.py	17.31 KB
grafana4_datasource.py	6.15 KB
grafana4_org.py	7.73 KB
grafana4_user.py	5.52 KB
grafana_dashboard.py	17.74 KB
grafana_datasource.py	5.31 KB
grains.py	15.57 KB
group.py	9.84 KB
heat.py	9.69 KB
helm.py	10.39 KB
hg.py	6.33 KB
highstate_doc.py	1.41 KB
host.py	8.64 KB
http.py	7.46 KB
icinga2.py	9.07 KB
idem.py	3.91 KB
ifttt.py	2.12 KB
incron.py	5.71 KB
influxdb08_database.py	2.85 KB
influxdb08_user.py	3.39 KB
influxdb_continuous_query.py	2.83 KB
influxdb_database.py	2.11 KB
influxdb_retention_policy.py	4.82 KB
influxdb_user.py	4.84 KB
infoblox_a.py	4.24 KB
infoblox_cname.py	4.19 KB
infoblox_host_record.py	6.59 KB
infoblox_range.py	6.85 KB
ini_manage.py	12.67 KB
ipmi.py	8.42 KB
ipset.py	9.66 KB
iptables.py	27.65 KB
jboss7.py	23.95 KB
jenkins.py	3.36 KB
junos.py	17.78 KB
kapacitor.py	6.46 KB
kernelpkg.py	6.42 KB
keyboard.py	2.01 KB
keystone.py	27.12 KB
keystone_domain.py	2.81 KB
keystone_endpoint.py	4.69 KB
keystone_group.py	3.25 KB
keystone_project.py	3.36 KB
keystone_role.py	2.33 KB
keystone_role_grant.py	4.08 KB
keystone_service.py	2.89 KB
keystone_user.py	3.47 KB
keystore.py	5.67 KB
kmod.py	8.59 KB
kubernetes.py	24.87 KB
layman.py	2.44 KB
ldap.py	19.78 KB
libcloud_dns.py	5.7 KB
libcloud_loadbalancer.py	5.66 KB
libcloud_storage.py	5.13 KB
linux_acl.py	24.42 KB
locale.py	2.52 KB
logadm.py	4.67 KB
logrotate.py	3.86 KB
loop.py	7.74 KB
lvm.py	13.33 KB
lvs_server.py	6.28 KB
lvs_service.py	4.38 KB
lxc.py	22.17 KB
lxd.py	7.88 KB
lxd_container.py	22.25 KB
lxd_image.py	10.59 KB
lxd_profile.py	7.11 KB
mac_assistive.py	1.55 KB
mac_keychain.py	5.59 KB
mac_xattr.py	3.15 KB
macdefaults.py	2.65 KB
macpackage.py	6.76 KB
makeconf.py	6.87 KB
marathon_app.py	4.45 KB
mdadm_raid.py	6.41 KB
memcached.py	3.95 KB
modjk.py	2.84 KB
modjk_worker.py	6.49 KB
module.py	18.64 KB
mongodb_database.py	1.65 KB
mongodb_user.py	6.26 KB
monit.py	2.68 KB
mount.py	50.32 KB
mssql_database.py	3 KB
mssql_login.py	3.64 KB
mssql_role.py	2.37 KB
mssql_user.py	3.51 KB
msteams.py	2.53 KB
mysql_database.py	6.05 KB
mysql_grants.py	8.49 KB
mysql_query.py	13.07 KB
mysql_user.py	9.51 KB
net_napalm_yang.py	9.15 KB
netacl.py	31.92 KB
netconfig.py	33.42 KB
netntp.py	12.51 KB
netsnmp.py	11.33 KB
netusers.py	16.1 KB
network.py	23.97 KB
neutron_network.py	3.96 KB
neutron_secgroup.py	4 KB
neutron_secgroup_rule.py	4.75 KB
neutron_subnet.py	4.29 KB
nexus.py	4.97 KB
nfs_export.py	4.92 KB
nftables.py	19.5 KB
npm.py	11.21 KB
ntp.py	2.12 KB
nxos.py	10.37 KB
nxos_upgrade.py	3.5 KB
openstack_config.py	3.26 KB
openvswitch_bridge.py	4.36 KB
openvswitch_db.py	2.24 KB
openvswitch_port.py	17.24 KB
opsgenie.py	4.07 KB
pagerduty.py	1.89 KB
pagerduty_escalation_policy.py	5.42 KB
pagerduty_schedule.py	6.09 KB
pagerduty_service.py	3.93 KB
pagerduty_user.py	1.18 KB
panos.py	48.13 KB
pbm.py	20.46 KB
pcs.py	36.46 KB
pdbedit.py	3.43 KB
pecl.py	3.65 KB
pip_state.py	38.55 KB
pkg.py	138.08 KB
pkgbuild.py	11.37 KB
pkgng.py	685 B
pkgrepo.py	27.53 KB
portage_config.py	5.01 KB
ports.py	5.65 KB
postgres_cluster.py	4.19 KB
postgres_database.py	6.08 KB
postgres_extension.py	5.68 KB
postgres_group.py	8.52 KB
postgres_initdb.py	2.84 KB
postgres_language.py	3.94 KB
postgres_privileges.py	7.86 KB
postgres_schema.py	4.34 KB
postgres_tablespace.py	6.62 KB
postgres_user.py	9.49 KB
powerpath.py	2.34 KB
probes.py	15.06 KB
process.py	1.32 KB
proxy.py	4.94 KB
pushover.py	3.13 KB
pyenv.py	6.07 KB
pyrax_queues.py	2.97 KB
quota.py	1.4 KB
rabbitmq_cluster.py	1.84 KB
rabbitmq_plugin.py	2.77 KB
rabbitmq_policy.py	4.59 KB
rabbitmq_upstream.py	7.9 KB
rabbitmq_user.py	8.89 KB
rabbitmq_vhost.py	3.04 KB
rbac_solaris.py	6.67 KB
rbenv.py	7.36 KB
rdp.py	1.28 KB
redismod.py	4.76 KB
reg.py	19.22 KB
restconf.py	6.41 KB
rsync.py	4.45 KB
rvm.py	6.56 KB
salt_proxy.py	1.34 KB
saltmod.py	33.12 KB
saltutil.py	8.91 KB
schedule.py	12.47 KB
selinux.py	18.61 KB
serverdensity_device.py	6.41 KB
service.py	37.89 KB
slack.py	4.98 KB
smartos.py	44.83 KB
smtp.py	2.3 KB
snapper.py	7.24 KB
solrcloud.py	4.48 KB
splunk.py	4.32 KB
splunk_search.py	3.17 KB
sqlite3.py	14.7 KB
ssh_auth.py	19.57 KB
ssh_known_hosts.py	7.92 KB
stateconf.py	494 B
status.py	2.21 KB
statuspage.py	17.29 KB
supervisord.py	10.48 KB
svn.py	8.14 KB
sysctl.py	4.11 KB
sysfs.py	2.13 KB
syslog_ng.py	2.97 KB
sysrc.py	2.82 KB
telemetry_alert.py	7.04 KB
test.py	13.09 KB
testinframod.py	1.35 KB
timezone.py	3.42 KB
tls.py	1.81 KB
tomcat.py	9.72 KB
trafficserver.py	8.82 KB
tuned.py	3.32 KB
uptime.py	1.87 KB
user.py	38.63 KB
vagrant.py	11.4 KB
vault.py	3.28 KB
vbox_guest.py	4.05 KB
victorops.py	3.32 KB
virt.py	80.41 KB
virtualenv_mod.py	11.21 KB
webutil.py	3.89 KB
win_certutil.py	4.8 KB
win_dacl.py	7.96 KB
win_dism.py	14.97 KB
win_dns_client.py	8.32 KB
win_firewall.py	6.87 KB
win_iis.py	31.56 KB
win_lgpo.py	24.99 KB
win_lgpo_reg.py	10.96 KB
win_license.py	1.6 KB
win_network.py	14.18 KB
win_path.py	6.39 KB
win_pki.py	5.56 KB
win_powercfg.py	3.79 KB
win_servermanager.py	10.4 KB
win_shortcut.py	7.81 KB
win_smtp_server.py	10.01 KB
win_snmp.py	6.64 KB
win_system.py	13.78 KB
win_wua.py	16.27 KB
win_wusa.py	3.53 KB
winrepo.py	2.74 KB
wordpress.py	4.82 KB
x509.py	27.86 KB
x509_v2.py	64.78 KB
xml.py	1.75 KB
xmpp.py	2.61 KB
zabbix_action.py	9.35 KB
zabbix_host.py	27.25 KB
zabbix_hostgroup.py	5.64 KB
zabbix_mediatype.py	16.89 KB
zabbix_template.py	35.14 KB
zabbix_user.py	17.6 KB
zabbix_usergroup.py	9.64 KB
zabbix_usermacro.py	9.69 KB
zabbix_valuemap.py	8.11 KB
zcbuildout.py	5.16 KB
zenoss.py	2.89 KB
zfs.py	34.48 KB
zk_concurrency.py	5.81 KB
zone.py	46.48 KB
zookeeper.py	11.55 KB
zpool.py	13.4 KB

Edit: boto_dynamodb.py

"""
Manage DynamoDB Tables
======================

.. versionadded:: 2015.5.0

Create and destroy DynamoDB tables. Be aware that this interacts with Amazon's
services, and so may incur charges.

This module uses ``boto``, which can be installed via package, or pip.

This module accepts explicit DynamoDB credentials but can also utilize
IAM roles assigned to the instance through Instance Profiles. Dynamic
credentials are then automatically obtained from AWS API and no further
configuration is necessary. More information available `here
<http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html>`_.

If IAM roles are not used you need to specify them either in a pillar file or
in the minion's config file:

.. code-block:: yaml

keyid: GKTADJGHEIQSXMKKRBJ08H
    key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs
    region: us-east-1

It's also possible to specify ``key``, ``keyid`` and ``region`` via a
profile, either passed in as a dict, or as a string to pull from
pillars or minion config:

.. code-block:: yaml

myprofile:
        keyid: GKTADJGHEIQSXMKKRBJ08H
        key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs
            region: us-east-1

.. code-block:: yaml

Ensure DynamoDB table does not exist:
      boto_dynamodb.absent:
        - table_name: new_table
        - keyid: GKTADJGHEIQSXMKKRBJ08H
        - key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs
        - region: us-east-1

Ensure DynamoDB table exists:
      boto_dynamodb.present:
        - table_name: new_table
        - read_capacity_units: 1
        - write_capacity_units: 2
        - hash_key: primary_id
        - hash_key_data_type: N
        - range_key: start_timestamp
        - range_key_data_type: N
        - keyid: GKTADJGHEIQSXMKKRBJ08H
        - key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs
        - region: us-east-1
        - local_indexes:
            - index:
                - name: "primary_id_end_timestamp_index"
                - hash_key: primary_id
                - hash_key_data_type: N
                - range_key: end_timestamp
                - range_key_data_type: N
        - global_indexes:
            - index:
                - name: "name_end_timestamp_index"
                - hash_key: name
                - hash_key_data_type: S
                - range_key: end_timestamp
                - range_key_data_type: N
                - read_capacity_units: 3
                - write_capacity_units: 4

It's possible to specify cloudwatch alarms that will be setup along with the
DynamoDB table. Note the alarm name will be defined by the name attribute
provided, plus the DynamoDB resource name.

.. code-block:: yaml

Ensure DynamoDB table exists:
      boto_dynamodb.present:
        - name: new_table
        - read_capacity_units: 1
        - write_capacity_units: 2
        - hash_key: primary_id
        - hash_key_data_type: N
        - range_key: start_timestamp
        - range_key_data_type: N
        - alarms:
             ConsumedWriteCapacityUnits:
                name: 'DynamoDB ConsumedWriteCapacityUnits **MANAGED BY SALT**'
                attributes:
                  metric: ConsumedWriteCapacityUnits
                  namespace: AWS/DynamoDB
                  statistic: Sum
                  comparison: '>='
                  # threshold_percent is used to calculate the actual threshold
                  # based on the provisioned capacity for the table.
                  threshold_percent: 0.75
                  period: 300
                  evaluation_periods: 2
                  unit: Count
                  description: 'DynamoDB ConsumedWriteCapacityUnits'
                  alarm_actions: [ 'arn:aws:sns:us-east-1:1234:my-alarm' ]
                  insufficient_data_actions: []
                  ok_actions: [ 'arn:aws:sns:us-east-1:1234:my-alarm' ]
        - keyid: GKTADJGHEIQSXMKKRBJ08H
        - key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs
        - region: us-east-1

You can also use alarms from pillars, and override values from the pillar
alarms by setting overrides on the resource. Note that 'boto_dynamodb_alarms'
will be used as a default value for all resources, if defined and can be
used to ensure alarms are always set for a resource.

Setting the alarms in a pillar:

.. code-block:: yaml

boto_dynamodb_alarms:
      ConsumedWriteCapacityUnits:
        name: 'DynamoDB ConsumedWriteCapacityUnits **MANAGED BY SALT**'
        attributes:
          metric: ConsumedWriteCapacityUnits
          namespace: AWS/DynamoDB
          statistic: Sum
          comparison: '>='
          # threshold_percent is used to calculate the actual threshold
          # based on the provisioned capacity for the table.
          threshold_percent: 0.75
          period: 300
          evaluation_periods: 2
          unit: Count
          description: 'DynamoDB ConsumedWriteCapacityUnits'
          alarm_actions: [ 'arn:aws:sns:us-east-1:1234:my-alarm' ]
          insufficient_data_actions: []
          ok_actions: [ 'arn:aws:sns:us-east-1:1234:my-alarm' ]

import copy
import datetime
import logging
import math
import sys

import salt.utils.dictupdate as dictupdate

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s %(name)s %(levelname)s %(message)s",
    stream=sys.stdout,
)
log = logging.getLogger()

class GsiNotUpdatableError(Exception):
    """Raised when a global secondary index cannot be updated."""

def __virtual__():
    """
    Only load if boto_dynamodb is available.
    """
    if "boto_dynamodb.exists" in __salt__:
        return "boto_dynamodb"
    return (False, "boto_dynamodb module could not be loaded")

def present(
    name=None,
    table_name=None,
    region=None,
    key=None,
    keyid=None,
    profile=None,
    read_capacity_units=None,
    write_capacity_units=None,
    alarms=None,
    alarms_from_pillar="boto_dynamodb_alarms",
    hash_key=None,
    hash_key_data_type=None,
    range_key=None,
    range_key_data_type=None,
    local_indexes=None,
    global_indexes=None,
    backup_configs_from_pillars="boto_dynamodb_backup_configs",
):
    """
    Ensure the DynamoDB table exists. Table throughput can be updated after
    table creation.

Global secondary indexes (GSIs) are managed with some exceptions:

- If a GSI deletion is detected, a failure will occur (deletes should be
      done manually in the AWS console).

- If multiple GSIs are added in a single Salt call, a failure will occur
      (boto supports one creation at a time). Note that this only applies after
      table creation; multiple GSIs can be created during table creation.

- Updates to existing GSIs are limited to read/write capacity only
      (DynamoDB limitation).

name
        Name of the DynamoDB table

table_name
        Name of the DynamoDB table (deprecated)

region
        Region to connect to.

key
        Secret key to be used.

keyid
        Access key to be used.

profile
        A dict with region, key and keyid, or a pillar key (string)
        that contains a dict with region, key and keyid.

read_capacity_units
        The read throughput for this table

write_capacity_units
        The write throughput for this table

hash_key
        The name of the attribute that will be used as the hash key
        for this table

hash_key_data_type
        The DynamoDB datatype of the hash key

range_key
        The name of the attribute that will be used as the range key
        for this table

range_key_data_type
        The DynamoDB datatype of the range key

local_indexes
        The local indexes you would like to create

global_indexes
        The global indexes you would like to create

backup_configs_from_pillars
        Pillars to use to configure DataPipeline backups
    """
    ret = {"name": name, "result": True, "comment": "", "changes": {}}
    if table_name:
        ret["warnings"] = [
            "boto_dynamodb.present: `table_name` is deprecated."
            " Please use `name` instead."
        ]
        ret["name"] = table_name
        name = table_name

comments = []
    changes_old = {}
    changes_new = {}

# Ensure DynamoDB table exists
    table_exists = __salt__["boto_dynamodb.exists"](name, region, key, keyid, profile)
    if not table_exists:
        if __opts__["test"]:
            ret["result"] = None
            ret["comment"] = "DynamoDB table {} would be created.".format(name)
            return ret
        else:
            is_created = __salt__["boto_dynamodb.create_table"](
                name,
                region,
                key,
                keyid,
                profile,
                read_capacity_units,
                write_capacity_units,
                hash_key,
                hash_key_data_type,
                range_key,
                range_key_data_type,
                local_indexes,
                global_indexes,
            )
            if not is_created:
                ret["result"] = False
                ret["comment"] = "Failed to create table {}".format(name)
                _add_changes(ret, changes_old, changes_new)
                return ret

comments.append("DynamoDB table {} was successfully created".format(name))
            changes_new["table"] = name
            changes_new["read_capacity_units"] = read_capacity_units
            changes_new["write_capacity_units"] = write_capacity_units
            changes_new["hash_key"] = hash_key
            changes_new["hash_key_data_type"] = hash_key_data_type
            changes_new["range_key"] = range_key
            changes_new["range_key_data_type"] = range_key_data_type
            changes_new["local_indexes"] = local_indexes
            changes_new["global_indexes"] = global_indexes
    else:
        comments.append("DynamoDB table {} exists".format(name))

# Ensure DynamoDB table provisioned throughput matches
    description = __salt__["boto_dynamodb.describe"](name, region, key, keyid, profile)
    provisioned_throughput = description.get("Table", {}).get(
        "ProvisionedThroughput", {}
    )
    current_write_capacity_units = provisioned_throughput.get("WriteCapacityUnits")
    current_read_capacity_units = provisioned_throughput.get("ReadCapacityUnits")
    throughput_matches = (
        current_write_capacity_units == write_capacity_units
        and current_read_capacity_units == read_capacity_units
    )
    if not throughput_matches:
        if __opts__["test"]:
            ret["result"] = None
            comments.append("DynamoDB table {} is set to be updated.".format(name))
        else:
            is_updated = __salt__["boto_dynamodb.update"](
                name,
                throughput={
                    "read": read_capacity_units,
                    "write": write_capacity_units,
                },
                region=region,
                key=key,
                keyid=keyid,
                profile=profile,
            )
            if not is_updated:
                ret["result"] = False
                ret["comment"] = "Failed to update table {}".format(name)
                _add_changes(ret, changes_old, changes_new)
                return ret

comments.append("DynamoDB table {} was successfully updated".format(name))
            changes_old["read_capacity_units"] = (current_read_capacity_units,)
            changes_old["write_capacity_units"] = (current_write_capacity_units,)
            changes_new["read_capacity_units"] = (read_capacity_units,)
            changes_new["write_capacity_units"] = (write_capacity_units,)
    else:
        comments.append("DynamoDB table {} throughput matches".format(name))

provisioned_indexes = description.get("Table", {}).get("GlobalSecondaryIndexes", [])

_ret = _global_indexes_present(
        provisioned_indexes,
        global_indexes,
        changes_old,
        changes_new,
        comments,
        name,
        region,
        key,
        keyid,
        profile,
    )
    if not _ret["result"]:
        comments.append(_ret["comment"])
        ret["result"] = _ret["result"]
        if ret["result"] is False:
            ret["comment"] = ",\n".join(comments)
            _add_changes(ret, changes_old, changes_new)
            return ret

_ret = _alarms_present(
        name,
        alarms,
        alarms_from_pillar,
        write_capacity_units,
        read_capacity_units,
        region,
        key,
        keyid,
        profile,
    )
    ret["changes"] = dictupdate.update(ret["changes"], _ret["changes"])
    comments.append(_ret["comment"])
    if not _ret["result"]:
        ret["result"] = _ret["result"]
        if ret["result"] is False:
            ret["comment"] = ",\n".join(comments)
            _add_changes(ret, changes_old, changes_new)
            return ret

# Ensure backup datapipeline is present
    datapipeline_configs = copy.deepcopy(
        __salt__["pillar.get"](backup_configs_from_pillars, [])
    )
    for config in datapipeline_configs:
        datapipeline_ret = _ensure_backup_datapipeline_present(
            name=name,
            schedule_name=config["name"],
            period=config["period"],
            utc_hour=config["utc_hour"],
            s3_base_location=config["s3_base_location"],
        )
        # Add comments and changes if successful changes were made (True for live mode,
        # None for test mode).
        if datapipeline_ret["result"] in [True, None]:
            ret["result"] = datapipeline_ret["result"]
            comments.append(datapipeline_ret["comment"])
            if datapipeline_ret.get("changes"):
                ret["changes"]["backup_datapipeline_{}".format(config["name"])] = (
                    datapipeline_ret.get("changes"),
                )
        else:
            ret["comment"] = ",\n".join([ret["comment"], datapipeline_ret["comment"]])
            _add_changes(ret, changes_old, changes_new)
            return ret

ret["comment"] = ",\n".join(comments)
    _add_changes(ret, changes_old, changes_new)
    return ret

def _add_changes(ret, changes_old, changes_new):
    if changes_old:
        ret["changes"]["old"] = changes_old
    if changes_new:
        ret["changes"]["new"] = changes_new

def _global_indexes_present(
    provisioned_indexes,
    global_indexes,
    changes_old,
    changes_new,
    comments,
    name,
    region,
    key,
    keyid,
    profile,
):
    """Handles global secondary index for the table present state."""
    ret = {"result": True}
    if provisioned_indexes:
        provisioned_gsi_config = {
            index["IndexName"]: index for index in provisioned_indexes
        }
    else:
        provisioned_gsi_config = {}
    provisioned_index_names = set(provisioned_gsi_config.keys())

# Map of index name to given Salt config for this run. This loop is complicated
    # because global_indexes is made up of OrderedDicts and lists.
    gsi_config = {}
    if global_indexes:
        for index in global_indexes:
            # Each index config is a key that maps to a list of OrderedDicts.
            index_config = next(iter(index.values()))
            index_name = None
            for entry in index_config:
                # Key by the name field in the index config.
                if entry.keys() == ["name"]:
                    index_name = next(iter(entry.values()))
            if not index_name:
                ret["result"] = False
                ret["comment"] = "Index name not found for table {}".format(name)
                return ret
            gsi_config[index_name] = index

(
        existing_index_names,
        new_index_names,
        index_names_to_be_deleted,
    ) = _partition_index_names(provisioned_index_names, set(gsi_config.keys()))

if index_names_to_be_deleted:
        ret["result"] = False
        ret["comment"] = (
            "Deletion of GSIs ({}) is not supported! Please do this "
            "manually in the AWS console.".format(", ".join(index_names_to_be_deleted))
        )
        return ret
    elif len(new_index_names) > 1:
        ret["result"] = False
        ret["comment"] = (
            "Creation of multiple GSIs ({}) is not supported due to API "
            "limitations. Please create them one at a time.".format(new_index_names)
        )
        return ret

if new_index_names:
        # Given the length check above, new_index_names should have a single element here.
        index_name = next(iter(new_index_names))
        _add_global_secondary_index(
            ret,
            name,
            index_name,
            changes_old,
            changes_new,
            comments,
            gsi_config,
            region,
            key,
            keyid,
            profile,
        )
        if not ret["result"]:
            return ret

if existing_index_names:
        _update_global_secondary_indexes(
            ret,
            changes_old,
            changes_new,
            comments,
            existing_index_names,
            provisioned_gsi_config,
            gsi_config,
            name,
            region,
            key,
            keyid,
            profile,
        )
        if not ret["result"]:
            return ret

if "global_indexes" not in changes_old and "global_indexes" not in changes_new:
        comments.append("All global secondary indexes match")
    return ret

def _partition_index_names(provisioned_index_names, index_names):
    """Returns 3 disjoint sets of indexes: existing, to be created, and to be deleted."""
    existing_index_names = set()
    new_index_names = set()
    for name in index_names:
        if name in provisioned_index_names:
            existing_index_names.add(name)
        else:
            new_index_names.add(name)
    index_names_to_be_deleted = provisioned_index_names - existing_index_names
    return existing_index_names, new_index_names, index_names_to_be_deleted

def _add_global_secondary_index(
    ret,
    name,
    index_name,
    changes_old,
    changes_new,
    comments,
    gsi_config,
    region,
    key,
    keyid,
    profile,
):
    """Updates ret iff there was a failure or in test mode."""
    if __opts__["test"]:
        ret["result"] = None
        ret["comment"] = "Dynamo table {} will have a GSI added: {}".format(
            name, index_name
        )
        return
    changes_new.setdefault("global_indexes", {})
    success = __salt__["boto_dynamodb.create_global_secondary_index"](
        name,
        __salt__["boto_dynamodb.extract_index"](
            gsi_config[index_name], global_index=True
        ),
        region=region,
        key=key,
        keyid=keyid,
        profile=profile,
    )

if success:
        comments.append("Created GSI {}".format(index_name))
        changes_new["global_indexes"][index_name] = gsi_config[index_name]
    else:
        ret["result"] = False
        ret["comment"] = "Failed to create GSI {}".format(index_name)

def _update_global_secondary_indexes(
    ret,
    changes_old,
    changes_new,
    comments,
    existing_index_names,
    provisioned_gsi_config,
    gsi_config,
    name,
    region,
    key,
    keyid,
    profile,
):
    """Updates ret iff there was a failure or in test mode."""
    try:
        provisioned_throughputs, index_updates = _determine_gsi_updates(
            existing_index_names, provisioned_gsi_config, gsi_config
        )
    except GsiNotUpdatableError as e:
        ret["result"] = False
        ret["comment"] = str(e)
        return

if index_updates:
        if __opts__["test"]:
            ret["result"] = None
            ret["comment"] = "Dynamo table {} will have GSIs updated: {}".format(
                name, ", ".join(index_updates.keys())
            )
            return
        changes_old.setdefault("global_indexes", {})
        changes_new.setdefault("global_indexes", {})
        success = __salt__["boto_dynamodb.update_global_secondary_index"](
            name,
            index_updates,
            region=region,
            key=key,
            keyid=keyid,
            profile=profile,
        )

if success:
            comments.append(
                "Updated GSIs with new throughputs {}".format(index_updates)
            )
            for index_name in index_updates:
                changes_old["global_indexes"][index_name] = provisioned_throughputs[
                    index_name
                ]
                changes_new["global_indexes"][index_name] = index_updates[index_name]
        else:
            ret["result"] = False
            ret["comment"] = "Failed to update GSI throughputs {}".format(index_updates)

def _determine_gsi_updates(existing_index_names, provisioned_gsi_config, gsi_config):
    # index name -> {'read': <read throughput>, 'write': <write throughput>}
    provisioned_throughputs = {}
    index_updates = {}
    for index_name in existing_index_names:
        current_config = provisioned_gsi_config[index_name]
        new_config = __salt__["boto_dynamodb.extract_index"](
            gsi_config[index_name], global_index=True
        ).schema()

# The provisioned config will have more fields than the new config, so only consider
        # fields in new_config.
        for key in new_config:
            if key in current_config and key != "ProvisionedThroughput":
                new_value = new_config[key]
                current_value = current_config[key]
                # This is a special case since the Projection value can contain a list (not
                # correctly comparable with == as order doesn't matter).
                if key == "Projection":
                    if new_value["ProjectionType"] != current_value["ProjectionType"]:
                        raise GsiNotUpdatableError("GSI projection types do not match")
                    elif set(new_value.get("NonKeyAttributes", [])) != set(
                        current_value.get("NonKeyAttributes", [])
                    ):
                        raise GsiNotUpdatableError(
                            "NonKeyAttributes do not match for GSI projection"
                        )
                elif new_value != current_value:
                    raise GsiNotUpdatableError(
                        "GSI property {} cannot be updated for index {}".format(
                            key, index_name
                        )
                    )

current_throughput = current_config.get("ProvisionedThroughput")
        current_read = current_throughput.get("ReadCapacityUnits")
        current_write = current_throughput.get("WriteCapacityUnits")
        provisioned_throughputs[index_name] = {
            "read": current_read,
            "write": current_write,
        }
        new_throughput = new_config.get("ProvisionedThroughput")
        new_read = new_throughput.get("ReadCapacityUnits")
        new_write = new_throughput.get("WriteCapacityUnits")
        if current_read != new_read or current_write != new_write:
            index_updates[index_name] = {"read": new_read, "write": new_write}

return provisioned_throughputs, index_updates

def _alarms_present(
    name,
    alarms,
    alarms_from_pillar,
    write_capacity_units,
    read_capacity_units,
    region,
    key,
    keyid,
    profile,
):
    """helper method for present.  ensure that cloudwatch_alarms are set"""
    # load data from alarms_from_pillar
    tmp = copy.deepcopy(__salt__["config.option"](alarms_from_pillar, {}))
    # merge with data from alarms
    if alarms:
        tmp = dictupdate.update(tmp, alarms)
    # set alarms, using boto_cloudwatch_alarm.present
    merged_return_value = {"name": name, "result": True, "comment": "", "changes": {}}
    for _, info in tmp.items():
        # add dynamodb table to name and description
        info["name"] = name + " " + info["name"]
        info["attributes"]["description"] = (
            name + " " + info["attributes"]["description"]
        )
        # add dimension attribute
        info["attributes"]["dimensions"] = {"TableName": [name]}
        if (
            info["attributes"]["metric"] == "ConsumedWriteCapacityUnits"
            and "threshold" not in info["attributes"]
        ):
            info["attributes"]["threshold"] = math.ceil(
                write_capacity_units * info["attributes"]["threshold_percent"]
            )
            del info["attributes"]["threshold_percent"]
            # the write_capacity_units is given in unit / second. So we need
            # to multiply by the period to get the proper threshold.
            # http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/MonitoringDynamoDB.html
            info["attributes"]["threshold"] *= info["attributes"]["period"]
        if (
            info["attributes"]["metric"] == "ConsumedReadCapacityUnits"
            and "threshold" not in info["attributes"]
        ):
            info["attributes"]["threshold"] = math.ceil(
                read_capacity_units * info["attributes"]["threshold_percent"]
            )
            del info["attributes"]["threshold_percent"]
            # the read_capacity_units is given in unit / second. So we need
            # to multiply by the period to get the proper threshold.
            # http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/MonitoringDynamoDB.html
            info["attributes"]["threshold"] *= info["attributes"]["period"]
        # set alarm
        kwargs = {
            "name": info["name"],
            "attributes": info["attributes"],
            "region": region,
            "key": key,
            "keyid": keyid,
            "profile": profile,
        }
        results = __states__["boto_cloudwatch_alarm.present"](**kwargs)
        if not results["result"]:
            merged_return_value["result"] = results["result"]
        if results.get("changes", {}) != {}:
            merged_return_value["changes"][info["name"]] = results["changes"]
        if "comment" in results:
            merged_return_value["comment"] += results["comment"]
    return merged_return_value

def _ensure_backup_datapipeline_present(
    name, schedule_name, period, utc_hour, s3_base_location
):

kwargs = {
        "name": "{}-{}-backup".format(name, schedule_name),
        "pipeline_objects": {
            "DefaultSchedule": {
                "name": schedule_name,
                "fields": {
                    "period": period,
                    "type": "Schedule",
                    "startDateTime": _next_datetime_with_utc_hour(
                        name, utc_hour
                    ).isoformat(),
                },
            },
        },
        "parameter_values": {
            "myDDBTableName": name,
            "myOutputS3Loc": "{}/{}/".format(s3_base_location, name),
        },
    }
    return __states__["boto_datapipeline.present"](**kwargs)

def _get_deterministic_value_for_table_name(table_name, max_value):
    """
    For a given table_name, returns hash of the table_name limited by max_value.
    """
    return hash(table_name) % max_value

def _next_datetime_with_utc_hour(table_name, utc_hour):
    """
    Datapipeline API is throttling us, as all the pipelines are started at the same time.
    We would like to uniformly distribute the startTime over a 60 minute window.

Return the next future utc datetime where
        hour == utc_hour
        minute = A value between 0-59 (depending on table name)
        second = A value between 0-59 (depending on table name)
    """
    today = datetime.date.today()

# The minute and second values generated are deterministic, as we do not want
    # pipeline definition to change for every run.
    start_date_time = datetime.datetime(
        year=today.year,
        month=today.month,
        day=today.day,
        hour=utc_hour,
        minute=_get_deterministic_value_for_table_name(table_name, 60),
        second=_get_deterministic_value_for_table_name(table_name, 60),
    )

if start_date_time < datetime.datetime.utcnow():
        one_day = datetime.timedelta(days=1)
        start_date_time += one_day

return start_date_time

def absent(name, region=None, key=None, keyid=None, profile=None):
    """
    Ensure the DynamoDB table does not exist.

name
        Name of the DynamoDB table.

region
        Region to connect to.

key
        Secret key to be used.

keyid
        Access key to be used.

profile
        A dict with region, key and keyid, or a pillar key (string)
        that contains a dict with region, key and keyid.
    """
    ret = {"name": name, "result": True, "comment": "", "changes": {}}
    exists = __salt__["boto_dynamodb.exists"](name, region, key, keyid, profile)
    if not exists:
        ret["comment"] = "DynamoDB table {} does not exist".format(name)
        return ret

if __opts__["test"]:
        ret["comment"] = "DynamoDB table {} is set to be deleted".format(name)
        ret["result"] = None
        return ret

is_deleted = __salt__["boto_dynamodb.delete"](name, region, key, keyid, profile)
    if is_deleted:
        ret["comment"] = "Deleted DynamoDB table {}".format(name)
        ret["changes"].setdefault("old", "Table {} exists".format(name))
        ret["changes"].setdefault("new", "Table {} deleted".format(name))
    else:
        ret["comment"] = "Failed to delete DynamoDB table {}".format(name)
        ret["result"] = False
    return ret