169163334 - Initial secrets-tool commit

Adds admin_users map and keyvault policy

This adds an admin_users map as well as a new policy in the keyvault
module. When run, this will apply an administrator policy for users in
the admin_users map. With these permissions, the admin users will be
able to manage secrets and keys in keyvault.

169163334 - Initial secrets-tool commit

Adds admin_users map and keyvault policy

This adds an admin_users map as well as a new policy in the keyvault
module. When run, this will apply an administrator policy for users in
the admin_users map. With these permissions, the admin users will be
able to manage secrets and keys in keyvault.

170237669 - Makes the read only policy for keyvault optional and only create the policy if a principal_id is passed

170237669 - Adds new operator keyvault for secrets

This is a new keyvault specifically for storing operator secrets and
things that would not be accessible to applications. The primary use
case for this is for launching things like postgres (root postgres
creds) and other services which would require secrets to be added to the
terraform configuration. This approach avoids adding secrets to
terraform.

An accompanying script will be added to populate the new keyvault.

terraform/.gitignore

@@ -1 +1,2 @@
 .terraform
+.vscode/

terraform/modules/keyvault/main.tf

@@ -19,7 +19,8 @@ resource "azurerm_key_vault" "keyvault" {
   }
 }
 
-resource "azurerm_key_vault_access_policy" "keyvault" {
+resource "azurerm_key_vault_access_policy" "keyvault_k8s_policy" {
+  count        = length(var.principal_id) > 0 ? 1 : 0
   key_vault_id = azurerm_key_vault.keyvault.id
 
   tenant_id = data.azurerm_client_config.current.tenant_id
@@ -34,3 +35,25 @@ resource "azurerm_key_vault_access_policy" "keyvault" {
   ]
 }
 
+# Admin Access
+resource "azurerm_key_vault_access_policy" "keyvault_admin_policy" {
+  for_each     = var.admin_principals
+  key_vault_id = azurerm_key_vault.keyvault.id
+
+  tenant_id = data.azurerm_client_config.current.tenant_id
+  object_id = each.value
+
+  key_permissions = [
+    "get",
+    "list",
+    "create",
+    "update",
+    "delete",
+  ]
+
+  secret_permissions = [
+    "get",
+    "list",
+    "set",
+  ]
+}

terraform/modules/keyvault/variables.tf

@@ -27,3 +27,8 @@ variable "principal_id" {
   type        = string
   description = "The service principal_id of the k8s cluster"
 }
+
+variable "admin_principals" {
+  type        = map
+  description = "A list of user principals who need access to manage the keyvault"
+}

terraform/providers/dev/keyvault.tf

@@ -1,9 +1,11 @@
 module "keyvault" {
-  source       = "../../modules/keyvault"
-  name         = var.name
-  region       = var.region
-  owner        = var.owner
-  environment  = var.environment
-  tenant_id    = var.tenant_id
-  principal_id = "f9bcbe58-8b73-4957-aee2-133dc3e58063"
+  source           = "../../modules/keyvault"
+  name             = var.name
+  region           = var.region
+  owner            = var.owner
+  environment      = var.environment
+  tenant_id        = var.tenant_id
+  principal_id     = "f9bcbe58-8b73-4957-aee2-133dc3e58063"
+  admin_principals = var.admin_users
 }
+

terraform/providers/dev/variables.tf

@@ -71,3 +71,11 @@ variable "tenant_id" {
   type    = string
   default = "b5ab0e1e-09f8-4258-afb7-fb17654bc5b3"
 }
+
+variable "admin_users" {
+  type = map
+  default = {
+    "Rob Gil"         = "2ca63d41-d058-4e06-aef6-eb517a53b631"
+    "Daniel Corrigan" = "d5bb69c2-3b88-4e96-b1a2-320400f1bf1b"
+  }
+}

terraform/secrets-tool/.gitignore

@@ -0,0 +1,4 @@
+bin/
+include/
+lib/
+

terraform/secrets-tool/commands/secrets.py

@@ -0,0 +1,35 @@
+import click
+import logging
+from utils.keyvault.secrets import SecretsClient
+
+logger = logging.getLogger(__name__)
+
+#loggers = [logging.getLogger(name) for name in logging.root.manager.loggerDict]
+#print(loggers)
+
+@click.group()
+@click.option('--keyvault', required=True, help="Specify the keyvault to operate on")
+@click.pass_context
+def secrets(ctx, keyvault):
+    ctx.ensure_object(dict)
+    ctx.obj['keyvault'] = keyvault
+
+@click.command('create')
+@click.option('--key', 'key', required=True, help="Key for the secret to create")
+@click.option('--value', 'value', required=True, prompt=True, hide_input=True, confirmation_prompt=True, help="Value for the secret to create")
+@click.pass_context
+def create_secret(ctx, key, value):
+    """Creates a secret in the specified KeyVault"""
+    keyvault = SecretsClient(vault_url=ctx.obj['keyvault'])
+    keyvault.set_secret(key, value)
+
+@click.command('list')
+@click.pass_context
+def list_secrets(ctx):
+    """Lists the secrets in the specified KeyVault"""
+    keyvault = SecretsClient(vault_url=ctx.obj['keyvault'])
+    click.echo(keyvault.list_secrets())
+
+
+secrets.add_command(create_secret)
+secrets.add_command(list_secrets)

terraform/secrets-tool/commands/terraform.py

@@ -0,0 +1,67 @@
+import os
+import click
+import logging
+import subprocess
+
+from utils.keyvault.secrets import SecretsClient
+
+logger = logging.getLogger(__name__)
+
+PROCESS='terraform'
+
+@click.group()
+@click.pass_context
+def terraform(ctx):
+    pass
+
+@click.command('plan')
+@click.pass_context
+def plan(ctx):
+    keyvault = SecretsClient(vault_url="https://cloudzero-dev-keyvault.vault.azure.net/")
+    # Set env variables for TF
+    for secret in keyvault.list_secrets():
+        name = 'TF_VAR_' + secret
+        val = keyvault.get_secret(secret)
+        #print(val)
+        os.environ[name] = val
+    env = os.environ.copy() 
+    command = "{} {}".format(PROCESS, 'plan')
+    with subprocess.Popen(command, env=env, stdout=subprocess.PIPE, shell=True) as proc:
+        for line in proc.stdout:
+            logging.info(line.decode("utf-8") )
+
+@click.command('apply')
+@click.pass_context
+def apply(ctx):
+    keyvault = SecretsClient(vault_url="https://cloudzero-dev-keyvault.vault.azure.net/")
+    # Set env variables for TF
+    for secret in keyvault.list_secrets():
+        name = 'TF_VAR_' + secret
+        val = keyvault.get_secret(secret)
+        #print(val)
+        os.environ[name] = val
+    env = os.environ.copy() 
+    command = "{} {}".format(PROCESS, 'apply -auto-approve')
+    with subprocess.Popen(command, env=env, stdout=subprocess.PIPE, shell=True) as proc:
+        for line in proc.stdout:
+            logging.info(line.decode("utf-8") )
+
+@click.command('destroy')
+@click.pass_context
+def destroy(ctx):
+    keyvault = SecretsClient(vault_url="https://cloudzero-dev-keyvault.vault.azure.net/")
+    # Set env variables for TF
+    for secret in keyvault.list_secrets():
+        name = 'TF_VAR_' + secret
+        val = keyvault.get_secret(secret)
+        #print(val)
+        os.environ[name] = val
+    env = os.environ.copy() 
+    command = "{} {}".format(PROCESS, 'destroy')
+    with subprocess.Popen(command, env=env, stdout=subprocess.PIPE, shell=True) as proc:
+        for line in proc.stdout:
+            logging.info(line.decode("utf-8") )
+
+terraform.add_command(plan)
+terraform.add_command(apply)
+terraform.add_command(destroy)

terraform/secrets-tool/config.py

@@ -0,0 +1,28 @@
+import os
+import yaml
+import logging.config
+import logging
+import coloredlogs
+
+LOGGING_PATH=os.path.dirname(os.path.abspath(__file__))
+
+def setup_logging(default_path='{}/logging.yaml'.format(LOGGING_PATH), default_level=logging.INFO, env_key='LOG_CFG'):
+    path = default_path
+    value = os.getenv(env_key, None)
+    if value:
+        path = value
+    if os.path.exists(path):
+        with open(path, 'rt') as f:
+            try:
+                config = yaml.safe_load(f.read())
+                logging.config.dictConfig(config)
+                coloredlogs.install()
+            except Exception as e:
+                print(e)
+                print('Error in Logging Configuration. Using default configs')
+                logging.basicConfig(level=default_level)
+                coloredlogs.install(level=default_level)
+    else:
+        logging.basicConfig(level=default_level)
+        coloredlogs.install(level=default_level)
+        print('Failed to load configuration file. Using default configs')

terraform/secrets-tool/logging.yaml

@@ -0,0 +1,60 @@
+version: 1
+disable_existing_loggers: true
+
+formatters:
+    standard:
+        format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+    error:
+        format: "%(levelname)s <PID %(process)d:%(processName)s> %(name)s.%(funcName)s(): %(message)s"
+
+handlers:
+    console:
+        class: logging.StreamHandler
+        level: DEBUG
+        formatter: standard
+        stream: ext://sys.stdout
+
+    file_handler:
+        class: logging.handlers.RotatingFileHandler
+        level: INFO
+        formatter: standard
+        filename: secrets-tool.log
+        maxBytes: 10485760 # 10MB
+        backupCount: 20
+        encoding: utf8
+
+root:
+    level: INFO
+    handlers: [console]
+    propogate: yes
+
+loggers:
+    root:
+        level: INFO
+        handlers: [console]
+        propogate: no
+    click:
+        level: INFO
+        handlers: [console]
+        propogate: yes
+    azure.keyvault:
+        level: INFO
+        handlers: [console]
+        propogate: yes
+    azure.core:
+        level: ERROR
+        handlers: [console]
+        propogate: no
+    utils.keyvault.secrets:
+        level: DEBUG
+        handlers: [console]
+        propogate: yes
+    commands:
+        level: INFO
+        handlers: [console]
+        propogate: yes
+    main:
+        level: INFO
+        handlers: [console]
+        propogate: no
+

terraform/secrets-tool/requirements.txt

@@ -0,0 +1,54 @@
+adal==1.2.2
+antlr4-python3-runtime==4.7.2
+applicationinsights==0.11.9
+argcomplete==1.10.3
+astroid==2.3.3
+azure-cli-core==2.0.77
+azure-cli-nspkg==3.0.4
+azure-cli-telemetry==1.0.4
+azure-common==1.1.23
+azure-core==1.1.1
+azure-identity==1.1.0
+azure-keyvault==4.0.0
+azure-keyvault-keys==4.0.0
+azure-keyvault-secrets==4.0.0
+azure-mgmt-resource==4.0.0
+azure-nspkg==3.0.2
+bcrypt==3.1.7
+certifi==2019.11.28
+cffi==1.13.2
+chardet==3.0.4
+Click==7.0
+colorama==0.4.3
+coloredlogs==10.0
+cryptography==2.8
+humanfriendly==4.18
+idna==2.8
+isodate==0.6.0
+isort==4.3.21
+jmespath==0.9.4
+knack==0.6.3
+lazy-object-proxy==1.4.3
+mccabe==0.6.1
+msal==1.0.0
+msal-extensions==0.1.3
+msrest==0.6.10
+msrestazure==0.6.2
+oauthlib==3.1.0
+paramiko==2.7.1
+portalocker==1.5.2
+pycparser==2.19
+Pygments==2.5.2
+PyJWT==1.7.1
+pylint==2.4.4
+PyNaCl==1.3.0
+pyOpenSSL==19.1.0
+python-dateutil==2.8.1
+PyYAML==5.2
+requests==2.22.0
+requests-oauthlib==1.3.0
+six==1.13.0
+tabulate==0.8.6
+typed-ast==1.4.0
+urllib3==1.25.7
+wrapt==1.11.2

terraform/secrets-tool/secrets-tool

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# CLI
+import click
+
+import config
+import logging
+
+from commands.secrets import secrets
+from commands.terraform import terraform
+
+config.setup_logging()
+logger = logging.getLogger(__name__)
+
+PROCESS='terraform'
+
+# Define core command group
+@click.group()
+def cli():
+    pass
+
+# Add additional command groups
+cli.add_command(secrets)
+cli.add_command(terraform)
+
+
+if __name__ == "__main__":
+    try:
+        cli()
+    except Exception as e:
+        print(e)
+
+'''
+    try:
+        keyvault = secrets(vault_url="https://cloudzero-dev-keyvault.vault.azure.net/")
+        keyvault.set_secret('dbuser','foo')
+        #print(keyvault.get_secret('db-user').value)
+
+        # Set env variables for TF
+        for secret in keyvault.list_secrets():
+            name = 'TF_VAR_' + secret
+            val = keyvault.get_secret(secret)
+            #print(val)
+            os.environ[name] = val
+        env = os.environ.copy() 
+        command = "{} {}".format(PROCESS, sys.argv[1])
+        with subprocess.Popen(command, env=env, stdout=subprocess.PIPE, shell=True) as proc:
+            for line in proc.stdout:
+                logging.info(line.decode("utf-8") )
+        
+    except Exception as e:
+        print(e, traceback.print_stack)
+'''

terraform/secrets-tool/utils/keyvault/auth.py

@@ -0,0 +1,10 @@
+import logging
+
+from azure.identity import InteractiveBrowserCredential
+
+logger = logging.getLogger(__name__)
+
+class Auth:
+    def __init__(self, vault_url, *args, **kwargs):
+        self.credentials = InteractiveBrowserCredential()
+        self.vault_url = vault_url

terraform/secrets-tool/utils/keyvault/keys.py

@@ -0,0 +1,19 @@
+import logging
+from azure.keyvault.keys import KeyClient
+from .auth import Auth
+
+logger = logging.getLogger(__name__)
+
+KEY_SIZE=2048
+KEY_TYPE='rsa'
+
+class keys(Auth):
+    def __init__(self, *args, **kwargs):
+        super(keys, self).__init__(*args, **kwargs)
+        self.key_client = KeyClient(vault_url=self.vault_url, credential=self.credentials)
+
+    def get_key(self):
+        return self.key_client
+
+    def create_key(self):
+        pass

terraform/secrets-tool/utils/keyvault/secrets.py

@@ -0,0 +1,26 @@
+import logging
+from .auth import Auth
+from azure.keyvault.secrets import SecretClient
+
+logger = logging.getLogger(__name__)
+
+class SecretsClient(Auth):
+    def __init__(self, *args, **kwargs):
+        super(SecretsClient, self).__init__(*args, **kwargs)
+        self.secret_client = SecretClient(vault_url=self.vault_url, credential=self.credentials)
+
+    def get_secret(self, key):
+        secret = self.secret_client.get_secret(key)
+        return secret.value
+
+    def set_secret(self, key: str, value: str):
+        secret = self.secret_client.set_secret(key, value)
+        logger.debug('Set value for key: {}'.format(key))
+        return secret
+
+    def list_secrets(self):
+        secrets = list()
+        secret_properties = self.secret_client.list_properties_of_secrets()
+        for secret in secret_properties:
+            secrets.append(secret.name)
+        return secrets