diff --git a/.circleci/config.yml b/.circleci/config.yml index 3fddae2e..2851bce1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -185,6 +185,20 @@ workflows: branches: only: - master + - azure-aks/update-container-image: + cluster-name: atat-cluster + container-image-updates: "atst-beat=${AZURE_SERVER_NAME}/atat:atat-${CIRCLE_SHA1}" + namespace: atat + resource-name: deployment.apps/atst-beat + resource-group: atat + # uncomment below for debugging + # show-kubectl-command: true + requires: + - azure-migration + filters: + branches: + only: + - master - aws-ecr/build-and-push-image: extra-build-args: "--build-arg CSP=aws" repo: atat @@ -230,3 +244,17 @@ workflows: branches: only: - master + - aws-eks/update-container-image: + cluster-name: atat + container-image-updates: "atst-beat=${AWS_ECR_ACCOUNT_URL}/atat:atat-${CIRCLE_SHA1}" + namespace: atat + resource-name: deployment.apps/atst-beat + aws-region: "${AWS_REGION}" + # uncomment below for debugging + # show-kubectl-command: true + requires: + - aws-migration + filters: + branches: + only: + - master diff --git a/atst/domain/environments.py b/atst/domain/environments.py index 829fe3fa..ba723cc5 100644 --- a/atst/domain/environments.py +++ b/atst/domain/environments.py @@ -1,4 +1,4 @@ -from sqlalchemy import text, func, or_ +from sqlalchemy import func, or_ from sqlalchemy.orm.exc import NoResultFound from typing import List from uuid import UUID @@ -130,7 +130,7 @@ class Environments(object): results = ( cls.base_provision_query(now) .filter(Environment.cloud_id != None) - .filter(Environment.root_user_info == text("'null'")) + .filter(Environment.root_user_info == None) ).all() return [id_ for id_, in results] @@ -143,7 +143,7 @@ class Environments(object): results = ( cls.base_provision_query(now) .filter(Environment.cloud_id != None) - .filter(Environment.root_user_info != text("'null'")) - .filter(Environment.baseline_info == text("'null'")) + .filter(Environment.root_user_info != None) + .filter(Environment.baseline_info == None) ).all() return [id_ for id_, in results] diff --git a/atst/models/environment.py b/atst/models/environment.py index fb30e6d4..f3d85502 100644 --- a/atst/models/environment.py +++ b/atst/models/environment.py @@ -26,8 +26,8 @@ class Environment( creator = relationship("User") cloud_id = Column(String) - root_user_info = Column(JSONB) - baseline_info = Column(JSONB) + root_user_info = Column(JSONB(none_as_null=True)) + baseline_info = Column(JSONB(none_as_null=True)) claimed_until = Column(TIMESTAMP(timezone=True)) @@ -68,6 +68,10 @@ class Environment( else: return self.ProvisioningStatus.COMPLETED + @property + def is_pending(self): + return self.provisioning_status == self.ProvisioningStatus.PENDING + def __repr__(self): return "".format( self.name, diff --git a/atst/models/utils.py b/atst/models/utils.py index a3df73e3..6059d33e 100644 --- a/atst/models/utils.py +++ b/atst/models/utils.py @@ -47,3 +47,4 @@ def claim_for_update(resource, minutes=30): db.session.query(Model).filter(Model.id == resource.id).filter( Model.claimed_until != None ).update({"claimed_until": None}, synchronize_session="fetch") + db.session.commit() diff --git a/atst/queue.py b/atst/queue.py index d71532b1..5fc12a3f 100644 --- a/atst/queue.py +++ b/atst/queue.py @@ -5,7 +5,20 @@ celery = Celery(__name__) def update_celery(celery, app): celery.conf.update(app.config) - celery.conf.CELERYBEAT_SCHEDULE = {} + celery.conf.CELERYBEAT_SCHEDULE = { + "beat-dispatch_create_environment": { + "task": "atst.jobs.dispatch_create_environment", + "schedule": 60, + }, + "beat-dispatch_create_atat_admin_user": { + "task": "atst.jobs.dispatch_create_atat_admin_user", + "schedule": 60, + }, + "beat-dispatch_create_environment_baseline": { + "task": "atst.jobs.dispatch_create_environment_baseline", + "schedule": 60, + }, + } class ContextTask(celery.Task): def __call__(self, *args, **kwargs): diff --git a/atst/routes/applications/settings.py b/atst/routes/applications/settings.py index 9fc13e2b..2d5e8422 100644 --- a/atst/routes/applications/settings.py +++ b/atst/routes/applications/settings.py @@ -25,6 +25,7 @@ def get_environments_obj_for_app(application): env_data = { "id": env.id, "name": env.name, + "pending": env.is_pending, "edit_form": EditEnvironmentForm(obj=env), "member_count": len(env.roles), "members": [env_role.application_role.user_name for env_role in env.roles], diff --git a/deploy/aws/aws.yml b/deploy/aws/aws.yml index b30f7fb4..8f73a761 100644 --- a/deploy/aws/aws.yml +++ b/deploy/aws/aws.yml @@ -29,9 +29,6 @@ spec: containers: - name: atst image: 904153757533.dkr.ecr.us-east-2.amazonaws.com/atat:latest - resources: - requests: - memory: "500Mi" envFrom: - configMapRef: name: atst-envvars @@ -148,9 +145,57 @@ spec: "worker", "--loglevel=info" ] - resources: - requests: - memory: "500Mi" + envFrom: + - configMapRef: + name: atst-envvars + - configMapRef: + name: atst-worker-envvars + volumeMounts: + - name: atst-config + mountPath: "/opt/atat/atst/atst-overrides.ini" + subPath: atst-overrides.ini + volumes: + - name: atst-config + secret: + secretName: atst-config-ini + items: + - key: override.ini + path: atst-overrides.ini + mode: 0644 +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + app: atst + name: atst-beat + namespace: atat +spec: + selector: + matchLabels: + role: beat + replicas: 1 + strategy: + type: RollingUpdate + template: + metadata: + labels: + app: atst + role: beat + spec: + securityContext: + fsGroup: 101 + containers: + - name: atst-beat + image: 904153757533.dkr.ecr.us-east-2.amazonaws.com/atat:latest + args: [ + "/opt/atat/atst/.venv/bin/python", + "/opt/atat/atst/.venv/bin/celery", + "-A", + "celery_worker.celery", + "beat", + "--loglevel=info" + ] envFrom: - configMapRef: name: atst-envvars diff --git a/deploy/azure/azure.yml b/deploy/azure/azure.yml index e599b9f5..31274b01 100644 --- a/deploy/azure/azure.yml +++ b/deploy/azure/azure.yml @@ -29,9 +29,6 @@ spec: containers: - name: atst image: pwatat.azurecr.io/atat:latest - resources: - requests: - memory: "500Mi" envFrom: - configMapRef: name: atst-envvars @@ -149,9 +146,57 @@ spec: "worker", "--loglevel=info" ] - resources: - requests: - memory: "500Mi" + envFrom: + - configMapRef: + name: atst-envvars + - configMapRef: + name: atst-worker-envvars + volumeMounts: + - name: atst-config + mountPath: "/opt/atat/atst/atst-overrides.ini" + subPath: atst-overrides.ini + volumes: + - name: atst-config + secret: + secretName: atst-config-ini + items: + - key: override.ini + path: atst-overrides.ini + mode: 0644 +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + app: atst + name: atst-beat + namespace: atat +spec: + selector: + matchLabels: + role: beat + replicas: 1 + strategy: + type: RollingUpdate + template: + metadata: + labels: + app: atst + role: beat + spec: + securityContext: + fsGroup: 101 + containers: + - name: atst-beat + image: pwatat.azurecr.io/atat:latest + args: [ + "/opt/atat/atst/.venv/bin/python", + "/opt/atat/atst/.venv/bin/celery", + "-A", + "celery_worker.celery", + "beat", + "--loglevel=info" + ] envFrom: - configMapRef: name: atst-envvars diff --git a/script/dev_queue b/script/dev_queue index 4e8720ee..ef0d5ccc 100755 --- a/script/dev_queue +++ b/script/dev_queue @@ -4,7 +4,7 @@ set -e -WORKER="pipenv run celery -A celery_worker.celery worker --loglevel=info -B" +WORKER="pipenv run celery -A celery_worker.celery worker --loglevel=info -B -c 1" if [[ `command -v entr` ]]; then find atst | entr -r $WORKER diff --git a/templates/portfolios/applications/settings.html b/templates/portfolios/applications/settings.html index a4b1c38d..b297765d 100644 --- a/templates/portfolios/applications/settings.html +++ b/templates/portfolios/applications/settings.html @@ -218,7 +218,6 @@