From 1594103847b0ffe81af303163b3ea9450e89ed38 Mon Sep 17 00:00:00 2001 From: Patrick Smith Date: Tue, 28 Aug 2018 16:14:51 -0400 Subject: [PATCH 1/4] Update secret key for atst-config-ini --- deploy/kubernetes/atst.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/kubernetes/atst.yml b/deploy/kubernetes/atst.yml index 7ee669d8..0de8ab7e 100644 --- a/deploy/kubernetes/atst.yml +++ b/deploy/kubernetes/atst.yml @@ -24,7 +24,7 @@ spec: fsGroup: 101 containers: - name: atst - image: registry.atat.codes:443/atst-prod:e38bc2f + image: registry.atat.codes:443/atst-prod:0696894 resources: requests: memory: "2500Mi" @@ -74,7 +74,7 @@ spec: secret: secretName: atst-config-ini items: - - key: atst-overrides.ini + - key: override.ini path: atst-overrides.ini mode: 0644 - name: nginx-auth-tls From 4e16346ed438653c329cd2d0d767c6a93c82490d Mon Sep 17 00:00:00 2001 From: Patrick Smith Date: Tue, 28 Aug 2018 20:35:31 -0400 Subject: [PATCH 2/4] Don't verify hash of uploaded files I believe this is a bug in `libcloud`. We're passing an iterator (as required by libcloud -- https://github.com/apache/libcloud/blob/trunk/libcloud/storage/base.py#L592) for the stream, but when verifying the hash of the uploaded file, `libcloud` goes through the stream twice: https://github.com/apache/libcloud/blob/trunk/libcloud/storage/base.py#L614-L621 After the sending the file stream as an upload, when generating the hash, the iterator has already been iterated through so the second go-through returns an empty iterator. Thus, the hash will never match unless an empty file is uploaded. This change reaches into the container's driver so that we can pass the `verify_hash` kwarg, which cannot be specified on the container's methods. --- atst/uploader.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/atst/uploader.py b/atst/uploader.py index 167c3d03..9f1ef62a 100644 --- a/atst/uploader.py +++ b/atst/uploader.py @@ -26,9 +26,11 @@ class Uploader: ) object_name = uuid4().hex - self.container.upload_object_via_stream( - iterator=fyle.stream.__iter__(), + self.container.driver._put_object( + stream=iter(fyle.stream), + container=self.container, object_name=object_name, + verify_hash=False, extra={"acl": "private"}, ) return (fyle.filename, object_name) From 501caf767bc6e240a8e4e2f74a8551c2b61b77ee Mon Sep 17 00:00:00 2001 From: Patrick Smith Date: Wed, 29 Aug 2018 10:40:04 -0400 Subject: [PATCH 3/4] Upload with a temp file instead of streaming Using a stream is a no-go due to a bug in libcloud: https://issues.apache.org/jira/browse/LIBCLOUD-935?focusedCommentId=16152982&page=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel#comment-16152982 Instead, write the uploaded file to a named tempfile and pass that to the uploader. --- atst/uploader.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/atst/uploader.py b/atst/uploader.py index 9f1ef62a..e63ba4ce 100644 --- a/atst/uploader.py +++ b/atst/uploader.py @@ -1,4 +1,6 @@ +from tempfile import NamedTemporaryFile from uuid import uuid4 + from libcloud.storage.types import Provider from libcloud.storage.providers import get_driver @@ -26,13 +28,13 @@ class Uploader: ) object_name = uuid4().hex - self.container.driver._put_object( - stream=iter(fyle.stream), - container=self.container, - object_name=object_name, - verify_hash=False, - extra={"acl": "private"}, - ) + with NamedTemporaryFile() as tempfile: + tempfile.write(fyle.stream.read()) + self.container.upload_object( + file_path=tempfile.name, + object_name=object_name, + extra={"acl": "private"}, + ) return (fyle.filename, object_name) def download(self, path): From 474b3b2f50b11dc3ed965afa85221dc53152d036 Mon Sep 17 00:00:00 2001 From: dandds Date: Wed, 29 Aug 2018 11:01:50 -0400 Subject: [PATCH 4/4] update script should ensure uploads directory is present --- script/update | 3 +++ 1 file changed, 3 insertions(+) diff --git a/script/update b/script/update index 46d5d549..8907003e 100755 --- a/script/update +++ b/script/update @@ -4,6 +4,9 @@ source "$(dirname "${0}")"/../script/include/global_header.inc.sh +# create upload directory for app +mkdir uploads | true + # Enable DB migration MIGRATE_DB="true"