WIP: Upload build logs to S3

binderhub/app.py

@@ -457,6 +457,12 @@ def _add_slash(self, proposal):
         """
     )
 
+    s3_logs_endpoint = Unicode("", help="S3 endpoint", config=True)
+    s3_logs_access_key = Unicode("", help="S3 access key ", config=True)
+    s3_logs_secret_key = Unicode("", help="S3 secret key", config=True)
+    s3_logs_bucket = Unicode("", help="S3 bucket", config=True)
+    s3_logs_region = Unicode("", help="S3 region", config=True)
+
     # FIXME: Come up with a better name for it?
     builder_required = Bool(
         True,
@@ -626,6 +632,11 @@ def initialize(self, *args, **kwargs):
                 "auth_enabled": self.auth_enabled,
                 "event_log": self.event_log,
                 "normalized_origin": self.normalized_origin,
+                "s3_logs_endpoint": self.s3_logs_endpoint,
+                "s3_logs_access_key": self.s3_logs_access_key,
+                "s3_logs_secret_key": self.s3_logs_secret_key,
+                "s3_logs_bucket": self.s3_logs_bucket,
+                "s3_logs_region": self.s3_logs_region,
             }
         )
         if self.auth_enabled:

binderhub/builder.py

@@ -2,10 +2,15 @@
 Handlers for working with version control services (i.e. GitHub) for builds.
 """
 
+import boto3
+from botocore import UNSIGNED
+from botocore.config import Config
 import hashlib
 from http.client import responses
 import json
+import os
 import string
+from tempfile import NamedTemporaryFile
 import time
 import escapism
 
@@ -366,6 +371,7 @@ async def get(self, provider_prefix, _unescaped_spec):
             sticky_builds=self.settings['sticky_builds'],
         )
 
+        templogfile = NamedTemporaryFile(mode='wt', delete=False)
         with BUILDS_INPROGRESS.track_inprogress():
             build_starttime = time.perf_counter()
             pool = self.settings['build_pool']
@@ -419,9 +425,21 @@ async def get(self, provider_prefix, _unescaped_spec):
                         failed = True
                         BUILD_TIME.labels(status='failure').observe(time.perf_counter() - build_starttime)
                         BUILD_COUNT.labels(status='failure', **self.repo_metric_labels).inc()
-
+                        done = True
+                    templogfile.write(payload.get("message"))
+                    app_log.debug(f'log: {payload.get("message")}')
                 await self.emit(event)
 
+        templogfile.close()
+        app_log.debug(templogfile.name)
+        loglink = await self.upload_log(templogfile.name, 'repo2docker.log')
+        if loglink:
+            await self.emit({
+                'phase': 'built',
+                'message': f'Build log: {loglink}\n',
+            })
+        os.remove(templogfile.name)
+
         # Launch after building an image
         if not failed:
             BUILD_TIME.labels(status='success').observe(time.perf_counter() - build_starttime)
@@ -446,6 +464,59 @@ async def get(self, provider_prefix, _unescaped_spec):
         # well-behaved clients will close connections after they receive the launch event.
         await gen.sleep(60)
 
+    def _s3_upload(self, logfile, destkey):
+        s3 = boto3.resource(
+            "s3",
+            endpoint_url=self.settings["s3_logs_endpoint"],
+            aws_access_key_id=self.settings["s3_logs_access_key"],
+            aws_secret_access_key=self.settings["s3_logs_secret_key"],
+            region_name=self.settings["s3_logs_region"],
+            config=Config(signature_version="s3v4"),
+        )
+        s3.Bucket(self.settings["s3_logs_bucket"]).upload_file(
+            logfile,
+            destkey,
+            ExtraArgs={"ContentType": "text/plain"},
+        )
+        # For simple S3 servers you can easily build the canonical URL
+        # For AWS S3 this can be more complicated depending on which region your bucket is in
+        # boto3 doesn't have a way to just get the public URL, so instead we create a pre-signed
+        # URL but discard the parameters since the object is public
+        s3unsigned = boto3.client(
+            "s3",
+            endpoint_url=self.settings["s3_logs_endpoint"],
+            aws_access_key_id=self.settings["s3_logs_access_key"],
+            aws_secret_access_key=self.settings["s3_logs_secret_key"],
+            region_name=self.settings["s3_logs_region"],
+            config=Config(UNSIGNED)
+        )
+        link = s3unsigned.generate_presigned_url(
+            'get_object', ExpiresIn=0, Params={
+                'Bucket': self.settings["s3_logs_bucket"],
+                'Key': destkey,
+            })
+        return link.split('?', 1)[0]
+
+    async def upload_log(self, logfile, destname):
+        if (self.settings["s3_logs_endpoint"] and
+            self.settings["s3_logs_access_key"] and
+            self.settings["s3_logs_secret_key"] and
+            self.settings["s3_logs_bucket"]
+        ):
+            dest = f"buildlogs/{self.image_name}/{destname}"
+            app_log.debug(f'Uploading log to %s/%s/%s',
+                self.settings["s3_logs_endpoint"],
+                self.settings["s3_logs_bucket"],
+                dest
+            )
+            # Since we only need one method wrap the sync boto3 library instead of using
+            # aioboto3 which depends on some compiled dependencies
+            loop = IOLoop.current()
+            link = await loop.run_in_executor(
+                self.settings['executor'], self._s3_upload, logfile, dest)
+            app_log.info('Log is available at %s', link)
+            return link
+
     async def launch(self, kube, provider):
         """Ask JupyterHub to launch the image."""
         # Load the spec-specific configuration if it has been overridden

requirements.txt

@@ -3,6 +3,7 @@
 kubernetes==9.0.*
 escapism
 traitlets
+boto3
 docker
 jinja2
 prometheus_client