Changeset 267579 in webkit

Timestamp:

Sep 25, 2020, 10:55:03 AM (6 years ago)

Author:

Jonathan Bedard

Message:

results.webkit.org: Use s3 for cold storage
​https://bugs.webkit.org/show_bug.cgi?id=216662
<rdar://problem/69092010>

Rubber-stamped by Aakash Jain.

• Scripts/libraries/resultsdbpy/resultsdbpy/init.py:
• Scripts/libraries/resultsdbpy/resultsdbpy/model/archive_context.py:

(ArchiveContext.init): Differentiate between cache storage and long-term storage of archives.
(ArchiveContext.enter): Connect to the cold storage archive, if one is available.
(ArchiveContext.exit): Disconnect from the cold storage archive, if applicable.
(ArchiveContext.register): Save to cold storage instead of cache storage by default.
(ArchiveContext.find_archive):

• Scripts/libraries/resultsdbpy/resultsdbpy/model/model.py:

(Model.init): Pass S3 credentials to ArchiveContext.

• Scripts/libraries/resultsdbpy/resultsdbpy/model/s3_archiver.py: Added.

(S3Archiver):
(S3Archiver.Credentials):
(S3Archiver.init): Connect to S3 and configure our bucket.
(S3Archiver._cipher): Construct new AES cipher, if a key was provided.
(S3Archiver.enter): Create S3 resource, if one is not available.
(S3Archiver.exit): Teardown S3 resource.
(S3Archiver.save): Save an archive to S3 by it's hash.
(S3Archiver.retrieve): Retreive an archive from S3 by its hash.

• Scripts/libraries/resultsdbpy/setup.py: Add boto3.

Location:

trunk/Tools

Files:

• ChangeLog (modified) (1 diff)
• Scripts/libraries/resultsdbpy/resultsdbpy/__init__.py (modified) (1 diff)
• Scripts/libraries/resultsdbpy/resultsdbpy/model/archive_context.py (modified) (6 diffs)
• Scripts/libraries/resultsdbpy/resultsdbpy/model/model.py (modified) (2 diffs)
• Scripts/libraries/resultsdbpy/resultsdbpy/model/s3_archiver.py (added)
• Scripts/libraries/resultsdbpy/setup.py (modified) (2 diffs)

trunk/Tools/ChangeLog

@@ +1 @@
+2020-09-25  Jonathan Bedard  <jbedard@apple.com>
+
+        results.webkit.org: Use s3 for cold storage
+        https://bugs.webkit.org/show_bug.cgi?id=216662
+        <rdar://problem/69092010>
+
+        Rubber-stamped by Aakash Jain.
+
+        * Scripts/libraries/resultsdbpy/resultsdbpy/__init__.py:
+        * Scripts/libraries/resultsdbpy/resultsdbpy/model/archive_context.py:
+        (ArchiveContext.__init__): Differentiate between cache storage and long-term storage of archives.
+        (ArchiveContext.__enter__): Connect to the cold storage archive, if one is available.
+        (ArchiveContext.__exit__): Disconnect from the cold storage archive, if applicable.
+        (ArchiveContext.register): Save to cold storage instead of cache storage by default.
+        (ArchiveContext.find_archive):
+        * Scripts/libraries/resultsdbpy/resultsdbpy/model/model.py:
+        (Model.__init__): Pass S3 credentials to ArchiveContext.
+        * Scripts/libraries/resultsdbpy/resultsdbpy/model/s3_archiver.py: Added.
+        (S3Archiver):
+        (S3Archiver.Credentials):
+        (S3Archiver.__init__): Connect to S3 and configure our bucket.
+        (S3Archiver._cipher): Construct new AES cipher, if a key was provided.
+        (S3Archiver.__enter__): Create S3 resource, if one is not available.
+        (S3Archiver.__exit__): Teardown S3 resource.
+        (S3Archiver.save): Save an archive to S3 by it's hash.
+        (S3Archiver.retrieve): Retreive an archive from S3 by its hash.
+        * Scripts/libraries/resultsdbpy/setup.py: Add boto3.
+
 2020-09-25  Youenn Fablet  <youenn@apple.com>
 

trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/__init__.py

@@ -44 +44 @@
     )
 
-version = Version(1, 0, 2)
+version = Version(1, 1, 0)
 
 name = 'resultsdbpy'

trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/archive_context.py

@@ -23 +23 @@
 import calendar
 import io
-import json
 import time
 import zipfile
 
 from cassandra.cqlengine import columns
-from cassandra.cqlengine.models import Model
-from collections import OrderedDict
 from datetime import datetime
 from resultsdbpy.controller.commit import Commit
-from resultsdbpy.controller.configuration import Configuration
 from resultsdbpy.model.archiver import Archiver
 from resultsdbpy.model.cassandra_archiver import CassandraArchiver
 from resultsdbpy.model.commit_context import CommitContext
 from resultsdbpy.model.configuration_context import ClusteredByConfiguration
+from resultsdbpy.model.s3_archiver import S3Archiver
 from resultsdbpy.model.upload_context import UploadContext
 
@@ -82 +79 @@
         return zipfile.ZipFile(archive, mode='r')
 
-    def __init__(self, configuration_context, commit_context, ttl_seconds=None):
+    def __init__(self, configuration_context, commit_context, ttl_seconds=None, s3_credentials=None):
         self.configuration_context = configuration_context
         self.commit_context = commit_context
         self.cassandra = self.configuration_context.cassandra
-        self.archiver = CassandraArchiver(self.cassandra)
+        self.cached_archiver = CassandraArchiver(self.cassandra)
+        self.cold_archiver = S3Archiver(
+            bucket='{}-results-archive'.format(self.cassandra.keyspace.replace('_', '-')),
+            credentials=s3_credentials,
+            ttl_seconds=ttl_seconds,
+        ) if s3_credentials else None
         self.ttl_seconds = ttl_seconds
 
@@ -96 +98 @@
         self.configuration_context.__enter__()
         self.commit_context.__enter__()
-        self.archiver.__enter__()
+        self.cached_archiver.__enter__()
+        if self.cold_archiver:
+            self.cold_archiver.__enter__()
 
     def __exit__(self, *args, **kwargs):
         self.commit_context.__exit__(*args, **kwargs)
         self.configuration_context.__exit__(*args, **kwargs)
-        self.archiver.__exit__(*args, **kwargs)
+        self.cached_archiver.__exit__(*args, **kwargs)
+        if self.cold_archiver:
+            self.cold_archiver.__exit__()
 
     def register(self, archive, configuration, commits, suite, timestamp=None):
@@ -117 +123 @@
                     UploadContext.SuitesByConfiguration.__table_name__, configuration, suite=suite, branch=branch, ttl=ttl,
                 )
-                digest = self.archiver.save(archive, retain_for=ttl)
+
+                archiver_to_use = self.cold_archiver or self.cached_archiver
+                size = Archiver.archive_size(archive)
+                digest = archiver_to_use.save(archive, retain_for=ttl)
 
                 self.configuration_context.insert_row_with_configuration(
@@ -124 +133 @@
                     sdk=configuration.sdk or '?', start_time=timestamp,
                     digest=digest,
-                    size=Archiver.archive_size(archive),
+                    size=size,
                 )
 
@@ -168 +177 @@
 
                     if not archive_by_digest.get(value.get('digest')):
-                        archive = self.archiver.retrieve(value.get('digest'), value.get('size', None))
+                        archive = self.cached_archiver.retrieve(value.get('digest'), value.get('size', None))
                         if not archive:
-                            continue
+                            if not self.cold_archiver:
+                                continue
+
+                            archive = self.cold_archiver.retrieve(value.get('digest'), value.get('size', None))
+                            if not archive:
+                                continue
+
+                            # If we retrieved an archive from the cold_archiver, it's pretty likely that
+                            # the same archive will be retrieved in the near future. Cache the archive for 6 hours
+                            self.cached_archiver.save(archive, retain_for=60 * 60 * 6)
+
                         archive_by_digest[value.get('digest')] = archive
 

trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/model.py

@@ -46 +46 @@
         value = columns.Text(required=True)
 
-    def __init__(self, redis, cassandra, repositories=[], default_ttl_seconds=TTL_YEAR * 5, archive_ttl_seconds=TTL_WEEK * 8, async_processing=False):
+    def __init__(self, redis, cassandra, repositories=[], default_ttl_seconds=TTL_YEAR * 5, archive_ttl_seconds=TTL_WEEK * 8, async_processing=False, s3_credentials=None):
         if default_ttl_seconds is not None and default_ttl_seconds < 4 * self.TTL_WEEK:
             raise ValueError('TTL must be at least 4 weeks')
@@ -100 +100 @@
             commit_context=self.commit_context,
             ttl_seconds=self.archive_ttl_seconds,
+            s3_credentials=s3_credentials,
         )
 

trunk/Tools/Scripts/libraries/resultsdbpy/setup.py

@@ -60 +60 @@
     ],
     install_requires=[
+        'boto3',
         'cassandra-driver',
         'fakeredis',
@@ -66 +67 @@
         'gunicorn',
         'lupa',
+        'pycryptodome',
         'redis',
         'xmltodict',