Skip to content
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 61 additions & 0 deletions s3-pit-restore
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,41 @@ class TestS3PitRestore(unittest.TestCase):
print("Restoring and checking for dmarker_restore test")
self.assertTrue(self.check_tree(path, content))

class TestS3PitRestoreSameBucket(unittest.TestCase):
def check_versioning(self, s3):
bucket_versioning = s3.BucketVersioning(args.bucket)
bucket_versioning.load()

print("Checking bucket versioning ... ", end='', flush=True)
self.assertNotEqual(bucket_versioning.status, None)
print("enabled!")

def test_no_op(self):
print('Running test_no_op ...')
test_content = str(uuid.uuid4())
test_key = f'test_no_op/{str(uuid.uuid4())}'

s3 = boto3.resource('s3', endpoint_url=args.endpoint_url)
self.check_versioning(s3)

print("Preparing ...")
object = s3.Object(args.bucket, test_key)
object.put(Body=test_content)
time.sleep(1)

args.prefix = test_key
args.timestamp = None
args.from_timestamp = None

print("Restoring ...")
do_restore()

print("Checking ...")
result = s3.meta.client.list_object_versions(Bucket=args.bucket, Prefix=test_key)
self.assertEqual(1, len(result['Versions']))
self.assertEqual(0, len(result.get("DeleteMarkers", [])))


def signal_handler(signal, frame):
executor.shutdown(wait=False)
for future in list(futures.keys()):
Expand Down Expand Up @@ -293,6 +328,10 @@ def do_restore():
dest = args.dest
last_obj = {}
last_obj["Key"] = ""
# The key that was given for the latest version that was flagged with IsLatest=true.
is_latest_key = None
# The etag that was given for the latest version that was flagged with IsLatest=true.
is_latest_etag = None

if args.debug: boto3.set_stream_logger('botocore')

Expand Down Expand Up @@ -327,6 +366,10 @@ def do_restore():
# We've had a newer version or a delete of this key
continue

if obj["IsLatest"]:
is_latest_key = obj["Key"]
is_latest_etag = obj["ETag"]

version_date = obj["LastModified"]

if version_date > pit_end_date or version_date < pit_start_date:
Expand All @@ -348,6 +391,16 @@ def do_restore():
# This version needs to be restored..
last_obj = obj

# We can skip the restore if the current version is equivalent to the newest version
# of the object and if we want to restore it to the same bucket and path.
# is_latest_key == obj["Key"] also ensures that the object is not currently deleted,
# because a version with IsLatest=true was observed.
if is_latest_key == obj["Key"] and \
is_latest_etag == obj["ETag"] and \
args.bucket == args.dest_bucket and \
not args.dest_prefix:
continue

if handled_by_glacier(obj):
continue

Expand Down Expand Up @@ -425,6 +478,14 @@ if __name__=='__main__':
itersuite = unittest.TestLoader().loadTestsFromTestCase(TestS3PitRestore)
runner.run(itersuite)

# Restore in same bucket, ignoring original dest_bucket
args.dest_bucket = args.bucket
itersuite = unittest.TestLoader().loadTestsFromTestCase(TestS3PitRestore)
runner.run(itersuite)

itersuite = unittest.TestLoader().loadTestsFromTestCase(TestS3PitRestoreSameBucket)
runner.run(itersuite)

# Restore back dest_bucket state
args.dest_bucket = dest_bucket
if args.dest_bucket is not None:
Expand Down