Skip to content

Commit c288c4f

Browse files
brucegucodebrucegucode
authored
Add e2e test for rdsproxy (#115)
Issue #, if available: aws-controllers-k8s/community#1440 Description of changes: - Add e2e test for rds `dbproxy` resource Two major issues were observed while testing locally 1. This test needs a complex dependency chain and it will cause bootstrap logic pretty complex. I'm inclined to either hard code it for our rds tests account or we just remove it for now. The dependency chain is like this : create one base rds instance -> create one aws secret service for this rds instance -> create an iam role for this secret -> create rds proxy using iam role & secret . Deletion is the opposite way. Currently we're not able to create a rds instance in bootstrap logic or create a aws secret service secret. 2. The e2e test lasts pretty long. creation will take around 10 min, deletion will take around 15 min, the total time would be around 30min in total which will greatly increase e2e test time. By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
1 parent 84ca2f6 commit c288c4f

File tree

7 files changed

+417
-3
lines changed

7 files changed

+417
-3
lines changed

‎apis/v1alpha1/ack-generate-metadata.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
ack_generate_info:
2-
build_date: "2022-08-13T17:21:53Z"
2+
build_date: "2022-08-22T20:21:30Z"
33
build_hash: fe61d04673fd4d9848d5f726b01e0689a16d3733
4-
go_version: go1.18.1
4+
go_version: go1.18.2
55
version: v0.19.3-1-gfe61d04
66
api_directory_checksum: 6a967cc8a62d521d4f4816dbccc48f81d0cb271d
77
api_version: v1alpha1

‎pkg/resource/db_proxy/hooks.go‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License"). You may
4+
// not use this file except in compliance with the License. A copy of the
5+
// License is located at
6+
//
7+
// http://aws.amazon.com/apache2.0/
8+
//
9+
// or in the "license" file accompanying this file. This file is distributed
10+
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11+
// express or implied. See the License for the specific language governing
12+
// permissions and limitations under the License.
13+
14+
package db_proxy
15+
16+
import (
17+
"context"
18+
19+
ackcompare "github.com/aws-controllers-k8s/runtime/pkg/compare"
20+
ackrtlog "github.com/aws-controllers-k8s/runtime/pkg/runtime/log"
21+
22+
svcapitypes "github.com/aws-controllers-k8s/rds-controller/apis/v1alpha1"
23+
svcsdk "github.com/aws/aws-sdk-go/service/rds"
24+
25+
"github.com/aws-controllers-k8s/rds-controller/pkg/util"
26+
)
27+
28+
// syncTags keeps the resource's tags in sync
29+
//
30+
// NOTE(jaypipes): RDS' Tagging APIs differ from other AWS APIs in the
31+
// following ways:
32+
//
33+
// 1. The names of the tagging API operations are different. Other APIs use the
34+
// Tagris `ListTagsForResource`, `TagResource` and `UntagResource` API
35+
// calls. RDS uses `ListTagsForResource`, `AddTagsToResource` and
36+
// `RemoveTagsFromResource`.
37+
//
38+
// 2. Even though the name of the `ListTagsForResource` API call is the same,
39+
// the structure of the input and the output are different from other APIs.
40+
// For the input, instead of a `ResourceArn` field, RDS names the field
41+
// `ResourceName`, but actually expects an ARN, not the proxy
42+
// name. This is the same for the `AddTagsToResource` and
43+
// `RemoveTagsFromResource` input shapes. For the output shape, the field is
44+
// called `TagList` instead of `Tags` but is otherwise the same struct with
45+
// a `Key` and `Value` member field.
46+
func (rm *resourceManager) syncTags(
47+
ctx context.Context,
48+
desired *resource,
49+
latest *resource,
50+
) (err error) {
51+
rlog := ackrtlog.FromContext(ctx)
52+
exit := rlog.Trace("rm.syncTags")
53+
defer func() { exit(err) }()
54+
55+
arn := (*string)(latest.ko.Status.ACKResourceMetadata.ARN)
56+
57+
toAdd, toDelete := util.ComputeTagsDelta(
58+
desired.ko.Spec.Tags, latest.ko.Spec.Tags,
59+
)
60+
61+
if len(toDelete) > 0 {
62+
rlog.Debug("removing tags from proxy", "tags", toDelete)
63+
_, err = rm.sdkapi.RemoveTagsFromResourceWithContext(
64+
ctx,
65+
&svcsdk.RemoveTagsFromResourceInput{
66+
ResourceName: arn,
67+
TagKeys: toDelete,
68+
},
69+
)
70+
rm.metrics.RecordAPICall("UPDATE", "RemoveTagsFromResource", err)
71+
if err != nil {
72+
return err
73+
}
74+
}
75+
76+
// NOTE(jaypipes): According to the RDS API documentation, adding a tag
77+
// with a new value overwrites any existing tag with the same key. So, we
78+
// don't need to do anything to "update" a Tag. Simply including it in the
79+
// AddTagsToResource call is enough.
80+
if len(toAdd) > 0 {
81+
rlog.Debug("adding tags to proxy", "tags", toAdd)
82+
_, err = rm.sdkapi.AddTagsToResourceWithContext(
83+
ctx,
84+
&svcsdk.AddTagsToResourceInput{
85+
ResourceName: arn,
86+
Tags: sdkTagsFromResourceTags(toAdd),
87+
},
88+
)
89+
rm.metrics.RecordAPICall("UPDATE", "AddTagsToResource", err)
90+
if err != nil {
91+
return err
92+
}
93+
}
94+
return nil
95+
}
96+
97+
// getTags retrieves the resource's associated tags
98+
func (rm *resourceManager) getTags(
99+
ctx context.Context,
100+
resourceARN string,
101+
) ([]*svcapitypes.Tag, error) {
102+
resp, err := rm.sdkapi.ListTagsForResourceWithContext(
103+
ctx,
104+
&svcsdk.ListTagsForResourceInput{
105+
ResourceName: &resourceARN,
106+
},
107+
)
108+
rm.metrics.RecordAPICall("GET", "ListTagsForResource", err)
109+
if err != nil {
110+
return nil, err
111+
}
112+
tags := make([]*svcapitypes.Tag, 0, len(resp.TagList))
113+
for _, tag := range resp.TagList {
114+
tags = append(tags, &svcapitypes.Tag{
115+
Key: tag.Key,
116+
Value: tag.Value,
117+
})
118+
}
119+
return tags, nil
120+
}
121+
122+
// compareTags adds a difference to the delta if the supplied resources have
123+
// different tag collections
124+
func compareTags(
125+
delta *ackcompare.Delta,
126+
a *resource,
127+
b *resource,
128+
) {
129+
if len(a.ko.Spec.Tags) != len(b.ko.Spec.Tags) {
130+
delta.Add("Spec.Tags", a.ko.Spec.Tags, b.ko.Spec.Tags)
131+
} else if len(a.ko.Spec.Tags) > 0 {
132+
if !util.EqualTags(a.ko.Spec.Tags, b.ko.Spec.Tags) {
133+
delta.Add("Spec.Tags", a.ko.Spec.Tags, b.ko.Spec.Tags)
134+
}
135+
}
136+
}
137+
138+
// sdkTagsFromResourceTags transforms a *svcapitypes.Tag array to a *svcsdk.Tag
139+
// array.
140+
func sdkTagsFromResourceTags(
141+
rTags []*svcapitypes.Tag,
142+
) []*svcsdk.Tag {
143+
tags := make([]*svcsdk.Tag, len(rTags))
144+
for i := range rTags {
145+
tags[i] = &svcsdk.Tag{
146+
Key: rTags[i].Key,
147+
Value: rTags[i].Value,
148+
}
149+
}
150+
return tags
151+
}

‎test/e2e/bootstrap_resources.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,15 @@
1717

1818
from dataclasses import dataclass
1919
from acktest.bootstrapping.vpc import VPC
20+
from acktest.bootstrapping.iam import Role
2021
from acktest.bootstrapping import Resources
2122
from e2e import bootstrap_directory
2223

2324

2425
@dataclass
2526
class BootstrapResources(Resources):
2627
ClusterVPC: VPC
28+
RDSProxyRole: Role
2729

2830
_bootstrap_resources = None
2931

‎test/e2e/db_proxy.py‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License"). You may
4+
# not use this file except in compliance with the License. A copy of the
5+
# License is located at
6+
#
7+
# http://aws.amazon.com/apache2.0/
8+
#
9+
# or in the "license" file accompanying this file. This file is distributed
10+
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11+
# express or implied. See the License for the specific language governing
12+
# permissions and limitations under the License.
13+
14+
"""Utilities for working with DB proxy resources"""
15+
16+
import datetime
17+
import time
18+
import typing
19+
20+
import boto3
21+
import pytest
22+
23+
DEFAULT_WAIT_UNTIL_TIMEOUT_SECONDS = 60*10
24+
DEFAULT_WAIT_UNTIL_INTERVAL_SECONDS = 15
25+
DEFAULT_WAIT_UNTIL_DELETED_TIMEOUT_SECONDS = 60*10
26+
DEFAULT_WAIT_UNTIL_DELETED_INTERVAL_SECONDS = 15
27+
28+
ProxyMatchFunc = typing.NewType(
29+
'ProxyMatchFunc',
30+
typing.Callable[[dict], bool],
31+
)
32+
33+
class StatusMatcher:
34+
def __init__(self, status):
35+
self.match_on = status
36+
37+
def __call__(self, record: dict) -> bool:
38+
return 'Status' in record and record['Status'] == self.match_on
39+
40+
41+
def status_matches(status: str) -> ProxyMatchFunc:
42+
return StatusMatcher(status)
43+
44+
45+
def wait_until(
46+
db_proxy_id: str,
47+
match_fn: ProxyMatchFunc,
48+
timeout_seconds: int = DEFAULT_WAIT_UNTIL_TIMEOUT_SECONDS,
49+
interval_seconds: int = DEFAULT_WAIT_UNTIL_INTERVAL_SECONDS,
50+
) -> None:
51+
"""Waits until a DB proxy with a supplied ID is returned from the RDS API
52+
and the matching functor returns True.
53+
54+
Usage:
55+
from e2e.db_proxy import wait_until, status_matches
56+
57+
wait_until(
58+
proxy_id,
59+
status_matches("available"),
60+
)
61+
62+
Raises:
63+
pytest.fail upon timeout
64+
"""
65+
now = datetime.datetime.now()
66+
timeout = now + datetime.timedelta(seconds=timeout_seconds)
67+
68+
while not match_fn(get(db_proxy_id)):
69+
if datetime.datetime.now() >= timeout:
70+
pytest.fail("failed to match DBProxy before timeout")
71+
time.sleep(interval_seconds)
72+
73+
74+
def wait_until_deleted(
75+
db_proxy_id: str,
76+
timeout_seconds: int = DEFAULT_WAIT_UNTIL_DELETED_TIMEOUT_SECONDS,
77+
interval_seconds: int = DEFAULT_WAIT_UNTIL_DELETED_INTERVAL_SECONDS,
78+
) -> None:
79+
"""Waits until a DB proxy with a supplied ID is no longer returned from
80+
the RDS API.
81+
82+
Usage:
83+
from e2e.db_proxy import wait_until_deleted
84+
85+
wait_until_deleted(proxy_id)
86+
87+
Raises:
88+
pytest.fail upon timeout or if the DB proxy goes to any other status
89+
other than 'deleting'
90+
"""
91+
now = datetime.datetime.now()
92+
timeout = now + datetime.timedelta(seconds=timeout_seconds)
93+
94+
while True:
95+
if datetime.datetime.now() >= timeout:
96+
pytest.fail(
97+
"Timed out waiting for DB proxy to be "
98+
"deleted in RDS API"
99+
)
100+
time.sleep(interval_seconds)
101+
102+
latest = get(db_proxy_id)
103+
if latest is None:
104+
break
105+
106+
if latest['Status'] != "deleting":
107+
pytest.fail(
108+
"Status is not 'deleting' for DB proxy that was "
109+
"deleted. Status is " + latest['Status']
110+
)
111+
112+
113+
def get(db_proxy_id):
114+
"""Returns a dict containing the DB proxy record from the RDS API.
115+
116+
If no such DB proxy exists, returns None.
117+
"""
118+
c = boto3.client('rds')
119+
try:
120+
resp = c.describe_db_proxies(DBProxyName=db_proxy_id)
121+
assert len(resp['DBProxies']) == 1
122+
return resp['DBProxies'][0]
123+
except c.exceptions.DBProxyNotFoundFault:
124+
return None
125+
126+
127+
def get_tags(db_proxy_arn):
128+
"""Returns a dict containing the DB proxy's tag records from the RDS API.
129+
130+
If no such DB proxy exists, returns None.
131+
"""
132+
c = boto3.client('rds')
133+
try:
134+
resp = c.list_tags_for_resource(
135+
ResourceName=db_proxy_arn,
136+
)
137+
return resp['TagList']
138+
except c.exceptions.DBProxyNotFoundFault:
139+
return None

‎test/e2e/resources/db_proxy.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
apiVersion: rds.services.k8s.aws/v1alpha1
2+
kind: DBProxy
3+
metadata:
4+
name: $DB_PROXY_NAME
5+
spec:
6+
name: $DB_PROXY_NAME
7+
engineFamily: $DB_PROXY_ENGINE_FAMILY
8+
roleARN: $IAM_ROLE_ARN
9+
auth:
10+
- secretARN: $SECRET_ARN
11+
authScheme: SECRETS
12+
iamAuth: DISABLED
13+
description: $DESCRIPTION
14+
vpcSubnetIDs:
15+
- $PUBLIC_SUBNET_1
16+
- $PUBLIC_SUBNET_2

‎test/e2e/service_bootstrap.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
from e2e import bootstrap_directory
1919
from acktest.bootstrapping import Resources, BootstrapFailureException
2020
from acktest.bootstrapping.vpc import VPC
21+
from acktest.bootstrapping.iam import Role
2122
from e2e.bootstrap_resources import BootstrapResources
2223

2324

@@ -26,6 +27,7 @@ def service_bootstrap() -> Resources:
2627

2728
resources = BootstrapResources(
2829
ClusterVPC=VPC(name_prefix="cluster-vpc", num_public_subnet=2, num_private_subnet=2),
30+
RDSProxyRole=Role("rds-proxy-role", "rds.amazonaws.com", managed_policies=["arn:aws:iam::aws:policy/SecretsManagerReadWrite"])
2931
)
3032

3133
try:
@@ -38,4 +40,4 @@ def service_bootstrap() -> Resources:
3840
if __name__ == "__main__":
3941
config = service_bootstrap()
4042
# Write config to current directory by default
41-
config.serialize(bootstrap_directory)
43+
config.serialize(bootstrap_directory)

0 commit comments

Comments
 (0)