diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index b556905e58..81b2074b90 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -53,7 +53,7 @@ jobs: java-version: '8' distribution: 'temurin' - name: build (8) - run: mvn -T 8 clean verify --no-transfer-progress -B -V + run: mvn -T 8 clean verify -DskipTests --no-transfer-progress -B -V - name: Upload artifacts uses: actions/upload-artifact@v4 with: @@ -88,12 +88,12 @@ jobs: with: name: target-11 path: target/* - + docker-build: needs: - build-8 runs-on: ubuntu-22.04 - timeout-minutes: 30 + timeout-minutes: 40 steps: - uses: actions/checkout@v4 - name: Download build-8 artifacts @@ -160,7 +160,7 @@ jobs: -f docker-compose.ranger-knox.yml \ -f docker-compose.ranger-ozone.yml up -d - - name: Check status of containers and remove them + - name: Check status of containers run: | sleep 60 containers=(ranger ranger-zk ranger-solr ranger-postgres ranger-usersync ranger-tagsync ranger-kms ranger-hadoop ranger-hbase ranger-kafka ranger-hive ranger-knox ozone-om ozone-scm ozone-datanode); @@ -176,8 +176,38 @@ jobs: if [[ $flag == true ]]; then echo "All required containers are up and running"; - docker stop $(docker ps -q) && docker rm $(docker ps -aq); - else - docker stop $(docker ps -q) && docker rm $(docker ps -aq); - exit 1; fi + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + # Install Robot Framework and dependencies + - name: Install Robot Framework + run: | + python -m pip install --upgrade pip + python --version + pip install apache-ranger + pip install robotframework + pip install robotframework-requests + pip install robotframework-jsonlibrary + robot --version || true + + - name: Run Ranger REST API SmokeTests + run: | + cd dev-support/smoketests/ranger + mkdir -p /tmp/smoketests/ranger + robot --outputdir /tmp/smoketests/ranger --loglevel DEBUG -P apitests user_management.robot policy_management.robot custom.robot + + - name: Upload Robot Framework Test Result Artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: Robot-Framework-Artifacts + path: /tmp/smoketests/ranger + + - name: Remove Containers + if: always() + run: | + docker stop $(docker ps -q) && docker rm $(docker ps -aq); diff --git a/dev-support/smoketests/ranger/apitests/__init__.py b/dev-support/smoketests/ranger/apitests/__init__.py new file mode 100644 index 0000000000..ed9d0b3c4e --- /dev/null +++ b/dev-support/smoketests/ranger/apitests/__init__.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/dev-support/smoketests/ranger/apitests/policy_management.py b/dev-support/smoketests/ranger/apitests/policy_management.py new file mode 100644 index 0000000000..da703564d5 --- /dev/null +++ b/dev-support/smoketests/ranger/apitests/policy_management.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from apache_ranger.model.ranger_service import * +from apache_ranger.client.ranger_client import * +from apache_ranger.model.ranger_policy import * + + +class TestPolicyManagement: + ROBOT_LIBRARY_SCOPE = 'SUITE' + + def __init__(self, ranger_url, username, password): + self.ranger = RangerClient(ranger_url, (username, password)) + self.login_user = username + self.ranger.session.verify = False + self.test_hive_policy_prefix = 'test_hive_policy' + self.test_hive_db_prefix = 'test_hive_db' + self.test_hive_table_prefix = 'test_hive_table' + return + + def get_hive_policy(self, service_name, policy_name): + return self.ranger.get_policy(service_name, policy_name) + + def delete_hive_policy(self, service_name, policy_name): + return self.ranger.delete_policy(service_name, policy_name) + + @staticmethod + def _create_policy_item_accesses(access_types): + ret = [] + for access_type in access_types: + ret.append(RangerPolicyItemAccess({'type': access_type})) + return ret + + @staticmethod + def _create_policy_item(users, access_types): + allow_item = RangerPolicyItem() + allow_item.users = users + allow_item.accesses = TestPolicyManagement._create_policy_item_accesses(access_types) + return allow_item + + @staticmethod + def _create_policy_item_with_delegate_admin(users, access_types): + allow_item = TestPolicyManagement._create_policy_item(users, access_types) + allow_item.delegateAdmin = True + return allow_item + + @staticmethod + def _create_hive_policy_resource(db_name, table_name, column_name): + resources = { + 'database': RangerPolicyResource({'values': [db_name]}), + 'table': RangerPolicyResource({'values': [table_name]}), + 'column': RangerPolicyResource({'values': [column_name]}) + } + return resources + + def create_hive_policy(self, service_name, policy_name, db_name, table_name): + policy = RangerPolicy() + policy.service = service_name + policy.name = policy_name + policy.resources = TestPolicyManagement._create_hive_policy_resource(db_name, table_name, "*") + allow_item = TestPolicyManagement._create_policy_item_with_delegate_admin(['test_user_1'], ['create', 'alter']) + deny_item = TestPolicyManagement._create_policy_item([self.login_user], ['drop']) + policy.policyItems = [allow_item] + policy.denyPolicyItems = [deny_item] + + created_policy = self.ranger.create_policy(policy) + print(f'Created policy: name={created_policy.name}, id={created_policy.id}') + return created_policy + + def get_all_policies(self): + all_policies = self.ranger.find_policies() + return all_policies + + def create_policies_in_bulk(self, service_name, count): + count = int(count) + for i in range(count): + policy_name = f'{self.test_hive_policy_prefix}_{i}' + db_name = f'{self.test_hive_db_prefix}_{i}' + table_name = f'{self.test_hive_table_prefix}_{i}' + self.create_hive_policy(service_name, policy_name, db_name, table_name) + return + + def delete_policies_in_bulk(self, service_name, count): + count = int(count) + for i in range(count): + policy_name = f'{self.test_hive_policy_prefix}_{i}' + self.delete_hive_policy(service_name, policy_name) + return + + +class TestServiceManagement: + ROBOT_LIBRARY_SCOPE = 'SUITE' + + def __init__(self, ranger_url, username, password): + self.ranger = RangerClient(ranger_url, (username, password)) + self.ranger.session.verify = False + return + + def create_service(self, service_name, service_type, configs): + service = RangerService() + service.name = service_name + service.type = service_type + service.configs = configs + return self.ranger.create_service(service) + + def delete_service(self, service_name): + return self.ranger.delete_service(service_name) + diff --git a/dev-support/smoketests/ranger/apitests/user_management.py b/dev-support/smoketests/ranger/apitests/user_management.py new file mode 100644 index 0000000000..f7382c0293 --- /dev/null +++ b/dev-support/smoketests/ranger/apitests/user_management.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from apache_ranger.client.ranger_client import * +from apache_ranger.utils import * +from apache_ranger.model.ranger_user_mgmt import * +from apache_ranger.client.ranger_user_mgmt_client import * + + +class TestUserManagement: + def __init__(self, ranger_url, username, password): + self.ranger = RangerClient(ranger_url, (username, password)) + self.ranger.session.verify = False + self.ugclient = RangerUserMgmtClient(self.ranger) + return + + ROBOT_LIBRARY_SCOPE = 'SUITE' + + def find_users(self): + print('Listing all users!') + users = self.ugclient.find_users() + print(f'{len(users.list)} users found') + return users + + def find_groups(self): + print('Listing all groups!') + groups = self.ugclient.find_groups() + print(f'{len(groups.list)} groups found') + return groups + + def create_user(self, user_name, role): + user = RangerUser({'name': user_name, + 'firstName': user_name, + 'lastName': 'lnu', + 'emailAddress': user_name + '@test.org', + 'password': 'Welcome1', + 'userRoleList': [role], + 'otherAttributes': '{ "dept": "test" }'}) + + created_user = self.ugclient.create_user(user) + print(f'User {created_user.name} created!') + return created_user + + def create_group(self, group_name): + group = RangerGroup({'name': group_name, 'otherAttributes': '{ "dept": "test" }'}) + created_group = self.ugclient.create_group(group) + print(f'Group {created_group.name} created!') + return created_group + + def add_to_group(self, group_name, group_id, user_id): + group_user = RangerGroupUser({'name': group_name, 'parentGroupId': group_id, 'userId': user_id}) + created_group_user = self.ugclient.create_group_user(group_user) + print(f'Created group-user: {created_group_user}') + return created_group_user + + def list_users_in_group(self, group_name): + users = self.ugclient.get_users_in_group(group_name) + return users + + def list_groups_for_user(self, user_name): + groups = self.ugclient.get_groups_for_user(user_name) + return groups + + def list_group_users(self): + group_users = self.ugclient.find_group_users() + print(f'{len(group_users.list)} group-users found') + + for group_user in group_users.list: + print(f'id: {group_user.id}, groupId: {group_user.parentGroupId}, userId: {group_user.userId}') + return group_users + + def delete_user_by_id(self, id): + self.ugclient.delete_user_by_id(id, True) + return + + def delete_group_by_id(self, id): + self.ugclient.delete_group_by_id(id, True) + return + + def delete_group_user_by_id(self, id): + self.ugclient.delete_group_user_by_id(id) + return + diff --git a/dev-support/smoketests/ranger/custom.robot b/dev-support/smoketests/ranger/custom.robot new file mode 100644 index 0000000000..5a5602eac3 --- /dev/null +++ b/dev-support/smoketests/ranger/custom.robot @@ -0,0 +1,23 @@ +*** Settings *** +Library policy_management.TestPolicyManagement http://localhost:6080 admin rangerR0cks! WITH NAME admin_p +Library policy_management.TestPolicyManagement http://localhost:6080 test_user_1 Welcome1 WITH NAME user_t +Library policy_management.TestPolicyManagement http://localhost:6080 finance_user Welcome1 WITH NAME user_f +Library Collections +Library JSONLibrary + +*** Variables *** + + +*** Test Cases *** +Admin User Succeeds To Create Policy Regular User Fails + [Documentation] A regular user fails to create hive policy whereas an admin user succeeds. + ${response} admin_p.Create Hive Policy dev_hive test_policy_custom_1 test_db_custom_1 test_table_custom_1 + Log ${response} + Run Keyword And Expect Error RangerServiceException* user_t.Create Hive Policy dev_hive test_policy_custom_2 test_db_custom_2 test_table_custom_2 + + +Regular User With Delegate-Admin Succeeds To Delete Policy Where Regular User Fails + [Documentation] A regular user with delegated-admin succeeds to delete hive policy whereas a regular user w/o delegated-admin fails + Run Keyword And Expect Error RangerServiceException* user_f.Delete Hive Policy dev_hive test_policy_custom_1 + ${response} user_t.Delete Hive Policy dev_hive test_policy_custom_1 + Log ${response} \ No newline at end of file diff --git a/dev-support/smoketests/ranger/policy_management.robot b/dev-support/smoketests/ranger/policy_management.robot new file mode 100644 index 0000000000..1ad94b1408 --- /dev/null +++ b/dev-support/smoketests/ranger/policy_management.robot @@ -0,0 +1,87 @@ +*** Settings *** +Library policy_management.TestPolicyManagement http://localhost:6080 admin rangerR0cks! +Library policy_management.TestServiceManagement http://localhost:6080 admin rangerR0cks! +Library Collections +Library JSONLibrary + +*** Variables *** +${HIVE_DEFAULT_SVC} dev_hive +${HIVE_DEFAULT_POLICY} all - global +${HIVE_TEST_SVC} test_hive +${HIVE_TEST_POLICY} test_policy +@{TEST_DB} test_db + +*** Test Cases *** +Create Hive Test Service + [Documentation] Create a test Hive service with specific configurations. + ${configs} Create Dictionary username=hive password=hive jdbc.driverClassName=org.apache.hive.jdbc.HiveDriver jdbc.url=jdbc:hive2://ranger-hadoop:10000 hadoop.security.authorization=${true} + ${response} Create Service test_hive hive ${configs} + Log ${response} + + +Delete Hive Test Service + [Documentation] Delete the test Hive service. + ${response} Delete Service ${HIVE_TEST_SVC} + + +Get All Policies + [Documentation] Get all policies in Ranger. + ${response} Get All Policies + ${service} Get Value From Json ${response} $[0].service + Should Be Equal ${service}[0] dev_hdfs + + +Validate Response - Get Default Hive Policy + [Documentation] Validate Response from the default hive policy: all - global + ${response} Get Hive Policy ${HIVE_DEFAULT_SVC} ${HIVE_DEFAULT_POLICY} + + ${service} Get Value From Json ${response} $.service + ${name} Get Value From Json ${response} $.name + ${isEnabled} Get Value From Json ${response} $.isEnabled + ${users} Get Value From Json ${response} $.policyItems[0].users + + Should Be Equal ${service}[0] ${HIVE_DEFAULT_SVC} + Should Be Equal ${name}[0] ${HIVE_DEFAULT_POLICY} + Should Be True ${isEnabled} + List Should Contain Value ${users}[0] hive + Log ${response} + + +Create Hive Test Policy + [Documentation] Create a test policy in Default Hive Service + ${response} Create Hive Policy dev_hive test_policy_78 test_db test_table + + ${id} Get Value From Json ${response} $.id + Set Suite Variable ${POLICY_ID} ${id} + Log ${response} + + +Delete Hive Test Policy + ${response} Delete Hive Policy dev_hive test_policy_78 + Log ${response} + +# Depends on an earlier Test +Validate Successive Hive Test Policy + [Documentation] Create the test policy again in Default Hive Service + ${response} Create Hive Policy dev_hive test_policy_78 test_db test_table + + ${id} Get Value From Json ${response} $.id + ${result} Evaluate ${POLICY_ID}[0] + 1 + Should Be Equal ${id}[0] ${result} + Log ${response} + + +Delete Successive Hive Test Policy + ${response} Delete Hive Policy dev_hive test_policy_78 + Log ${response} + + +Create 100 Policies + [Documentation] Creates 100 test policies in dev_hive service + Create Policies In Bulk dev_hive 100 + + +Delete 100 Policies + [Documentation] Deletes 100 test policies in dev_hive service + Delete Policies In Bulk dev_hive 100 + diff --git a/dev-support/smoketests/ranger/user_management.robot b/dev-support/smoketests/ranger/user_management.robot new file mode 100644 index 0000000000..51e58cf893 --- /dev/null +++ b/dev-support/smoketests/ranger/user_management.robot @@ -0,0 +1,68 @@ +*** Settings *** +Library user_management.TestUserManagement http://localhost:6080 admin rangerR0cks! +Library Collections +Library JSONLibrary + +*** Keywords *** +Create Test User + [Arguments] ${user_name} ${role} + ${response} Create User ${username} ${role} + Log ${response} + RETURN ${response} + + +*** Test Cases *** +Get All Users + Find Users + +Get All Groups + Find Groups + +Create Test User With Admin Role + ${response} Create Test User test_user_2 ROLE_SYS_ADMIN + ${id} Get Value From Json ${response} $.id + Set Suite Variable ${ADMIN_ID} ${id} + +Create Test User With User Role + ${response} Create Test User test_user_1 ROLE_USER + ${id} Get Value From Json ${response} $.id + Set Suite Variable ${USER_ID} ${id} + +Create Finance User With User Role + ${response} Create Test User finance_user ROLE_USER + +Create Test Group + ${response} Create Group test_group_1 + ${id} Get Value From Json ${response} $.id + Set Suite Variable ${GROUP_ID} ${id} + Log ${response} + +Add Test User To Test Group + ${response} Add To Group test_group_1 ${GROUP_ID}[0] ${USER_ID}[0] + ${users} List Users In Group test_group_1 + List Should Contain Value ${users} test_user_1 + + +#List Users In Hadoop Group +# [Documentation] Check existence of users: hdfs, yarn +# ${users} List Users In Group hadoop +# List Should Contain Value ${users} hdfs +# List Should Contain Value ${users} yarn +# +#List Groups For Ranger +# ${groups} List Groups For User ranger +# List Should Contain Value ${groups} ranger + + +List GroupUsers + ${response} List Group Users + Log ${response} + +#Delete Last User Created +# Delete User By Id ${USER_ID}[0] +# +#Delete Last Group Created +# Delete Group By Id ${GROUP_ID}[0] +# +#Delete Admin User Created +# Delete User By Id ${ADMIN_ID}[0]