Merge pull request #219 from trustyai-explainability/main

chore: Sync main brach

.github/pull_request_template.md

@@ -3,8 +3,6 @@ Many thanks for submitting your Pull Request :heart:!
 Please make sure that your PR meets the following requirements:
 
 - [ ] You have read the [contributors guide](https://github.com/trustyai-explainability/trustyai-explainability/blob/main/CONTRIBUTING.md)
-- [ ] Pull Request title is properly formatted: `FAI-XYZ Subject`
-- [ ] Pull Request title contains the target branch if not targeting main: `[0.3.x] FAI-XYZ Subject`
 - [ ] Pull Request contains link to the JIRA issue
 - [ ] Pull Request contains link to any dependent or related Pull Request
 - [ ] Pull Request contains description of the issue

.gitignore

@@ -22,6 +22,7 @@ dependency-reduced-pom.xml
 .run/
 .checkstyle
 tests/artifacts/*
+tests/custom_operators_config.yaml
 
 *.log
 release.properties

OWNERS

@@ -1,12 +1,13 @@
 approvers:
 - danielezonca
 - ruivieira
-- tteofili
+- dahlem
 - RobGeada
-- taneem-ibrahim
 reviewers:
 - danielezonca
 - ruivieira
-- tteofili
+- dahlem
 - RobGeada
-- taneem-ibrahim
+- christinaexyou
+- m-misiura
+- resoluteCoder

explainability-core/src/test/java/org/kie/trustyai/explainability/local/shap/background/CounterfactualGeneratorTest.java

@@ -42,6 +42,7 @@
 @ExtendWith(ThreadDumpOnTimeoutExtension.class)
 class CounterfactualGeneratorTest {
     int N_COUNTERFACTUALS_TO_GENERATE = 10;
+    int TIMEOUT = 30;
 
     @ParameterizedTest
     @ValueSource(ints = { 0, 1 })
@@ -74,7 +75,7 @@ void testDefaultGeneration(int seed) throws ExecutionException, InterruptedExcep
                 .withModel(model)
                 // Run the counterfactual on a different thread pool
                 .withCounterfactualConfig(CounterfactualUtils.getCounterfactualConfig((long) seed, 10_000L))
-                .withTimeoutSeconds(10)
+                .withTimeoutSeconds(TIMEOUT)
                 .withStepCount(10_000L)
                 .withGoalThreshold(.01)
                 .withRandom(rn)
@@ -121,7 +122,7 @@ void testChaining(int seed) throws ExecutionException, InterruptedException, Tim
                 .withModel(model)
                 // Run the counterfactual on a different thread pool
                 .withCounterfactualConfig(CounterfactualUtils.getCounterfactualConfig((long) seed, 30_000L))
-                .withTimeoutSeconds(10)
+                .withTimeoutSeconds(TIMEOUT)
                 .withStepCount(30_000L)
                 .withGoalThreshold(0.01)
                 .withRandom(rn)
@@ -163,7 +164,7 @@ void testChaining2Per(int seed) throws ExecutionException, InterruptedException,
                 .withModel(model)
                 // Run the counterfactual on a different thread pool
                 .withCounterfactualConfig(CounterfactualUtils.getCounterfactualConfig((long) seed, 30_000L))
-                .withTimeoutSeconds(10)
+                .withTimeoutSeconds(TIMEOUT)
                 .withStepCount(30_000L)
                 .withGoalThreshold(0.01)
                 .withRandom(rn)

explainability-service/pom.xml

@@ -44,6 +44,12 @@
             <artifactId>xstream</artifactId>
             <version>1.4.21</version>
           </dependency>
+          <!-- Override to io.netty:netty-handler:4.1.118.Final -->
+          <dependency>
+            <groupId>io.netty</groupId>
+            <artifactId>netty-handler</artifactId>
+            <version>4.1.118.Final</version>
+          </dependency>
         </dependencies>
     </dependencyManagement>
   <dependencies>

tests/Dockerfile

@@ -1,7 +1,8 @@
 FROM registry.access.redhat.com/ubi8:8.10-1020
-ARG ORG=trustyai-explainability
-ARG BRANCH=main
 ARG OC_CLI_URL=https://mirror.openshift.com/pub/openshift-v4/amd64/clients/ocp/4.14.33/openshift-client-linux.tar.gz
+ARG TRUSTYAI_TESTS_ORG=trustyai-explainability
+ARG TRUSTYAI_TESTS_REPO=trustyai-tests
+ARG TRUSTYAI_TESTS_BRANCH=main
 
 ENV HOME /root
 WORKDIR /root
@@ -21,23 +22,20 @@ ADD ${OC_CLI_URL} $HOME/peak/oc-cli.tar.gz
 RUN tar -C /usr/local/bin -xvf $HOME/peak/oc-cli.tar.gz && \
     chmod +x /usr/local/bin/oc
 
-COPY Pipfile Pipfile.lock $HOME/peak/
-
 # Install poetry to support the exeuction of trustyai-tests
 RUN curl -sSL https://install.python-poetry.org | python3 -
 ENV PATH="${PATH}:$HOME/.local/bin"
+
+# break cache if the repo has changed
+ADD https://api.github.com/repos/${TRUSTYAI_TESTS_ORG}/${TRUSTYAI_TESTS_REPO}/git/refs/heads/${TRUSTYAI_TESTS_BRANCH} version.json
 RUN cd $HOME/peak && \
-    git clone https://github.com/trustyai-explainability/trustyai-tests.git && \
+    git clone -b ${TRUSTYAI_TESTS_BRANCH} https://github.com/${TRUSTYAI_TESTS_ORG}/${TRUSTYAI_TESTS_REPO}.git && \
     cd trustyai-tests && \
     poetry install
 
-COPY setup/operatorsetup scripts/install.sh scripts/installandtest.sh $HOME/peak/
-COPY util $HOME/peak/operator-tests/trustyai-explainability
-COPY setup/odh-core-dsc.yaml setup/odh-core-dsci.yaml $HOME/kfdef/
-COPY scripts/setup.sh $HOME/peak/setup.sh
+COPY installandtest.sh custom_operators_config*.yaml $HOME/peak/
 
-RUN chmod -R 777 $HOME/kfdef && \
-    mkdir -p $HOME/.kube && \
+RUN mkdir -p $HOME/.kube && \
     chmod -R 777 $HOME/.kube && \
     chmod -R 777 $HOME/peak && \
     mkdir -p /peak && \

tests/Makefile

@@ -21,53 +21,103 @@ PYTEST_MARKERS=
 ARTIFACT_DIR=/tmp/artifacts
 LOCAL_ARTIFACT_DIR="${PWD}/artifacts"
 
+SKIP_OPERATORS_INSTALLATION?=false
+SKIP_DSC_INSTALLATION?=false
+USE_LOCAL_OPERATOR_CONFIG?=false
+
 BUILD_TOOL?=podman
-NO_CACHE?=false
+CACHE_ARG=
 LOCAL?=false
 TEARDOWN?=false
 PLATFORM?=linux/amd64
+TRUSTYAI_MANIFESTS_REPO=
 
-SERVICE_IMAGE?=quay.io/trustyai/trustyai-service:latest
-OPERATOR_IMAGE?=quay.io/trustyai/trustyai-service-operator:latest
+TRUSTYAI_TESTS_ORG?=$(GIT_ORG)
+TRUSTYAI_TESTS_REPO?=trustyai-tests
+TRUSTYAI_TESTS_BRANCH=$(GIT_BRANCH)
 
 all: test
 test: build run clean
 
 build:
-	${BUILD_TOOL} build -t $(IMAGE) --build-arg ORG=$(GIT_ORG) --build-arg BRANCH=$(GIT_BRANCH) --build-arg ODS_CI_REPO=$(ODS_CI_REPO) --build-arg ODS_CI_GITREF=$(ODS_CI_GITREF) --build-arg OC_CLI_URL=$(OC_CLI_URL) --platform=$(PLATFORM) .  --progress=plain
+	${BUILD_TOOL} build -t $(IMAGE) \
+ 		--build-arg ODS_CI_REPO=$(ODS_CI_REPO) \
+ 		--build-arg ODS_CI_GITREF=$(ODS_CI_GITREF) \
+ 		--build-arg OC_CLI_URL=$(OC_CLI_URL) \
+ 		--build-arg TRUSTYAI_TESTS_ORG=$(TRUSTYAI_TESTS_ORG) \
+ 		--build-arg TRUSTYAI_TESTS_REPO=$(TRUSTYAI_TESTS_REPO) \
+ 		--build-arg TRUSTYAI_TESTS_BRANCH=$(TRUSTYAI_TESTS_BRANCH) \
+ 		--platform=$(PLATFORM) .  --progress=plain $(CACHE_ARG)
 run:
 	# Confirm that we have a directory for storing any screenshots from selenium tests
 	mkdir -p ${LOCAL_ARTIFACT_DIR}/screenshots
-	oc config view --flatten --minify > /tmp/tests-kubeconfig
-	${BUILD_TOOL} run -e SKIP_INSTALL=$(SKIP_INSTALL) -e PYTEST_MARKERS=$(PYTEST_MARKERS) -e SKIP_OPERATOR_INSTALL=$(SKIP_OPERATOR_INSTALL) \
-	    -e SKIP_DSC_INSTALL=$(SKIP_DSC_INSTALL) -e ODHPROJECT=$(ODHPROJECT) \
-		-e OPENSHIFT_TESTUSER_NAME="$(OPENSHIFT_TESTUSER_NAME)" -e OPENSHIFT_TESTUSER_PASS="$(OPENSHIFT_TESTUSER_PASS)" -e OPENSHIFT_TESTUSER_LOGIN_PROVIDER=$(OPENSHIFT_TESTUSER_LOGIN_PROVIDER) -e ARTIFACT_DIR=$(ARTIFACT_DIR) \
-		-e LOCAL=$(LOCAL) -e TEARDOWN=$(TEARDOWN) \
-		-e SERVICE_IMAGE=$(SERVICE_IMAGE) -e OPERATOR_IMAGE=$(OPERATOR_IMAGE)  \
-		-it -v ${LOCAL_ARTIFACT_DIR}/:$(ARTIFACT_DIR):z -v /tmp/tests-kubeconfig:/tmp/kubeconfig:z $(IMAGE)
-
-
+	oc config view --flatten --minify > ${LOCAL_ARTIFACT_DIR}/tests-kubeconfig
+	${BUILD_TOOL} run -e PYTEST_MARKERS="$(PYTEST_MARKERS)" \
+		-e OPENSHIFT_TESTUSER_NAME="$(OPENSHIFT_TESTUSER_NAME)" -e OPENSHIFT_TESTUSER_PASS="$(OPENSHIFT_TESTUSER_PASS)" -e OPENSHIFT_TESTUSER_LOGIN_PROVIDER=$(OPENSHIFT_TESTUSER_LOGIN_PROVIDER) \
+ 		-e SKIP_DSC_INSTALLATION=$(SKIP_DSC_INSTALLATION) \
+ 		-e SKIP_OPERATORS_INSTALLATION=$(SKIP_OPERATORS_INSTALLATION) \
+ 		-e USE_LOCAL_OPERATOR_CONFIG=$(USE_LOCAL_OPERATOR_CONFIG) \
+		-e ARTIFACT_DIR=$(ARTIFACT_DIR) \
+		-e LOCAL=$(LOCAL) \
+		-e TEARDOWN=$(TEARDOWN) \
+		-e TRUSTYAI_MANIFESTS_REPO=$(TRUSTYAI_MANIFESTS_REPO) \
+		--platform=$(PLATFORM) \
+		-it -v ${LOCAL_ARTIFACT_DIR}/:$(ARTIFACT_DIR):z -v ${LOCAL_ARTIFACT_DIR}/tests-kubeconfig:/tmp/kubeconfig:z $(IMAGE)
 .SILENT:
-clean:
+clean: clean_odh clean_operators clean_authorino clean_maria clean_tests clean_projects clean_misc
+
+clean_odh:
 	echo "============== Cleaning ODH Installation =============="
 	oc delete -n $(ODHPROJECT) dsc default-dsc || true
 	oc delete -n $(ODHPROJECT) dsci default-dsci || true
 	oc delete project $(ODHPROJECT) || echo -e "\n\n==> If the project deletion failed, you can try to use this script to force it: https://raw.githubusercontent.com/jefferyb/useful-scripts/master/openshift/force-delete-openshift-project\n\n"
 
+clean_operators:
 	# Remove operator subscriptions, csvs
 	for operator in opendatahub authorino maria serverless servicemesh; do \
-  		echo "\n============== Deleting $$operator Operator Subscription and CSV ==============" ;\
-		oc delete $$(oc get subscription -n openshift-operators -o name | grep $$operator) -n openshift-operators || true ;\
-		oc delete $$(oc get clusterserviceversion -n openshift-operators -o name | grep $$operator) -n openshift-operators || true ;\
+  		operator_ns=$$(oc get subscription --all-namespaces | grep $$operator | awk '{print $$1}') ;\
+  		if [ ! -z $$operator_ns ]; then \
+  			echo "\n============== Deleting $$operator Operator Subscription and CSV from $$operator_ns==============" ;\
+			oc delete $$(oc get subscription -n $$operator_ns -o name | grep $$operator)  -n $$operator_ns || true ;\
+			oc delete $$(oc get clusterserviceversion -n $$operator_ns -o name | grep $$operator) -n $$operator_ns || true ;\
+		fi \
 	done
 
+clean_misc:
 	echo "\n============== Performing Cleanup =============="
 	# Other Cleanup
 	oc get mutatingwebhookconfiguration -o name | grep katib | grep $(ODHPROJECT) | xargs oc delete || true
+	oc get mutatingwebhookconfiguration -o name | grep odh-model-controller | xargs oc delete || true
 	oc get validatingwebhookconfiguration -o name | grep katib | grep $(ODHPROJECT) | xargs oc delete || true
 	oc delete identity htpasswd-provider:admin || true
 
+clean_maria:
+	echo "\n============== Maria Cleanup =============="
+	oc delete $$(oc get clusterrolebinding -o name | grep maria) || true
+	oc delete $$(oc get rolebinding -o name | grep maria) || true
+	oc delete $$(oc get clusterrole -o name | grep maria) || true
+	oc delete $$(oc get mariadboperators --all-namespaces -o name) -n openshift-operators || true
+	oc delete $$(oc get mariadboperators --all-namespaces -o name) -n mariadb-openshift-operators || true
+	oc get mutatingwebhookconfiguration -o name | grep mariadb-operator-webhook | xargs oc delete || true
+	oc get validatingwebhookconfiguration -o name | grep mariadb-operator-webhook | xargs oc delete || true
+
+
+clean_authorino:
+	echo "\n============== Authorino Cleanup =============="
+	oc delete clusterrolebinding authorino-operator-manager || true
+	oc delete $$(oc get clusterrolebinding -o name | grep authorino) || true
+	oc delete $$(oc get clusterrole --all-namespaces -o name | grep authorino) || true
+
+clean_tests:
+	oc delete project test-namespace || true
+	oc delete configmap model-serving-config -n opendatahub || true
+
+clean_projects:
 	echo "\n============== Deleting CI Projects =============="
 	# Delete test projects"
 	oc delete $$(oc get projects -o name | grep basictests-) || true
 	oc delete $$(oc get projects -o name | grep nightly-) || true
+	oc delete $$(oc get project -o name | grep -- "-openshift-operators") || true
+
+fetch_custom_operator_config:
+	wget -O custom_operators_config.yaml https://raw.githubusercontent.com/trustyai-explainability/trustyai-tests/refs/heads/main/trustyai_tests/setup/operators_config.yaml