From 466e893bbbca679e6709b818b419bff7edc3f69c Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Mon, 31 Mar 2025 15:23:03 -0400 Subject: [PATCH 1/6] scan s3 uploads for secrets --- .github/create_and_upload_logs.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/create_and_upload_logs.sh b/.github/create_and_upload_logs.sh index 76d5273d3..c6e41e7ee 100755 --- a/.github/create_and_upload_logs.sh +++ b/.github/create_and_upload_logs.sh @@ -16,6 +16,12 @@ mv raw.log.2 raw.log if [[ $1 == 1 ]]; then echo "::notice title=$SUITE$STORAGE $(uname -i) s3 logs and reports::$REPORT_INDEX_URL" + + set -e + git clone https://github.com/Altinity/actions.git + python actions/scripts/scan_artifacts.py files pipeline_url.log.txt version.log.txt raw.log nice-new-fails.log.txt fails.log.txt report.html _instances _service_logs + set +e + ./retry.sh 5 30 aws s3 cp pipeline_url.log.txt $JOB_S3_ROOT/pipeline_url.log.txt --content-type "\"text/plain; charset=utf-8\"" ./retry.sh 5 30 aws s3 cp version.log.txt $SUITE_REPORT_BUCKET_PATH/version.log.txt --content-type "\"text/plain; charset=utf-8\"" ./retry.sh 5 30 aws s3 cp raw.log $SUITE_REPORT_BUCKET_PATH/raw.log From fc5e1650bb3485ab70725022767ebec590920622 Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Tue, 1 Apr 2025 09:39:34 -0400 Subject: [PATCH 2/6] Fix scan directories --- .github/create_and_upload_logs.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/create_and_upload_logs.sh b/.github/create_and_upload_logs.sh index c6e41e7ee..2ba3bd25e 100755 --- a/.github/create_and_upload_logs.sh +++ b/.github/create_and_upload_logs.sh @@ -17,10 +17,12 @@ if [[ $1 == 1 ]]; then echo "::notice title=$SUITE$STORAGE $(uname -i) s3 logs and reports::$REPORT_INDEX_URL" - set -e git clone https://github.com/Altinity/actions.git - python actions/scripts/scan_artifacts.py files pipeline_url.log.txt version.log.txt raw.log nice-new-fails.log.txt fails.log.txt report.html _instances _service_logs - set +e + python actions/scripts/scan_artifacts.py files pipeline_url.log.txt version.log.txt raw.log nice-new-fails.log.txt fails.log.txt report.html $SUITE/_instances $SUITE/_service_logs + if [[ $? -ne 0 ]]; then + # Leaked strings were found + exit 1 + fi ./retry.sh 5 30 aws s3 cp pipeline_url.log.txt $JOB_S3_ROOT/pipeline_url.log.txt --content-type "\"text/plain; charset=utf-8\"" ./retry.sh 5 30 aws s3 cp version.log.txt $SUITE_REPORT_BUCKET_PATH/version.log.txt --content-type "\"text/plain; charset=utf-8\"" From e885e34577ebd8c332a4bf62ab7ab887ceb9a173 Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Tue, 1 Apr 2025 11:54:47 -0400 Subject: [PATCH 3/6] pin scanner version and don't scan database files --- .github/create_and_upload_logs.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/create_and_upload_logs.sh b/.github/create_and_upload_logs.sh index 2ba3bd25e..f056ece56 100755 --- a/.github/create_and_upload_logs.sh +++ b/.github/create_and_upload_logs.sh @@ -17,7 +17,10 @@ if [[ $1 == 1 ]]; then echo "::notice title=$SUITE$STORAGE $(uname -i) s3 logs and reports::$REPORT_INDEX_URL" + sudo rm --recursive --force $SUITE/_instances/*/database/ + git clone https://github.com/Altinity/actions.git + git --git-dir actions checkout --quiet 46f8da1ccc668e800684fffd21981b507c20d86f python actions/scripts/scan_artifacts.py files pipeline_url.log.txt version.log.txt raw.log nice-new-fails.log.txt fails.log.txt report.html $SUITE/_instances $SUITE/_service_logs if [[ $? -ne 0 ]]; then # Leaked strings were found @@ -30,7 +33,6 @@ then ./retry.sh 5 30 aws s3 cp nice-new-fails.log.txt $SUITE_REPORT_BUCKET_PATH/nice-new-fails.log.txt --content-type "\"text/plain; charset=utf-8\"" ./retry.sh 5 30 aws s3 cp fails.log.txt $SUITE_REPORT_BUCKET_PATH/fails.log.txt --content-type "\"text/plain; charset=utf-8\"" ./retry.sh 5 30 aws s3 cp report.html $SUITE_REPORT_BUCKET_PATH/report.html - sudo rm --recursive --force $SUITE/_instances/*/database/ ./retry.sh 5 30 "aws s3 cp --recursive . $SUITE_REPORT_BUCKET_PATH/"' --exclude "*" --include "*/_instances/*.log" --content-type "\"text/plain; charset=utf-8\"" --no-follow-symlinks' ./retry.sh 5 30 "aws s3 cp --recursive $SUITE/_service_logs/ $SUITE_REPORT_BUCKET_PATH/_service_logs/"' --exclude "*" --include "*.log" --content-type "\"text/plain; charset=utf-8\""' fi From 00ce25011a73d34099c371e97a7af69920a26281 Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Wed, 2 Apr 2025 08:17:38 -0400 Subject: [PATCH 4/6] eliminate false positives in server logs --- .github/create_and_upload_logs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/create_and_upload_logs.sh b/.github/create_and_upload_logs.sh index f056ece56..b64d696b1 100755 --- a/.github/create_and_upload_logs.sh +++ b/.github/create_and_upload_logs.sh @@ -20,8 +20,8 @@ then sudo rm --recursive --force $SUITE/_instances/*/database/ git clone https://github.com/Altinity/actions.git - git --git-dir actions checkout --quiet 46f8da1ccc668e800684fffd21981b507c20d86f - python actions/scripts/scan_artifacts.py files pipeline_url.log.txt version.log.txt raw.log nice-new-fails.log.txt fails.log.txt report.html $SUITE/_instances $SUITE/_service_logs + git --git-dir actions checkout --quiet 31b9ccc8b1c6e89f696c090ac8459bff401d77a7 + python actions/scripts/scan_artifacts.py files pipeline_url.log.txt version.log.txt raw.log nice-new-fails.log.txt fails.log.txt report.html $SUITE/_service_logs && python actions/scripts/scan_artifacts.py files --pattern '[A-Z_]*(SECRET|ACCESS_KEY|TOKEN)[A-Z_]*' $SUITE/_instances if [[ $? -ne 0 ]]; then # Leaked strings were found exit 1 From a254b2a508816e3a939aa4f4cd1987572ea81564 Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Wed, 2 Apr 2025 11:54:43 -0400 Subject: [PATCH 5/6] fix --- .github/create_and_upload_logs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/create_and_upload_logs.sh b/.github/create_and_upload_logs.sh index b64d696b1..4f9b006a5 100755 --- a/.github/create_and_upload_logs.sh +++ b/.github/create_and_upload_logs.sh @@ -20,7 +20,7 @@ then sudo rm --recursive --force $SUITE/_instances/*/database/ git clone https://github.com/Altinity/actions.git - git --git-dir actions checkout --quiet 31b9ccc8b1c6e89f696c090ac8459bff401d77a7 + git --git-dir actions checkout --quiet 4e8c6ced35e0ba476e0a0abe080c157413a6efd9 python actions/scripts/scan_artifacts.py files pipeline_url.log.txt version.log.txt raw.log nice-new-fails.log.txt fails.log.txt report.html $SUITE/_service_logs && python actions/scripts/scan_artifacts.py files --pattern '[A-Z_]*(SECRET|ACCESS_KEY|TOKEN)[A-Z_]*' $SUITE/_instances if [[ $? -ne 0 ]]; then # Leaked strings were found From 9efd9943dde33b12301f7f2a82f04315b8a3221d Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Wed, 2 Apr 2025 12:21:50 -0400 Subject: [PATCH 6/6] fix --- .github/create_and_upload_logs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/create_and_upload_logs.sh b/.github/create_and_upload_logs.sh index 4f9b006a5..05a8761cc 100755 --- a/.github/create_and_upload_logs.sh +++ b/.github/create_and_upload_logs.sh @@ -20,7 +20,7 @@ then sudo rm --recursive --force $SUITE/_instances/*/database/ git clone https://github.com/Altinity/actions.git - git --git-dir actions checkout --quiet 4e8c6ced35e0ba476e0a0abe080c157413a6efd9 + git --git-dir actions checkout --quiet ab328a10501cf1f40f46d46fe9bad1d5cfeb6279 python actions/scripts/scan_artifacts.py files pipeline_url.log.txt version.log.txt raw.log nice-new-fails.log.txt fails.log.txt report.html $SUITE/_service_logs && python actions/scripts/scan_artifacts.py files --pattern '[A-Z_]*(SECRET|ACCESS_KEY|TOKEN)[A-Z_]*' $SUITE/_instances if [[ $? -ne 0 ]]; then # Leaked strings were found