Migrate go-based CLI tools to new repo.

History available from https://git.psi.ch/MELANIE/rollout
Code was originally at path Software/00-General/go/src/scicat

.gitattributes

@@ -0,0 +1,14 @@
+# Set the default behavior, in case people don't have core.autocrlf set.
+* text=auto
+
+# Explicitly declare text files you want to always be normalized and converted
+*.go    text
+*.md    text
+*.txt   text
+*.sh    text
+*.py    text
+go.mod  text
+go.sum  text
+
+# Declare files that will always have CRLF line endings on checkout.
+*.bat text eol=crlf

.gitignore

@@ -0,0 +1 @@
+build

.gitlab-ci.yml

@@ -0,0 +1,177 @@
+# variables:
+#   IMAGE_TAG_NAME: "docker.psi.ch:5000/scicat/rollout/proposals"
+
+# .deploy-common:
+#   tags:
+#     - specific
+#     - shell
+#   rules:
+#     - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH'
+#       changes:
+#         - Software/00-General/proposals/Dockerfile
+#         - Software/00-General/proposals/helm/**/*
+#         - Software/00-General/proposals/**/*.py
+#         - .gitlab-ci.yml
+#   before_script: 
+#     - cd Software/00-General/proposals
+#   script:
+#     - >-
+#       helm upgrade sync helm/duo_facility_proposals/
+#       --install -n ${CI_ENVIRONMENT_NAME} --create-namespace
+#       --set "image.tag=${CI_COMMIT_SHORT_SHA}"
+#       --set "scicat_endpoint=${SCICAT_ENDPOINT}"
+#       --set "scicat_username=${SCICAT_USERNAME}"
+#       --set "scicat_password=${SCICAT_PASSWORD}"
+#       --set "duo_endpoint=${DUO_ENDPOINT}"
+#       --set "duo_secret=${DUO_SECRET}"
+#       --set-file "facilities_schedule=${FACILITIES_SCHEDULE}"
+
+# stages:
+#   - build
+#   - check-code-quality
+#   - deploy
+
+# build-docker-images:
+#   stage: build
+#   tags:
+#     - specific
+#     - shell
+#   rules:
+#     - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH'
+#       changes:
+#         - Software/00-General/proposals/Dockerfile
+#         - Software/00-General/proposals/helm/**/*
+#         - Software/00-General/proposals/**/*.py
+#         - .gitlab-ci.yml
+#   before_script:
+#     - cd Software/00-General/proposals
+#   script:
+#     - docker build --network host --target dev --pull -t "$IMAGE_TAG_NAME:dev-$CI_COMMIT_SHORT_SHA" .
+#     - docker build --network host --pull -t "$IMAGE_TAG_NAME:$CI_COMMIT_SHORT_SHA" .
+
+# black:
+#   stage: check-code-quality
+#   tags:
+#     - specific
+#     - shell
+#   rules:
+#     - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH'
+#       changes:
+#         - Software/00-General/proposals/**/*.py
+#   before_script:
+#     - cd Software/00-General/proposals
+#     - docker run -d --rm --name proposals -v $(pwd):/usr/src/proposals $IMAGE_TAG_NAME:dev-$CI_COMMIT_SHORT_SHA /bin/sh -c "while sleep 1000; do :; done"
+#   script:
+#     - docker exec proposals black --check --diff --no-color ./src
+#   after_script:
+#     - docker stop proposals
+
+# isort:
+#   stage: check-code-quality
+#   tags:
+#     - specific
+#     - shell
+#   rules:
+#     - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH'
+#       changes:
+#         - Software/00-General/proposals/**/*.py
+#   before_script:
+#     - cd Software/00-General/proposals
+#     - docker run -d --rm --name proposals -v $(pwd):/usr/src/proposals $IMAGE_TAG_NAME:dev-$CI_COMMIT_SHORT_SHA /bin/sh -c "while sleep 1000; do :; done"
+#   script:
+#     - docker exec proposals isort --check --diff ./src
+#   after_script:
+#     - docker stop proposals
+
+# pylint:
+#   stage: check-code-quality
+#   tags:
+#     - specific
+#     - shell
+#   allow_failure: true
+#   rules:
+#     - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH'
+#       changes:
+#         - Software/00-General/proposals/**/*.py
+#   before_script:
+#     - cd Software/00-General/proposals
+#     - docker run -d --rm --name proposals -v $(pwd):/usr/src/proposals $IMAGE_TAG_NAME:dev-$CI_COMMIT_SHORT_SHA /bin/sh -c "while sleep 1000; do :; done"
+#   script:
+#     - docker exec proposals pylint ./src
+#   after_script:
+#     - docker stop proposals
+
+# test:
+#   stage: check-code-quality
+#   tags:
+#     - specific
+#     - shell
+#   rules:
+#     - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH'
+#       changes:
+#         - Software/00-General/proposals/**/*.py
+#   before_script:
+#     - cd Software/00-General/proposals
+#     - docker run -d --rm --name proposals -v $(pwd):/usr/src/proposals $IMAGE_TAG_NAME:dev-$CI_COMMIT_SHORT_SHA /bin/sh -c "while sleep 1000; do :; done"
+#   script:
+#     - docker exec proposals pytest -p no:cacheprovider ./src/tests
+#   after_script:
+#     - docker stop proposals
+
+# docker-push:
+#   stage: deploy
+#   tags:
+#     - specific
+#     - shell
+#   rules:
+#     - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH'
+#       changes:
+#         - Software/00-General/proposals/Dockerfile
+#         - Software/00-General/proposals/helm/**/*
+#         - Software/00-General/proposals/**/*.py
+#         - .gitlab-ci.yml
+#   before_script:
+#     - cd Software/00-General/proposals
+#   script:
+#     - docker push $IMAGE_TAG_NAME:$CI_COMMIT_SHORT_SHA
+
+# deploy-k8s-qa:
+#   stage: deploy
+#   extends: .deploy-common
+#   environment: qa
+
+# deploy-k8s-prod:
+#   stage: deploy
+#   extends: .deploy-common
+#   environment: production
+#   before_script:
+#     - cd Software/00-General/proposals
+#     - docker build --network host --pull -t "$IMAGE_TAG_NAME:$CI_COMMIT_SHORT_SHA" -t "$IMAGE_TAG_NAME:latest" .
+#     - docker push "$IMAGE_TAG_NAME"
+#   rules:
+#     - if: '$CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $CI_PROJECT_ID == "5133"'
+#       changes:
+#         - Software/00-General/proposals/Dockerfile
+#         - Software/00-General/proposals/helm/**/*
+#         - Software/00-General/proposals/**/*.py
+#         - .gitlab-ci.yml
+#   after_script:
+#     - docker rmi $IMAGE_TAG_NAME:$CI_COMMIT_SHORT_SHA
+#     - docker rmi $IMAGE_TAG_NAME:latest
+
+# clean-up:
+#   stage: .post
+#   tags:
+#     - specific
+#     - shell
+#   rules:
+#     - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH'
+#       changes:
+#         - Software/00-General/proposals/Dockerfile
+#         - Software/00-General/proposals/helm/**/*
+#         - Software/00-General/proposals/**/*.py
+#         - .gitlab-ci.yml
+#       when: always
+#   script:
+#     - docker rmi $IMAGE_TAG_NAME:dev-$CI_COMMIT_SHORT_SHA
+#     - docker rmi $IMAGE_TAG_NAME:$CI_COMMIT_SHORT_SHA

build.sh

@@ -0,0 +1,41 @@
+#!/bin/sh
+# Compiles all tools
+# Usage: build.sh [OS list]
+# Valid OS: linux, darwin, windows
+set -e
+
+OS="${@:-linux windows}"
+APPS="datasetArchiver datasetGetProposal datasetRetriever datasetIngestor datasetCleaner"
+PREFIX="build"
+
+for APP in $APPS; do
+    for GOOS in $OS; do
+        echo "Building $APP for $GOOS"
+
+        GO_TOOL=cmd/${APP}
+        OUT="$PREFIX/$GOOS"
+        mkdir -p "$OUT"
+
+        case $GOOS in
+        windows)
+            GOARCH=amd64 GOOS=$GOOS go build -o $OUT/${APP}.exe $GO_TOOL/main.go
+            ;;
+        darwin)
+            GOARCH=amd64 GOOS=$GOOS go build -o $OUT/${APP}-darwin-amd64 $GO_TOOL/main.go
+            GOARCH=arm64 GOOS=$GOOS go build -o $OUT/${APP}-darwin-arm64 $GO_TOOL/main.go
+            # universal binary
+            lipo -create -output $OUT/${APP} $OUT/${APP}-darwin-amd64 $OUT/${APP}-darwin-arm64
+            rm $OUT/${APP}-darwin-amd64 $OUT/${APP}-darwin-arm64
+            ;;
+        linux)
+            GOARCH=amd64 GOOS=$GOOS go build -o $OUT/${APP} $GO_TOOL/main.go
+            ;;
+        *)
+            # Best effort for other OS versions
+            echo "Warning: Unsupported OS $OS"
+            GOARCH="${GOARCH:-amd64}" GOOS=$GOOS go build -o $OUT/${APP} $GO_TOOL/main.go
+            ;;
+        esac
+
+    done
+done 

cmd/datasetArchiver/main.go

@@ -0,0 +1,128 @@
+/*
+
+This script archives all datasets in state datasetCreated from a given ownerGroup
+
+*/
+
+package main
+
+import (
+	"bufio"
+	"crypto/tls"
+	"flag"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"scicat/datasetUtils"
+	"strings"
+	"time"
+
+	"github.com/fatih/color"
+)
+
+func main() {
+
+	var client = &http.Client{
+		Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: false}},
+		Timeout:   10 * time.Second}
+
+	const PROD_API_SERVER string = "https://dacat.psi.ch/api/v3"
+	const TEST_API_SERVER string = "https://dacat-qa.psi.ch/api/v3"
+	const DEV_API_SERVER string = "https://dacat-development.psi.ch/api/v3"
+	const LOCAL_API_SERVER string = "http://localhost:3000/api/v3"
+
+	const MANUAL string = "http://melanie.gitpages.psi.ch/SciCatPages"
+	const APP = "datasetArchiver"
+	const VERSION = "1.1.9"
+	var scanner = bufio.NewScanner(os.Stdin)
+
+	var APIServer string
+	var env string
+
+	// pass parameters
+	userpass := flag.String("user", "", "Defines optional username and password")
+	token := flag.String("token", "", "Defines optional API token instead of username:password")
+	tapecopies := flag.Int("tapecopies", 1, "Number of tapecopies to be used for archiving")
+	testenvFlag := flag.Bool("testenv", false, "Use test environment (qa) instead or production")
+	localenvFlag := flag.Bool("localenv", false, "Use local environment (local) instead or production")
+	devenvFlag := flag.Bool("devenv", false, "Use development environment instead or production")
+	nonInteractiveFlag := flag.Bool("noninteractive", false, "Defines if no questions will be asked, just do it - make sure you know what you are doing")
+	showVersion := flag.Bool("version", false, "Show version number and exit")
+
+	flag.Parse()
+
+	if *showVersion {
+		fmt.Printf("%s\n", VERSION)
+		return
+	}
+
+	// check for program version only if running interactively
+	datasetUtils.CheckForNewVersion(client, APP, VERSION, true)
+
+	if *testenvFlag {
+		APIServer = TEST_API_SERVER
+		env = "test"
+	} else if *devenvFlag {
+		APIServer = DEV_API_SERVER
+		env = "dev"
+	} else if *localenvFlag {
+		APIServer = LOCAL_API_SERVER
+		env = "local"
+	} else {
+		APIServer = PROD_API_SERVER
+		env = "production"
+	}
+
+	color.Set(color.FgGreen)
+	log.Printf("You are about to archive dataset(s) to the === %s === data catalog environment...", env)
+	color.Unset()
+
+	args := flag.Args()
+	ownerGroup := ""
+	inputdatasetList := make([]string, 0)
+
+	// argsWithoutProg := os.Args[1:]
+	if len(args) == 0 {
+		fmt.Printf("\n\nTool to archive datasets to the data catalog.\n\n")
+		fmt.Printf("Run script with the following options and parameter:\n\n")
+		fmt.Printf("datasetArchiver [options] (ownerGroup | space separated list of datasetIds) \n\n")
+		fmt.Printf("You must choose either an ownerGroup, in which case all archivable datasets\n")
+		fmt.Printf("of this ownerGroup not yet archived will be archived.\n")
+		fmt.Printf("Or you choose a (list of) datasetIds, in which case all archivable datasets\n")
+		fmt.Printf("of this list not yet archived will be archived.\n\n")
+		fmt.Printf("List of options:\n\n")
+		flag.PrintDefaults()
+		fmt.Printf("\n\nFor further help see " + MANUAL + "\n\n")
+		return
+	} else if len(args) == 1 && !strings.Contains(args[0], "/") {
+		ownerGroup = args[0]
+	} else {
+		inputdatasetList = args[0:]
+	}
+
+	user, _ := datasetUtils.Authenticate(client, APIServer, token, userpass)
+
+	archivableDatasets := datasetUtils.GetArchivableDatasets(client, APIServer, ownerGroup, inputdatasetList, user["accessToken"])
+	if len(archivableDatasets) > 0 {
+		archive := ""
+		if *nonInteractiveFlag {
+			archive = "y"
+		} else {
+			fmt.Printf("\nDo you want to archive these %v datasets (y/N) ? ", len(archivableDatasets))
+			scanner.Scan()
+			archive = scanner.Text()
+		}
+		if archive != "y" {
+			log.Fatalf("Okay the archive process is stopped here, no datasets will be archived\n")
+		} else {
+			log.Printf("You chose to archive the new datasets\n")
+			log.Printf("Submitting Archive Job for the ingested datasets.\n")
+			jobId := datasetUtils.CreateJob(client, APIServer, user, archivableDatasets, tapecopies)
+			fmt.Println(jobId)
+		}
+	} else {
+		log.Fatalf("No archivable datasets remaining")
+	}
+
+}