Skip to content

Commit dff73b4

Browse files
nickbpnickbp
authored
INFINITY-1119/INFINITY-2898 (MINOR DEVTOOL CHANGES) Publish/reuse default CLIs, use local GOPATH (d2iq-archive#1986)
* INFINITY-1119 (DEVTOOL CHANGES) Build and upload default CLIs, use local GOPATH - When releasing the SDK, include builds of a "default" CLI. All but two services can just use this default CLI, and thereby wouldn't directly need any golang tooling to be built. - When building Go binaries, create a local GOPATH in $REPO_ROOT/.gopath/ instead of modifying the host GOPATH. This is simplified by the fact that we already require libraries to be vendored. This cleanup results in some MINOR DEVELOPER-FACING TOOLING CHANGES, specifically with `build_framework.sh`: - The `--cli-only` option has been removed as most services won't be building a CLI here anyway. - `CLI_DIR` should be set to `disable` by any service who aren't customizing their CLI (nearly all services), and any local CLI code in those repos can be `rm -rf`ed. Those services also won't necessarily need `go` to build the service anymore! If a custom CLI is being used, the path to that CLI should be provided via `CLI_DIR`, see kafka in dcos-commons for an example. - Renamed optional `CLI_BUILD_SKIP_UPX` option to just `SKIP_UPX`, as the setting applies to all golang builds, not just CLIs. In practice it's unlikely that anyone's using this setting. * Build cleanup rabbit hole: Have services explicitly build their artifacts * Remove unused github_update, update docs * Update build_go_exe.sh to always require at least one platform be specified
1 parent 8ba4389 commit dff73b4

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+509
-856
lines changed

Diff for: ‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
target/
22
*.pyc
33
.cache/
4+
.gopath/
45

56
# dcos-docker artifacts
67
dcos-docker/

Diff for: ‎build.gradle

-7
Original file line numberDiff line numberDiff line change
@@ -112,13 +112,6 @@ allprojects {
112112
events 'passed', 'skipped', 'failed'
113113
}
114114
}
115-
116-
task copyExecutor(type: Copy) {
117-
from('src/../../../sdk/executor/build/distributions/') {
118-
include '**/*'
119-
}
120-
into('build/distributions')
121-
}
122115
}
123116

124117
ext {

Diff for: ‎build.sh

+47-50
Original file line numberDiff line numberDiff line change
@@ -1,73 +1,70 @@
1-
#!/bin/bash
2-
3-
# This script does a full build/test of SDK artifacts. This does not upload the artifacts, instead
4-
# see test.sh. This script (and test.sh) are executed by CI upon pull requests to the repository, or
5-
# may be run locally by developers.
6-
1+
#!/usr/bin/env bash
72
set -e
83

9-
PULLREQUEST="false"
10-
MERGE_FROM="master"
11-
while getopts 'pt:g:j:' opt; do
4+
usage() {
5+
# This script does a full build/test of the SDK's own artifacts: bootstrap, executor, and default CLI
6+
# This does not upload the artifacts, instead see test.sh or frameworks/*/build.sh.
7+
# This script (and test.sh) are executed by CI upon pull requests to the repository, or may be run locally by developers.
8+
echo "Syntax: build.sh [options]"
9+
echo "Options:"
10+
echo " -b Just build -- skip tests (or set GO_TESTS=false and JAVA_TESTS=false)"
11+
echo "Env:"
12+
echo " BOOTSTRAP_DIR: Custom directory where bootstrap is located, or 'disable' to disable bootstrap build"
13+
echo " CLI_DIR: Custom directory where default CLI is located, or 'disable' to disable default CLI build"
14+
echo " EXECUTOR_DIR: Custom directory where executor is located, or 'disable' to disable executor build"
15+
}
16+
17+
while getopts 'b' opt; do
1218
case $opt in
13-
p)
14-
PULLREQUEST="true"
15-
;;
16-
t)
17-
# hack for testing
18-
MERGE_FROM="$OPTARG"
19-
;;
20-
j)
21-
JAVA_TESTS="$OPTARG"
22-
;;
23-
g)
24-
GO_TESTS="$OPTARG"
19+
b)
20+
JAVA_TESTS="false"
21+
export GO_TESTS="false"
2522
;;
2623
\?)
27-
echo "Unknown option. supported: -p for pull request" >&2
28-
echo "$opt"
24+
usage
2925
exit 1
3026
;;
3127
esac
3228
done
3329
shift $((OPTIND-1))
3430

3531
REPO_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
36-
cd $REPO_ROOT_DIR
3732

38-
if [ x$PULLREQUEST = "xtrue" ]; then
39-
# GitHub notifier: reset any statuses from prior builds for this commit
40-
$REPO_ROOT_DIR/tools/github_update.py reset
33+
BOOTSTRAP_DIR=${BOOTSTRAP_DIR:=${REPO_ROOT_DIR}/sdk/bootstrap}
34+
CLI_DIR=${CLI_DIR:=${REPO_ROOT_DIR}/sdk/cli}
35+
EXECUTOR_DIR=${EXECUTOR_DIR:=${REPO_ROOT_DIR}/sdk/executor}
4136

42-
echo "Creating fake user and merging changes from master."
43-
# git won't let you update files without knowing a name
44-
git config user.email [email protected]
45-
git config user.name Infinity-tools-fake-user
46-
# Update local branch to include github version of master.
47-
git pull origin $MERGE_FROM --no-commit --ff
48-
fi
37+
DISABLED_VALUE="disable"
38+
39+
cd $REPO_ROOT_DIR
4940

41+
##
42+
# GO BUILD
43+
##
5044

51-
if [ x"${GO_TESTS:-true}" == x"true" ]; then
52-
if [ ! -d $GOPATH ]; then
53-
mkdir -p $GOPATH
54-
fi
45+
# Build Go bits: bootstrap and default CLI
46+
BOOTSTRAP_ARTIFACT=""
47+
if [ "$BOOTSTRAP_DIR" != $DISABLED_VALUE ]; then
48+
# Produces: ${BOOTSTRAP_DIR}/bootstrap.zip
49+
${BOOTSTRAP_DIR}/build.sh
50+
fi
51+
if [ "$CLI_DIR" != $DISABLED_VALUE ]; then
52+
# Produces: ${CLI_DIR}/dcos-service-cli-linux ${CLI_DIR}/dcos-service-cli-darwin ${CLI_DIR}/dcos-service-cli.exe
53+
${CLI_DIR}/build.sh
54+
fi
5555

56-
# TODO: Should we do something more along the lines of what is done in build_go_exe.sh?
57-
if [ ! -e $GOPATH/src ]; then
58-
ln -s $REPO_ROOT_DIR/govendor $GOPATH/src
59-
fi
56+
##
57+
# JAVA BUILD
58+
##
6059

61-
# Verify golang-based projects: SDK CLI (exercised via helloworld, our model service)
62-
# and SDK bootstrap, run unit tests
63-
for golang_sub_project in cli sdk/bootstrap; do
64-
pushd $golang_sub_project
65-
go test -v ./...
66-
popd
67-
done
60+
# Build Java bits: Executor
61+
if [ "$EXECUTOR_DIR" != $DISABLED_VALUE ]; then
62+
echo "Building executor in $EXECUTOR_DIR"
63+
./gradlew distZip -p $EXECUTOR_DIR
6864
fi
6965

66+
# Run ALL Java unit tests
7067
if [ x"${JAVA_TESTS:-true}" == x"true" ]; then
7168
# Build steps for SDK libraries:
72-
./gradlew clean jar test check
69+
./gradlew clean check
7370
fi

Diff for: ‎frameworks/cassandra/build.gradle

-3
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,6 @@ dependencies {
2424
testCompile project(":testing")
2525
}
2626

27-
distZip.dependsOn ":executor:distZip"
28-
distZip.finalizedBy copyExecutor
29-
3027
distributions {
3128
main {
3229
baseName = 'cassandra-scheduler'

Diff for: ‎frameworks/cassandra/build.sh

+17-4
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,24 @@
22
set -e
33

44
FRAMEWORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
5-
BUILD_DIR=$FRAMEWORK_DIR/build/distributions
5+
REPO_ROOT_DIR=$(dirname $(dirname $FRAMEWORK_DIR))
66

7-
$FRAMEWORK_DIR/../../tools/build_framework.sh \
7+
# Build SDK artifacts (executor, clis, bootstrap) to be included in our release, but skip SDK tests
8+
# since since that's not in our scope. Projects that aren't colocated in dcos-commons should skip
9+
# this step, and should omit the "REPO_ROOT_DIR" artifacts listed below.
10+
$REPO_ROOT_DIR/build.sh -b
11+
12+
# Build/test our scheduler.zip
13+
${REPO_ROOT_DIR}/gradlew -p ${FRAMEWORK_DIR} check distZip
14+
15+
# Build package with our scheduler.zip and the local SDK artifacts we built:
16+
$REPO_ROOT_DIR/tools/build_package.sh \
817
beta-cassandra \
918
$FRAMEWORK_DIR \
10-
--artifact "$BUILD_DIR/executor.zip" \
11-
--artifact "$BUILD_DIR/$(basename $FRAMEWORK_DIR)-scheduler.zip" \
19+
-a "$FRAMEWORK_DIR/build/distributions/$(basename $FRAMEWORK_DIR)-scheduler.zip" \
20+
-a "$REPO_ROOT_DIR/sdk/executor/build/distributions/executor.zip" \
21+
-a "$REPO_ROOT_DIR/sdk/bootstrap/bootstrap.zip" \
22+
-a "$REPO_ROOT_DIR/sdk/cli/dcos-service-cli-linux" \
23+
-a "$REPO_ROOT_DIR/sdk/cli/dcos-service-cli-darwin" \
24+
-a "$REPO_ROOT_DIR/sdk/cli/dcos-service-cli.exe" \
1225
$@

Diff for: ‎frameworks/cassandra/cli/dcos-cassandra/vendor

-1
This file was deleted.

Diff for: ‎frameworks/elastic/build.gradle

-3
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,6 @@ dependencies {
2424
testCompile project(":testing")
2525
}
2626

27-
distZip.dependsOn ":executor:distZip"
28-
distZip.finalizedBy copyExecutor
29-
3027
distributions {
3128
main {
3229
baseName = 'elastic-scheduler'

Diff for: ‎frameworks/elastic/build.sh

+17-4
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,29 @@
22
set -e
33

44
FRAMEWORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
5-
BUILD_DIR=$FRAMEWORK_DIR/build/distributions
5+
REPO_ROOT_DIR=$(dirname $(dirname $FRAMEWORK_DIR))
66

77
# grab TEMPLATE_x vars for use in universe template:
88
source $FRAMEWORK_DIR/versions.sh
99

10-
$FRAMEWORK_DIR/../../tools/build_framework.sh \
10+
# Build SDK artifacts (executor, clis, bootstrap) to be included in our release, but skip SDK tests
11+
# since since that's not in our scope. Projects that aren't colocated in dcos-commons should skip
12+
# this step, and should omit the "REPO_ROOT_DIR" artifacts listed below.
13+
$REPO_ROOT_DIR/build.sh -b
14+
15+
# Build/test our scheduler.zip
16+
${REPO_ROOT_DIR}/gradlew -p ${FRAMEWORK_DIR} check distZip
17+
18+
# Build package with our scheduler.zip and the local SDK artifacts we built:
19+
$REPO_ROOT_DIR/tools/build_package.sh \
1120
beta-elastic \
1221
$FRAMEWORK_DIR \
13-
--artifact "$BUILD_DIR/executor.zip" \
14-
--artifact "$BUILD_DIR/$(basename $FRAMEWORK_DIR)-scheduler.zip" \
22+
-a "$FRAMEWORK_DIR/build/distributions/$(basename $FRAMEWORK_DIR)-scheduler.zip" \
23+
-a "$REPO_ROOT_DIR/sdk/executor/build/distributions/executor.zip" \
24+
-a "$REPO_ROOT_DIR/sdk/bootstrap/bootstrap.zip" \
25+
-a "$REPO_ROOT_DIR/sdk/cli/dcos-service-cli-linux" \
26+
-a "$REPO_ROOT_DIR/sdk/cli/dcos-service-cli-darwin" \
27+
-a "$REPO_ROOT_DIR/sdk/cli/dcos-service-cli.exe" \
1528
$@
1629

1730
# Chain to build kibana as well

Diff for: ‎frameworks/elastic/cli/dcos-elastic/main.go

-12
This file was deleted.

Diff for: ‎frameworks/elastic/cli/dcos-elastic/vendor

-1
This file was deleted.

Diff for: ‎frameworks/hdfs/build.gradle

-3
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,6 @@ dependencies {
2424
testCompile project(":testing")
2525
}
2626

27-
distZip.dependsOn ":executor:distZip"
28-
distZip.finalizedBy copyExecutor
29-
3027
distributions {
3128
main {
3229
baseName = 'hdfs-scheduler'

Diff for: ‎frameworks/hdfs/build.sh

+17-4
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,24 @@
22
set -e
33

44
FRAMEWORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
5-
BUILD_DIR=$FRAMEWORK_DIR/build/distributions
5+
REPO_ROOT_DIR=$(dirname $(dirname $FRAMEWORK_DIR))
66

7-
$FRAMEWORK_DIR/../../tools/build_framework.sh \
7+
# Build SDK artifacts (executor, clis, bootstrap) to be included in our release, but skip SDK tests
8+
# since since that's not in our scope. Projects that aren't colocated in dcos-commons should skip
9+
# this step, and should omit the "REPO_ROOT_DIR" artifacts listed below.
10+
$REPO_ROOT_DIR/build.sh -b
11+
12+
# Build/test our scheduler.zip
13+
${REPO_ROOT_DIR}/gradlew -p ${FRAMEWORK_DIR} check distZip
14+
15+
# Build package with our scheduler.zip and the local SDK artifacts we built:
16+
$REPO_ROOT_DIR/tools/build_package.sh \
817
beta-hdfs \
918
$FRAMEWORK_DIR \
10-
--artifact "$BUILD_DIR/executor.zip" \
11-
--artifact "$BUILD_DIR/$(basename $FRAMEWORK_DIR)-scheduler.zip" \
19+
-a "$FRAMEWORK_DIR/build/distributions/$(basename $FRAMEWORK_DIR)-scheduler.zip" \
20+
-a "$REPO_ROOT_DIR/sdk/executor/build/distributions/executor.zip" \
21+
-a "$REPO_ROOT_DIR/sdk/bootstrap/bootstrap.zip" \
22+
-a "$REPO_ROOT_DIR/sdk/cli/dcos-service-cli-linux" \
23+
-a "$REPO_ROOT_DIR/sdk/cli/dcos-service-cli-darwin" \
24+
-a "$REPO_ROOT_DIR/sdk/cli/dcos-service-cli.exe" \
1225
$@

Diff for: ‎frameworks/hdfs/cli/dcos-hdfs/.gitignore

-4
This file was deleted.

Diff for: ‎frameworks/hdfs/cli/dcos-hdfs/main.go

-12
This file was deleted.

Diff for: ‎frameworks/hdfs/cli/dcos-hdfs/vendor

-1
This file was deleted.

Diff for: ‎frameworks/helloworld/build.gradle

+2-2
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@ task copyKeystoreApp(type: Copy) {
3131
into('build/distributions')
3232
}
3333

34-
distZip.dependsOn ":executor:distZip", ":keystore-app:integrationTestZip"
35-
distZip.finalizedBy copyExecutor, copyKeystoreApp
34+
distZip.dependsOn ":keystore-app:integrationTestZip"
35+
distZip.finalizedBy copyKeystoreApp
3636

3737
distributions {
3838
main {

Diff for: ‎frameworks/helloworld/build.sh

+21-10
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,26 @@
22
set -e
33

44
FRAMEWORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
5-
BUILD_DIR=$FRAMEWORK_DIR/build/distributions
5+
REPO_ROOT_DIR=$(dirname $(dirname $FRAMEWORK_DIR))
66

7-
FRAMEWORK_NAME=hello-world # dir name is 'helloworld', while framework name is 'hello-world'
7+
# Build SDK artifacts (executor, clis, bootstrap) to be included in our release, but skip SDK tests
8+
# since since that's not in our scope. Projects that aren't colocated in dcos-commons should skip
9+
# this step, and should omit the "REPO_ROOT_DIR" artifacts listed below.
10+
$REPO_ROOT_DIR/build.sh -b
811

9-
DOCUMENTATION_PATH="https://github.com/mesosphere/dcos-commons/blob/master/frameworks/helloworld/README.md" \
10-
$FRAMEWORK_DIR/../../tools/build_framework.sh \
11-
$FRAMEWORK_NAME \
12-
$FRAMEWORK_DIR \
13-
--artifact "$BUILD_DIR/executor.zip" \
14-
--artifact "$BUILD_DIR/${FRAMEWORK_NAME}-scheduler.zip" \
15-
--artifact "$BUILD_DIR/keystore-app.zip" \
16-
$@
12+
# Build/test our scheduler.zip and keystore-app.zip
13+
${REPO_ROOT_DIR}/gradlew -p ${FRAMEWORK_DIR} check distZip
14+
15+
# Build package with our scheduler.zip/keystore-app.zip and the local SDK artifacts we built:
16+
TEMPLATE_DOCUMENTATION_PATH="https://github.com/mesosphere/dcos-commons/blob/master/frameworks/helloworld/README.md" \
17+
$REPO_ROOT_DIR/tools/build_package.sh \
18+
hello-world \
19+
$FRAMEWORK_DIR \
20+
-a "$FRAMEWORK_DIR/build/distributions/keystore-app.zip" \
21+
-a "$FRAMEWORK_DIR/build/distributions/hello-world-scheduler.zip" \
22+
-a "$REPO_ROOT_DIR/sdk/executor/build/distributions/executor.zip" \
23+
-a "$REPO_ROOT_DIR/sdk/bootstrap/bootstrap.zip" \
24+
-a "$REPO_ROOT_DIR/sdk/cli/dcos-service-cli-linux" \
25+
-a "$REPO_ROOT_DIR/sdk/cli/dcos-service-cli-darwin" \
26+
-a "$REPO_ROOT_DIR/sdk/cli/dcos-service-cli.exe" \
27+
$@

Diff for: ‎frameworks/helloworld/cli/dcos-hello-world/.gitignore

-4
This file was deleted.

Diff for: ‎frameworks/helloworld/cli/dcos-hello-world/main.go

-12
This file was deleted.

Diff for: ‎frameworks/helloworld/cli/dcos-hello-world/vendor

-1
This file was deleted.

Diff for: ‎frameworks/kafka/build.gradle

-3
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,6 @@ dependencies {
2424
testCompile project(":testing")
2525
}
2626

27-
distZip.dependsOn ":executor:distZip"
28-
distZip.finalizedBy copyExecutor
29-
3027
distributions {
3128
main {
3229
baseName = 'kafka-scheduler'

0 commit comments

Comments
 (0)