Skip to content
This repository was archived by the owner on Jan 10, 2021. It is now read-only.

committing WIP on adding repo protections #14

Open
JJediny wants to merge 11 commits into
base: master
Choose a base branch
from
Open

committing WIP on adding repo protections #14

Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
6e6b428
add support for all available Terraform fields for repos
Jul 8, 2019
69eed45
first grep fails to return & make list alphabetical
JJediny Aug 21, 2019
ecaa518
Merge remote-tracking branch 'patch-2/patch-2' into merge-upstream
JJediny Aug 28, 2019
7f358f8
deconflicting upstream merges
JJediny Aug 28, 2019
605ec26
Update terraform-import-github-org.sh
JJediny Aug 28, 2019
e91aa03
Merge pull request #1 from 18F/merge-upstream
JJediny Aug 28, 2019
f6a5cbe
fix rebasing
JJediny Aug 28, 2019
8292feb
Update terraform-import-github-org.sh
JJediny Aug 30, 2019
636bef1
Merge pull request #2 from 18F/feature-add-protected-branches
JJediny Aug 30, 2019
e32b6af
Update terraform-import-github-org.sh
JJediny Sep 5, 2019
c425326
Merge pull request #3 from 18F/feature-add-protected-branches
afeld Sep 6, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
208 changes: 157 additions & 51 deletions terraform-import-github-org.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,22 +1,21 @@
#!/bin/bash
# set -euo pipefail
set -euo pipefail

###
## GLOBAL VARIABLES
###
# debug mode
set -x

# ENVIRONMENTAL VARIABLES
GITHUB_TOKEN=${GITHUB_TOKEN:-''}
ORG=${ORG:-''}
API_URL_PREFIX=${API_URL_PREFIX:-'https://api.github.com'}

###
## FUNCTIONS
###
# FUNCTIONS

# Public Repos
# You can only list 100 items per page, so you can only clone 100 at a time.
# This function uses the API to calculate how many pages of public repos you have.
get_public_pagination () {
public_pages=$(curl -I "${API_URL_PREFIX}/orgs/${ORG}/repos?access_token=${GITHUB_TOKEN}&type=public&per_page=100" | grep -Eo '&page=\d+' | grep -Eo '[0-9]+' | tail -1;)
public_pages=$(curl -I "${API_URL_PREFIX}/orgs/${ORG}/repos?access_token=${GITHUB_TOKEN}&per_page=100" | grep -Eo '&page=[0-9]+' | grep -Eo '[0-9]+' | tail -1;)
echo "${public_pages:-1}"
}
# This function uses the output from above and creates an array counting from 1->$
Expand All @@ -28,42 +27,129 @@ limit_public_pagination () {
import_public_repos () {
for PAGE in $(limit_public_pagination); do

for i in $(curl -s "${API_URL_PREFIX}/orgs/${ORG}/repos?access_token=${GITHUB_TOKEN}&type=public&page=${PAGE}&per_page=100" | jq -r 'sort_by(.name) | .[] | .name'); do


PUBLIC_REPO_DESCRIPTION=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${i}?access_token=${GITHUB_TOKEN}" | jq -r .description | sed "s/\"/'/g")
PUBLIC_REPO_DOWNLOADS=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${i}?access_token=${GITHUB_TOKEN}" | jq -r .has_downloads)

PUBLIC_REPO_WIKI=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${i}?access_token=${GITHUB_TOKEN}" | jq -r .has_wiki)

PUBLIC_REPO_ISSUES=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${i}?access_token=${GITHUB_TOKEN}" | jq -r .has_issues)
for i in $(curl -s "${API_URL_PREFIX}/orgs/${ORG}/repos?access_token=${GITHUB_TOKEN}&page=${PAGE}&per_page=100&sort=full_name" | jq -r 'sort_by(.name) | .[] | .name'); do

PUBLIC_REPO_ARCHIVED=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${i}?access_token=${GITHUB_TOKEN}" | jq -r .archived)
#avoid abusing the github api and reread the file from memory cache
PUBLIC_REPO_PAYLOAD=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${i}?access_token=${GITHUB_TOKEN}&" -H "Accept: application/vnd.github.mercy-preview+json")

PUBLIC_REPO_DESCRIPTION=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r '.description | select(type == "string")' | sed "s/\"/'/g")
PUBLIC_REPO_DOWNLOADS=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r .has_downloads)
PUBLIC_REPO_WIKI=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r .has_wiki)
PUBLIC_REPO_ISSUES=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r .has_issues)
PUBLIC_REPO_ARCHIVED=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r .archived)
PUBLIC_REPO_TOPICS=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r .topics)
PUBLIC_REPO_PROJECTS=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r .has_projects)
PUBLIC_REPO_MERGE_COMMIT=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r .allow_merge_commit)
PUBLIC_REPO_REBASE_MERGE=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r .allow_rebase_merge)
PUBLIC_REPO_SQUASH_MERGE=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r .allow_squash_merge)
PUBLIC_REPO_AUTO_INIT=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r '.auto_init == true')
PUBLIC_REPO_DEFAULT_BRANCH=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r .default_branch)
PUBLIC_REPO_GITIGNORE_TEMPLATE=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r .gitignore_template)
PUBLIC_REPO_LICENSE_TEMPLATE=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r '.license_template | select(type == "string")')
PUBLIC_REPO_HOMEPAGE_URL=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r '.homepage | select(type == "string")')

# Terraform doesn't like '.' in resource names, so if one exists then replace it with a dash
TERRAFORM_PUBLIC_REPO_NAME=$(echo "${i}" | tr "." "-")

cat >> github-public-repos.tf << EOF
resource "github_repository" "${TERRAFORM_PUBLIC_REPO_NAME}" {
name = "${i}"
private = false
description = "${PUBLIC_REPO_DESCRIPTION}"
has_wiki = "${PUBLIC_REPO_WIKI}"
has_downloads = "${PUBLIC_REPO_DOWNLOADS}"
has_issues = "${PUBLIC_REPO_ISSUES}"
archived = "${PUBLIC_REPO_ARCHIVED}"
name = "${i}"
topics = ${PUBLIC_REPO_TOPICS}
description = "${PUBLIC_REPO_DESCRIPTION}"
private = false
has_wiki = ${PUBLIC_REPO_WIKI}
has_projects = ${PUBLIC_REPO_PROJECTS}
has_downloads = ${PUBLIC_REPO_DOWNLOADS}
has_issues = ${PUBLIC_REPO_ISSUES}
archived = ${PUBLIC_REPO_ARCHIVED}
allow_merge_commit = ${PUBLIC_REPO_MERGE_COMMIT}
allow_rebase_merge = ${PUBLIC_REPO_REBASE_MERGE}
allow_squash_merge = ${PUBLIC_REPO_SQUASH_MERGE}
auto_init = ${PUBLIC_REPO_AUTO_INIT}
gitignore_template = ${PUBLIC_REPO_GITIGNORE_TEMPLATE}
license_template = "${PUBLIC_REPO_LICENSE_TEMPLATE}"
homepage_url = "${PUBLIC_REPO_HOMEPAGE_URL}"
}
EOF

# Import the Repo
terraform import "github_repository.${TERRAFORM_PUBLIC_REPO_NAME}" "${i}"
#terraform import "github_repository.${TERRAFORM_PUBLIC_REPO_NAME}" "${i}"

# Import function to make ${i} repo names available to it
import_repos_protected_branches
done
done
echo "~~~Completed with Public Repos~~~"
}

import_repos_protected_branches () {
# debug
#set -x

PROTECTED_BRANCH=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${i}/branches?access_token=${GITHUB_TOKEN}&protected=true" | jq -r 'sort_by(.name) | .[] | .name')

for protected_branch in ${PROTECTED_BRANCH}; do

#avoid abusing the github api and reread the file from memory cache
PROTECTION_BRANCH_PAYLOAD=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${PROTECTED_BRANCH}/branches?access_token=${GITHUB_TOKEN}&" -H "Accept: application/vnd.github.mercy-preview+json")

PUBLIC_REPO_DOWNLOADS=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r .has_downloads)
PUBLIC_REPO_WIKI=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r .has_wiki)
REPO_PROTECTION_PAYLOAD=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${i}/branches?access_token=${GITHUB_TOKEN}&" -H "Accept: application/vnd.github.mercy-preview+json")
PROTECTED_BRANCH_ENFORCE_ADMINS=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r .enforce_admins.enabled)
PROTECTED_BRANCH_REQUIRED_STATUS_CHECKS_STRICT=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r '.required_status_checks.strict')
PROTECTED_BRANCH_REQUIRED_STATUS_CHECKS_CONTEXTS=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r '.required_status_checks.contexts')
PROTECTED_BRANCH_REQUIRED_PULL_REQUEST_REVIEWS_USERS=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r '.required_pull_request_reviews.users[]?.login')
PROTECTED_BRANCH_REQUIRED_PULL_REQUEST_REVIEWS_TEAMS=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r '.required_pull_request_reviews.team[]?.slug')
PROTECTED_BRANCH_RESTRICTIONS_USERS=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r '.restrictions.users[]?.login')
PROTECTED_BRANCH_RESTRICTIONS_TEAMS=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r '.restrictions.teams[]?.slug')

# convert bash arrays into csv list
PROTECTED_BRANCH_RESTRICTIONS_USERS_LIST=$(echo "${PROTECTED_BRANCH_RESTRICTIONS_USERS}" | tr " " ", ")
PROTECTED_BRANCH_RESTRICTIONS_TEAMS_LIST=$(echo "${PROTECTED_BRANCH_RESTRICTIONS_TEAMS}" | tr " " ", ")
PROTECTED_BRANCH_REQUIRED_PULL_REQUEST_REVIEWS_USERS_LIST=$(echo "${PROTECTED_BRANCH_REQUIRED_PULL_REQUEST_REVIEWS_USERS}" | tr " " ", ")
PROTECTED_BRANCH_REQUIRED_PULL_REQUEST_REVIEWS_TEAMS_LIST=$(echo "${PROTECTED_BRANCH_REQUIRED_PULL_REQUEST_REVIEWS_TEAMS}" | tr " " ", ")


# write to terraform file
cat >> github-public-repos.tf << EOF
resource "github_branch_protection" "${i}" {
repository = "${i}"
branch = "${protected_branch}"
enforce_admins = ${PROTECTED_BRANCH_ENFORCE_ADMINS}

required_status_checks {
strict = ${PROTECTED_BRANCH_REQUIRED_STATUS_CHECKS_STRICT}
contexts = ${PROTECTED_BRANCH_REQUIRED_STATUS_CHECKS_CONTEXTS}
}

required_pull_request_reviews {
dismiss_stale_reviews = true
dismissal_users = ["${PROTECTED_BRANCH_REQUIRED_PULL_REQUEST_REVIEWS_USERS_LIST}"]
dismissal_teams = ["${PROTECTED_BRANCH_REQUIRED_PULL_REQUEST_REVIEWS_TEAMS_LIST}"]
}

restrictions {
users = ["${PROTECTED_BRANCH_RESTRICTIONS_USERS_LIST}"]
teams = ["${PROTECTED_BRANCH_RESTRICTIONS_TEAMS_LIST}"]
}
}
EOF

# terraform import github_repository
#terraform import "github_repository.${i}" "${i}"
# Import the Protected Branch
#terraform import "github_branch_protection.${protected_branch}" "${i}"
done
}

# for testing public_repos and protected_branchs functions only uncomment
#import_public_repos
#exit 0

# Private Repos
get_private_pagination () {
priv_pages=$(curl -I "${API_URL_PREFIX}/orgs/${ORG}/repos?access_token=${GITHUB_TOKEN}&type=private&per_page=100" | grep -Eo '&page=\d+' | grep -Eo '[0-9]+' | tail -1;)
priv_pages=$(curl -I "${API_URL_PREFIX}/orgs/${ORG}/repos?access_token=${GITHUB_TOKEN}&type=private&per_page=100" | grep -Eo '&page=[0-9]+' | grep -Eo '[0-9]+' | tail -1;)
echo "${priv_pages:-1}"
}

Expand All @@ -75,29 +161,47 @@ import_private_repos () {
for PAGE in $(limit_private_pagination); do

for i in $(curl -s "${API_URL_PREFIX}/orgs/${ORG}/repos?access_token=${GITHUB_TOKEN}&type=private&page=${PAGE}&per_page=100" | jq -r 'sort_by(.name) | .[] | .name'); do

PRIVATE_REPO_DESCRIPTION=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${i}?access_token=${GITHUB_TOKEN}" | jq -r .description | sed "s/\"/'/g")

PRIVATE_REPO_DOWNLOADS=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${i}?access_token=${GITHUB_TOKEN}" | jq -r .has_downloads)

PRIVATE_REPO_WIKI=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${i}?access_token=${GITHUB_TOKEN}" | jq -r .has_wiki)

PRIVATE_REPO_ISSUES=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${i}?access_token=${GITHUB_TOKEN}" | jq -r .has_issues)

PRIVATE_REPO_ARCHIVED=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${i}?access_token=${GITHUB_TOKEN}" | jq -r .archived)

#avoid abusing the github api and reread the file from memory cache
PRIVATE_REPO_PAYLOAD=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${i}?access_token=${GITHUB_TOKEN}" -H "Accept: application/vnd.github.mercy-preview+json")

PRIVATE_REPO_DESCRIPTION=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r '.description | select(type == "string")' | sed "s/\"/'/g")
PRIVATE_REPO_DOWNLOADS=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r .has_downloads)
PRIVATE_REPO_WIKI=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r .has_wiki)
PRIVATE_REPO_ISSUES=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r .has_issues)
PRIVATE_REPO_ARCHIVED=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r .archived)
PRIVATE_REPO_TOPICS=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r .topics)
PRIVATE_REPO_PROJECTS=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r .has_projects)
PRIVATE_REPO_MERGE_COMMIT=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r .allow_merge_commit)
PRIVATE_REPO_REBASE_MERGE=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r .allow_rebase_merge)
PRIVATE_REPO_SQUASH_MERGE=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r .allow_squash_merge)
PRIVATE_REPO_AUTO_INIT=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r .auto_init)
PRIVATE_REPO_DEFAULT_BRANCH=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r .default_branch)
PRIVATE_REPO_GITIGNORE_TEMPLATE=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r .gitignore_template)
PRIVATE_REPO_LICENSE_TEMPLATE=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r '.license_template | select(type == "string")')
PRIVATE_REPO_HOMEPAGE_URL=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r '.homepage | select(type == "string")')

# Terraform doesn't like '.' in resource names, so if one exists then replace it with a dash
TERRAFORM_PRIVATE_REPO_NAME=$(echo "${i}" | tr "." "-")

cat >> github-private-repos.tf << EOF
resource "github_repository" "${TERRAFORM_PRIVATE_REPO_NAME}" {
name = "${i}"
private = true
description = "${PRIVATE_REPO_DESCRIPTION}"
has_wiki = "${PRIVATE_REPO_WIKI}"
has_downloads = "${PRIVATE_REPO_DOWNLOADS}"
has_issues = "${PRIVATE_REPO_ISSUES}"
archived = "${PRIVATE_REPO_ARCHIVED}"
name = "${i}"
private = true
description = "${PRIVATE_REPO_DESCRIPTION}"
has_wiki = ${PRIVATE_REPO_WIKI}
has_projects = ${PRIVATE_REPO_PROJECTS}
has_downloads = ${PRIVATE_REPO_DOWNLOADS}
has_issues = ${PRIVATE_REPO_ISSUES}
archived = ${PRIVATE_REPO_ARCHIVED}
topics = ${PRIVATE_REPO_TOPICS}
allow_merge_commit = ${PRIVATE_REPO_MERGE_COMMIT}
allow_rebase_merge = ${PRIVATE_REPO_REBASE_MERGE}
allow_squash_merge = ${PRIVATE_REPO_SQUASH_MERGE}
auto_init = ${PRIVATE_REPO_AUTO_INIT}
gitignore_template = ${PRIVATE_REPO_GITIGNORE_TEMPLATE}
license_template = "${PRIVATE_REPO_LICENSE_TEMPLATE}"
homepage_url = "${PRIVATE_REPO_HOMEPAGE_URL}"
}

EOF
Expand Down Expand Up @@ -133,15 +237,15 @@ import_teams () {
TEAM_DESCRIPTION=$(curl -s "${API_URL_PREFIX}/teams/${i}?access_token=${GITHUB_TOKEN}&per_page=100" -H "Accept: application/vnd.github.hellcat-preview+json" | jq -r .description)

if [[ "${TEAM_PRIVACY}" == "closed" ]]; then
cat >> "github-teams-${TEAM_NAME_NO_SPACE}.tf" << EOF
cat >> "github-teams.tf" << EOF
resource "github_team" "${TEAM_NAME_NO_SPACE}" {
name = "${TEAM_NAME}"
description = "${TEAM_DESCRIPTION}"
privacy = "closed"
}
EOF
elif [[ "${TEAM_PRIVACY}" == "secret" ]]; then
cat >> "github-teams-${TEAM_NAME_NO_SPACE}.tf" << EOF
cat >> "github-teams.tf" << EOF
resource "github_team" "${TEAM_NAME_NO_SPACE}" {
name = "${TEAM_NAME}"
description = "${TEAM_DESCRIPTION}"
Expand All @@ -165,15 +269,15 @@ import_team_memberships () {
TEAM_ROLE=$(curl -s "${API_URL_PREFIX}/teams/${i}/memberships/${j}?access_token=${GITHUB_TOKEN}&per_page=100" -H "Accept: application/vnd.github.hellcat-preview+json" | jq -r .role)

if [[ "${TEAM_ROLE}" == "maintainer" ]]; then
cat >> "github-team-memberships-${TEAM_NAME}.tf" << EOF
cat >> "github-team-memberships.tf" << EOF
resource "github_team_membership" "${TEAM_NAME}-${j}" {
username = "${j}"
team_id = "\${github_team.${TEAM_NAME}.id}"
role = "maintainer"
}
EOF
elif [[ "${TEAM_ROLE}" == "member" ]]; then
cat >> "github-team-memberships-${TEAM_NAME}.tf" << EOF
cat >> "github-team-memberships.tf" << EOF
resource "github_team_membership" "${TEAM_NAME}-${j}" {
username = "${j}"
team_id = "\${github_team.${TEAM_NAME}.id}"
Expand Down Expand Up @@ -212,7 +316,7 @@ get_team_repos () {
PULL_PERMS=$(curl -s "${API_URL_PREFIX}/teams/${TEAM_ID}/repos/${ORG}/${i}?access_token=${GITHUB_TOKEN}" -H "Accept: application/vnd.github.v3.repository+json" | jq -r .permissions.pull )

if [[ "${ADMIN_PERMS}" == "true" ]]; then
cat >> "github-teams-${TEAM_NAME}.tf" << EOF
cat >> "github-teams.tf" << EOF
resource "github_team_repository" "${TEAM_NAME}-${TERRAFORM_TEAM_REPO_NAME}" {
team_id = "${TEAM_ID}"
repository = "${i}"
Expand All @@ -221,7 +325,7 @@ resource "github_team_repository" "${TEAM_NAME}-${TERRAFORM_TEAM_REPO_NAME}" {

EOF
elif [[ "${PUSH_PERMS}" == "true" ]]; then
cat >> "github-teams-${TEAM_NAME}.tf" << EOF
cat >> "github-teams.tf" << EOF
resource "github_team_repository" "${TEAM_NAME}-${TERRAFORM_TEAM_REPO_NAME}" {
team_id = "${TEAM_ID}"
repository = "${i}"
Expand All @@ -230,7 +334,7 @@ resource "github_team_repository" "${TEAM_NAME}-${TERRAFORM_TEAM_REPO_NAME}" {

EOF
elif [[ "${PULL_PERMS}" == "true" ]]; then
cat >> "github-teams-${TEAM_NAME}.tf" << EOF
cat >> "github-teams.tf" << EOF
resource "github_team_repository" "${TEAM_NAME}-${TERRAFORM_TEAM_REPO_NAME}" {
team_id = "${TEAM_ID}"
repository = "${i}"
Expand Down Expand Up @@ -260,6 +364,8 @@ import_all_team_resources () {
## DO IT YO
###
import_public_repos
#import_repos_protected_branches
# to test set the vars that you need here and then call the function so you can.
import_private_repos
import_users
import_all_team_resources