diff --git a/terraform-import-github-org.sh b/terraform-import-github-org.sh index 7c9eb30..046c2a1 100755 --- a/terraform-import-github-org.sh +++ b/terraform-import-github-org.sh @@ -1,22 +1,21 @@ #!/bin/bash -# set -euo pipefail +set -euo pipefail -### -## GLOBAL VARIABLES -### +# debug mode +set -x + +# ENVIRONMENTAL VARIABLES GITHUB_TOKEN=${GITHUB_TOKEN:-''} ORG=${ORG:-''} API_URL_PREFIX=${API_URL_PREFIX:-'https://api.github.com'} -### -## FUNCTIONS -### +# FUNCTIONS # Public Repos # You can only list 100 items per page, so you can only clone 100 at a time. # This function uses the API to calculate how many pages of public repos you have. get_public_pagination () { - public_pages=$(curl -I "${API_URL_PREFIX}/orgs/${ORG}/repos?access_token=${GITHUB_TOKEN}&type=public&per_page=100" | grep -Eo '&page=\d+' | grep -Eo '[0-9]+' | tail -1;) + public_pages=$(curl -I "${API_URL_PREFIX}/orgs/${ORG}/repos?access_token=${GITHUB_TOKEN}&per_page=100" | grep -Eo '&page=[0-9]+' | grep -Eo '[0-9]+' | tail -1;) echo "${public_pages:-1}" } # This function uses the output from above and creates an array counting from 1->$ @@ -28,42 +27,129 @@ limit_public_pagination () { import_public_repos () { for PAGE in $(limit_public_pagination); do - for i in $(curl -s "${API_URL_PREFIX}/orgs/${ORG}/repos?access_token=${GITHUB_TOKEN}&type=public&page=${PAGE}&per_page=100" | jq -r 'sort_by(.name) | .[] | .name'); do - - - PUBLIC_REPO_DESCRIPTION=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${i}?access_token=${GITHUB_TOKEN}" | jq -r .description | sed "s/\"/'/g") - PUBLIC_REPO_DOWNLOADS=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${i}?access_token=${GITHUB_TOKEN}" | jq -r .has_downloads) - - PUBLIC_REPO_WIKI=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${i}?access_token=${GITHUB_TOKEN}" | jq -r .has_wiki) - - PUBLIC_REPO_ISSUES=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${i}?access_token=${GITHUB_TOKEN}" | jq -r .has_issues) + for i in $(curl -s "${API_URL_PREFIX}/orgs/${ORG}/repos?access_token=${GITHUB_TOKEN}&page=${PAGE}&per_page=100&sort=full_name" | jq -r 'sort_by(.name) | .[] | .name'); do - PUBLIC_REPO_ARCHIVED=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${i}?access_token=${GITHUB_TOKEN}" | jq -r .archived) + #avoid abusing the github api and reread the file from memory cache + PUBLIC_REPO_PAYLOAD=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${i}?access_token=${GITHUB_TOKEN}&" -H "Accept: application/vnd.github.mercy-preview+json") + + PUBLIC_REPO_DESCRIPTION=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r '.description | select(type == "string")' | sed "s/\"/'/g") + PUBLIC_REPO_DOWNLOADS=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r .has_downloads) + PUBLIC_REPO_WIKI=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r .has_wiki) + PUBLIC_REPO_ISSUES=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r .has_issues) + PUBLIC_REPO_ARCHIVED=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r .archived) + PUBLIC_REPO_TOPICS=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r .topics) + PUBLIC_REPO_PROJECTS=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r .has_projects) + PUBLIC_REPO_MERGE_COMMIT=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r .allow_merge_commit) + PUBLIC_REPO_REBASE_MERGE=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r .allow_rebase_merge) + PUBLIC_REPO_SQUASH_MERGE=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r .allow_squash_merge) + PUBLIC_REPO_AUTO_INIT=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r '.auto_init == true') + PUBLIC_REPO_DEFAULT_BRANCH=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r .default_branch) + PUBLIC_REPO_GITIGNORE_TEMPLATE=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r .gitignore_template) + PUBLIC_REPO_LICENSE_TEMPLATE=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r '.license_template | select(type == "string")') + PUBLIC_REPO_HOMEPAGE_URL=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r '.homepage | select(type == "string")') # Terraform doesn't like '.' in resource names, so if one exists then replace it with a dash TERRAFORM_PUBLIC_REPO_NAME=$(echo "${i}" | tr "." "-") cat >> github-public-repos.tf << EOF resource "github_repository" "${TERRAFORM_PUBLIC_REPO_NAME}" { - name = "${i}" - private = false - description = "${PUBLIC_REPO_DESCRIPTION}" - has_wiki = "${PUBLIC_REPO_WIKI}" - has_downloads = "${PUBLIC_REPO_DOWNLOADS}" - has_issues = "${PUBLIC_REPO_ISSUES}" - archived = "${PUBLIC_REPO_ARCHIVED}" + name = "${i}" + topics = ${PUBLIC_REPO_TOPICS} + description = "${PUBLIC_REPO_DESCRIPTION}" + private = false + has_wiki = ${PUBLIC_REPO_WIKI} + has_projects = ${PUBLIC_REPO_PROJECTS} + has_downloads = ${PUBLIC_REPO_DOWNLOADS} + has_issues = ${PUBLIC_REPO_ISSUES} + archived = ${PUBLIC_REPO_ARCHIVED} + allow_merge_commit = ${PUBLIC_REPO_MERGE_COMMIT} + allow_rebase_merge = ${PUBLIC_REPO_REBASE_MERGE} + allow_squash_merge = ${PUBLIC_REPO_SQUASH_MERGE} + auto_init = ${PUBLIC_REPO_AUTO_INIT} + gitignore_template = ${PUBLIC_REPO_GITIGNORE_TEMPLATE} + license_template = "${PUBLIC_REPO_LICENSE_TEMPLATE}" + homepage_url = "${PUBLIC_REPO_HOMEPAGE_URL}" } EOF # Import the Repo - terraform import "github_repository.${TERRAFORM_PUBLIC_REPO_NAME}" "${i}" + #terraform import "github_repository.${TERRAFORM_PUBLIC_REPO_NAME}" "${i}" + + # Import function to make ${i} repo names available to it + import_repos_protected_branches done done +echo "~~~Completed with Public Repos~~~" +} + +import_repos_protected_branches () { +# debug +#set -x + +PROTECTED_BRANCH=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${i}/branches?access_token=${GITHUB_TOKEN}&protected=true" | jq -r 'sort_by(.name) | .[] | .name') + + for protected_branch in ${PROTECTED_BRANCH}; do + + #avoid abusing the github api and reread the file from memory cache + PROTECTION_BRANCH_PAYLOAD=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${PROTECTED_BRANCH}/branches?access_token=${GITHUB_TOKEN}&" -H "Accept: application/vnd.github.mercy-preview+json") + + PUBLIC_REPO_DOWNLOADS=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r .has_downloads) + PUBLIC_REPO_WIKI=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r .has_wiki) + REPO_PROTECTION_PAYLOAD=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${i}/branches?access_token=${GITHUB_TOKEN}&" -H "Accept: application/vnd.github.mercy-preview+json") + PROTECTED_BRANCH_ENFORCE_ADMINS=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r .enforce_admins.enabled) + PROTECTED_BRANCH_REQUIRED_STATUS_CHECKS_STRICT=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r '.required_status_checks.strict') + PROTECTED_BRANCH_REQUIRED_STATUS_CHECKS_CONTEXTS=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r '.required_status_checks.contexts') + PROTECTED_BRANCH_REQUIRED_PULL_REQUEST_REVIEWS_USERS=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r '.required_pull_request_reviews.users[]?.login') + PROTECTED_BRANCH_REQUIRED_PULL_REQUEST_REVIEWS_TEAMS=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r '.required_pull_request_reviews.team[]?.slug') + PROTECTED_BRANCH_RESTRICTIONS_USERS=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r '.restrictions.users[]?.login') + PROTECTED_BRANCH_RESTRICTIONS_TEAMS=$(echo "$PUBLIC_REPO_PAYLOAD" | jq -r '.restrictions.teams[]?.slug') + + # convert bash arrays into csv list + PROTECTED_BRANCH_RESTRICTIONS_USERS_LIST=$(echo "${PROTECTED_BRANCH_RESTRICTIONS_USERS}" | tr " " ", ") + PROTECTED_BRANCH_RESTRICTIONS_TEAMS_LIST=$(echo "${PROTECTED_BRANCH_RESTRICTIONS_TEAMS}" | tr " " ", ") + PROTECTED_BRANCH_REQUIRED_PULL_REQUEST_REVIEWS_USERS_LIST=$(echo "${PROTECTED_BRANCH_REQUIRED_PULL_REQUEST_REVIEWS_USERS}" | tr " " ", ") + PROTECTED_BRANCH_REQUIRED_PULL_REQUEST_REVIEWS_TEAMS_LIST=$(echo "${PROTECTED_BRANCH_REQUIRED_PULL_REQUEST_REVIEWS_TEAMS}" | tr " " ", ") + + + # write to terraform file + cat >> github-public-repos.tf << EOF +resource "github_branch_protection" "${i}" { + repository = "${i}" + branch = "${protected_branch}" + enforce_admins = ${PROTECTED_BRANCH_ENFORCE_ADMINS} + + required_status_checks { + strict = ${PROTECTED_BRANCH_REQUIRED_STATUS_CHECKS_STRICT} + contexts = ${PROTECTED_BRANCH_REQUIRED_STATUS_CHECKS_CONTEXTS} + } + + required_pull_request_reviews { + dismiss_stale_reviews = true + dismissal_users = ["${PROTECTED_BRANCH_REQUIRED_PULL_REQUEST_REVIEWS_USERS_LIST}"] + dismissal_teams = ["${PROTECTED_BRANCH_REQUIRED_PULL_REQUEST_REVIEWS_TEAMS_LIST}"] + } + + restrictions { + users = ["${PROTECTED_BRANCH_RESTRICTIONS_USERS_LIST}"] + teams = ["${PROTECTED_BRANCH_RESTRICTIONS_TEAMS_LIST}"] + } +} +EOF + + # terraform import github_repository + #terraform import "github_repository.${i}" "${i}" + # Import the Protected Branch + #terraform import "github_branch_protection.${protected_branch}" "${i}" + done } +# for testing public_repos and protected_branchs functions only uncomment +#import_public_repos +#exit 0 + # Private Repos get_private_pagination () { - priv_pages=$(curl -I "${API_URL_PREFIX}/orgs/${ORG}/repos?access_token=${GITHUB_TOKEN}&type=private&per_page=100" | grep -Eo '&page=\d+' | grep -Eo '[0-9]+' | tail -1;) + priv_pages=$(curl -I "${API_URL_PREFIX}/orgs/${ORG}/repos?access_token=${GITHUB_TOKEN}&type=private&per_page=100" | grep -Eo '&page=[0-9]+' | grep -Eo '[0-9]+' | tail -1;) echo "${priv_pages:-1}" } @@ -75,29 +161,47 @@ import_private_repos () { for PAGE in $(limit_private_pagination); do for i in $(curl -s "${API_URL_PREFIX}/orgs/${ORG}/repos?access_token=${GITHUB_TOKEN}&type=private&page=${PAGE}&per_page=100" | jq -r 'sort_by(.name) | .[] | .name'); do - - PRIVATE_REPO_DESCRIPTION=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${i}?access_token=${GITHUB_TOKEN}" | jq -r .description | sed "s/\"/'/g") - - PRIVATE_REPO_DOWNLOADS=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${i}?access_token=${GITHUB_TOKEN}" | jq -r .has_downloads) - - PRIVATE_REPO_WIKI=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${i}?access_token=${GITHUB_TOKEN}" | jq -r .has_wiki) - - PRIVATE_REPO_ISSUES=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${i}?access_token=${GITHUB_TOKEN}" | jq -r .has_issues) - - PRIVATE_REPO_ARCHIVED=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${i}?access_token=${GITHUB_TOKEN}" | jq -r .archived) + + #avoid abusing the github api and reread the file from memory cache + PRIVATE_REPO_PAYLOAD=$(curl -s "${API_URL_PREFIX}/repos/${ORG}/${i}?access_token=${GITHUB_TOKEN}" -H "Accept: application/vnd.github.mercy-preview+json") + + PRIVATE_REPO_DESCRIPTION=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r '.description | select(type == "string")' | sed "s/\"/'/g") + PRIVATE_REPO_DOWNLOADS=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r .has_downloads) + PRIVATE_REPO_WIKI=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r .has_wiki) + PRIVATE_REPO_ISSUES=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r .has_issues) + PRIVATE_REPO_ARCHIVED=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r .archived) + PRIVATE_REPO_TOPICS=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r .topics) + PRIVATE_REPO_PROJECTS=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r .has_projects) + PRIVATE_REPO_MERGE_COMMIT=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r .allow_merge_commit) + PRIVATE_REPO_REBASE_MERGE=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r .allow_rebase_merge) + PRIVATE_REPO_SQUASH_MERGE=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r .allow_squash_merge) + PRIVATE_REPO_AUTO_INIT=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r .auto_init) + PRIVATE_REPO_DEFAULT_BRANCH=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r .default_branch) + PRIVATE_REPO_GITIGNORE_TEMPLATE=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r .gitignore_template) + PRIVATE_REPO_LICENSE_TEMPLATE=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r '.license_template | select(type == "string")') + PRIVATE_REPO_HOMEPAGE_URL=$(echo "$PRIVATE_REPO_PAYLOAD" | jq -r '.homepage | select(type == "string")') # Terraform doesn't like '.' in resource names, so if one exists then replace it with a dash TERRAFORM_PRIVATE_REPO_NAME=$(echo "${i}" | tr "." "-") cat >> github-private-repos.tf << EOF resource "github_repository" "${TERRAFORM_PRIVATE_REPO_NAME}" { - name = "${i}" - private = true - description = "${PRIVATE_REPO_DESCRIPTION}" - has_wiki = "${PRIVATE_REPO_WIKI}" - has_downloads = "${PRIVATE_REPO_DOWNLOADS}" - has_issues = "${PRIVATE_REPO_ISSUES}" - archived = "${PRIVATE_REPO_ARCHIVED}" + name = "${i}" + private = true + description = "${PRIVATE_REPO_DESCRIPTION}" + has_wiki = ${PRIVATE_REPO_WIKI} + has_projects = ${PRIVATE_REPO_PROJECTS} + has_downloads = ${PRIVATE_REPO_DOWNLOADS} + has_issues = ${PRIVATE_REPO_ISSUES} + archived = ${PRIVATE_REPO_ARCHIVED} + topics = ${PRIVATE_REPO_TOPICS} + allow_merge_commit = ${PRIVATE_REPO_MERGE_COMMIT} + allow_rebase_merge = ${PRIVATE_REPO_REBASE_MERGE} + allow_squash_merge = ${PRIVATE_REPO_SQUASH_MERGE} + auto_init = ${PRIVATE_REPO_AUTO_INIT} + gitignore_template = ${PRIVATE_REPO_GITIGNORE_TEMPLATE} + license_template = "${PRIVATE_REPO_LICENSE_TEMPLATE}" + homepage_url = "${PRIVATE_REPO_HOMEPAGE_URL}" } EOF @@ -133,7 +237,7 @@ import_teams () { TEAM_DESCRIPTION=$(curl -s "${API_URL_PREFIX}/teams/${i}?access_token=${GITHUB_TOKEN}&per_page=100" -H "Accept: application/vnd.github.hellcat-preview+json" | jq -r .description) if [[ "${TEAM_PRIVACY}" == "closed" ]]; then - cat >> "github-teams-${TEAM_NAME_NO_SPACE}.tf" << EOF + cat >> "github-teams.tf" << EOF resource "github_team" "${TEAM_NAME_NO_SPACE}" { name = "${TEAM_NAME}" description = "${TEAM_DESCRIPTION}" @@ -141,7 +245,7 @@ resource "github_team" "${TEAM_NAME_NO_SPACE}" { } EOF elif [[ "${TEAM_PRIVACY}" == "secret" ]]; then - cat >> "github-teams-${TEAM_NAME_NO_SPACE}.tf" << EOF + cat >> "github-teams.tf" << EOF resource "github_team" "${TEAM_NAME_NO_SPACE}" { name = "${TEAM_NAME}" description = "${TEAM_DESCRIPTION}" @@ -165,7 +269,7 @@ import_team_memberships () { TEAM_ROLE=$(curl -s "${API_URL_PREFIX}/teams/${i}/memberships/${j}?access_token=${GITHUB_TOKEN}&per_page=100" -H "Accept: application/vnd.github.hellcat-preview+json" | jq -r .role) if [[ "${TEAM_ROLE}" == "maintainer" ]]; then - cat >> "github-team-memberships-${TEAM_NAME}.tf" << EOF + cat >> "github-team-memberships.tf" << EOF resource "github_team_membership" "${TEAM_NAME}-${j}" { username = "${j}" team_id = "\${github_team.${TEAM_NAME}.id}" @@ -173,7 +277,7 @@ resource "github_team_membership" "${TEAM_NAME}-${j}" { } EOF elif [[ "${TEAM_ROLE}" == "member" ]]; then - cat >> "github-team-memberships-${TEAM_NAME}.tf" << EOF + cat >> "github-team-memberships.tf" << EOF resource "github_team_membership" "${TEAM_NAME}-${j}" { username = "${j}" team_id = "\${github_team.${TEAM_NAME}.id}" @@ -212,7 +316,7 @@ get_team_repos () { PULL_PERMS=$(curl -s "${API_URL_PREFIX}/teams/${TEAM_ID}/repos/${ORG}/${i}?access_token=${GITHUB_TOKEN}" -H "Accept: application/vnd.github.v3.repository+json" | jq -r .permissions.pull ) if [[ "${ADMIN_PERMS}" == "true" ]]; then - cat >> "github-teams-${TEAM_NAME}.tf" << EOF + cat >> "github-teams.tf" << EOF resource "github_team_repository" "${TEAM_NAME}-${TERRAFORM_TEAM_REPO_NAME}" { team_id = "${TEAM_ID}" repository = "${i}" @@ -221,7 +325,7 @@ resource "github_team_repository" "${TEAM_NAME}-${TERRAFORM_TEAM_REPO_NAME}" { EOF elif [[ "${PUSH_PERMS}" == "true" ]]; then - cat >> "github-teams-${TEAM_NAME}.tf" << EOF + cat >> "github-teams.tf" << EOF resource "github_team_repository" "${TEAM_NAME}-${TERRAFORM_TEAM_REPO_NAME}" { team_id = "${TEAM_ID}" repository = "${i}" @@ -230,7 +334,7 @@ resource "github_team_repository" "${TEAM_NAME}-${TERRAFORM_TEAM_REPO_NAME}" { EOF elif [[ "${PULL_PERMS}" == "true" ]]; then - cat >> "github-teams-${TEAM_NAME}.tf" << EOF + cat >> "github-teams.tf" << EOF resource "github_team_repository" "${TEAM_NAME}-${TERRAFORM_TEAM_REPO_NAME}" { team_id = "${TEAM_ID}" repository = "${i}" @@ -260,6 +364,8 @@ import_all_team_resources () { ## DO IT YO ### import_public_repos +#import_repos_protected_branches +# to test set the vars that you need here and then call the function so you can. import_private_repos import_users import_all_team_resources