Added org github tokens to check untracked private repositories

Author: cuba0001

.github/workflows/check-untracked-repos.yml

@@ -36,8 +36,22 @@ jobs:
       - name: Install Rust stable
         uses: ./.github/actions/setup-rust
 
+      # Used to detect private repos in allowed-github-orgs
+      - name: Generate GitHub App tokens
+        uses: ./.github/actions/generate-tokens
+        id: generate-tokens
+        with:
+          app-id: ${{ secrets.SYNC_TEAM_GH_APP_ID }}
+          private-key: ${{ secrets.SYNC_TEAM_GH_APP_PRIVATE_KEY }}
+
       - name: Check untracked repositories
         id: check
+        env:
+          GITHUB_TOKEN_RUST_LANG: ${{ steps.generate-tokens.outputs.rust-lang-token }}
+          GITHUB_TOKEN_RUST_LANG_DEPRECATED: ${{ steps.generate-tokens.outputs.rust-lang-deprecated-token }}
+          GITHUB_TOKEN_RUST_LANG_NURSERY: ${{ steps.generate-tokens.outputs.rust-lang-nursery-token }}
+          GITHUB_TOKEN_RUST_ANALYZER: ${{ steps.generate-tokens.outputs.rust-analyzer-token }}
+          GITHUB_TOKEN_RUST_DEV_TOOLS: ${{ steps.generate-tokens.outputs.rust-dev-tools-token }}
         run: |
           cargo build --release
 

src/api/github.rs

@@ -58,18 +58,28 @@ impl GitHubApi {
         require_auth: bool,
         method: Method,
         url: &str,
+    ) -> Result<RequestBuilder, Error> {
+        self.prepare_with_token(require_auth, method, url, self.token.as_deref())
+    }
+
+    fn prepare_with_token(
+        &self,
+        require_auth: bool,
+        method: Method,
+        url: &str,
+        token: Option<&str>,
     ) -> Result<RequestBuilder, Error> {
         let url = if url.starts_with("https://") {
             Cow::Borrowed(url)
         } else {
             Cow::Owned(format!("{API_BASE}{url}"))
         };
-        if require_auth {
-            self.require_auth()?;
+        if require_auth && token.is_none() {
+            bail!("missing environment variable {}", TOKEN_VAR);
         }
 
         let mut req = self.http.request(method, url.as_ref());
-        if let Some(token) = &self.token {
+        if let Some(token) = token {
             req = req.header(
                 header::AUTHORIZATION,
                 HeaderValue::from_str(&format!("token {token}"))?,
@@ -122,11 +132,28 @@ impl GitHubApi {
     }
 
     pub(crate) async fn get<T>(&self, url: &str) -> Result<T, Error>
+    where
+        T: serde::de::DeserializeOwned,
+    {
+        self.get_inner(url, self.token.as_deref()).await
+    }
+
+    pub(crate) async fn get_with_token<T>(&self, url: &str, token: &str) -> Result<T, Error>
+    where
+        T: serde::de::DeserializeOwned,
+    {
+        self.get_inner(url, Some(token)).await
+    }
+
+    async fn get_inner<T>(&self, url: &str, token: Option<&str>) -> Result<T, Error>
     where
         T: serde::de::DeserializeOwned,
     {
         loop {
-            let response = self.prepare(false, Method::GET, url)?.send().await?;
+            let response = self
+                .prepare_with_token(false, Method::GET, url, token)?
+                .send()
+                .await?;
 
             let status = response.status();
             if status != StatusCode::OK {
@@ -419,3 +446,34 @@ pub struct CommitInfo {
     pub repo_name: String,
     pub created_at: DateTime<Utc>,
 }
+
+#[cfg(test)]
+mod tests {
+    use super::GitHubApi;
+    use reqwest::Method;
+    use reqwest::header::AUTHORIZATION;
+
+    #[test]
+    fn explicit_token_overrides_default_token() {
+        let github = GitHubApi {
+            http: reqwest::Client::new(),
+            token: Some("default-token".into()),
+        };
+
+        let request = github
+            .prepare_with_token(
+                false,
+                Method::GET,
+                "https://api.github.com/orgs/rust-lang/repos",
+                Some("organization-token"),
+            )
+            .unwrap()
+            .build()
+            .unwrap();
+
+        assert_eq!(
+            request.headers().get(AUTHORIZATION).unwrap(),
+            "token organization-token"
+        );
+    }
+}

src/ci.rs

@@ -270,15 +270,26 @@ async fn fetch_all_github_repos(
 
     for org in orgs_to_monitor {
         debug!("Fetching repos for org: {}", org);
+
+        let token_env_var = github_token_env_var_name(org);
+        let token = match std::env::var(&token_env_var) {
+            Ok(token) => Some(token),
+            Err(std::env::VarError::NotPresent) => None,
+            Err(std::env::VarError::NotUnicode(_)) => {
+                bail!("environment variable {token_env_var} is not valid Unicode")
+            }
+        };
+
         let mut page = 1;
 
         loop {
             let url = format!("orgs/{}/repos?per_page=100&page={}", org, page);
 
-            let repos: Vec<GitHubRepo> = github
-                .get(&url)
-                .await
-                .with_context(|| format!("Failed to fetch repos for org: {}", org))?;
+            let repos: Vec<GitHubRepo> = match token.as_deref() {
+                Some(token) => github.get_with_token(&url, token).await,
+                None => github.get(&url).await,
+            }
+            .with_context(|| format!("Failed to fetch repos for org: {}", org))?;
 
             if repos.is_empty() {
                 break;
@@ -295,6 +306,13 @@ async fn fetch_all_github_repos(
     Ok(all_repos)
 }
 
+fn github_token_env_var_name(org: &str) -> String {
+    format!(
+        "GITHUB_TOKEN_{}",
+        org.to_ascii_uppercase().replace('-', "_")
+    )
+}
+
 fn parse_tracked_repos(data: &Data) -> HashSet<(String, String)> {
     data.all_repos()
         .map(|repo| (repo.org.clone(), repo.name.clone()))
@@ -390,7 +408,10 @@ fn create_missing_repo_configs(data_dir: &Path, repos: &[UntrackedRepo]) -> anyh
 
 #[cfg(test)]
 mod tests {
-    use super::{GitHubRepo, UntrackedRepo, create_missing_repo_configs, find_untracked_repos};
+    use super::{
+        GitHubRepo, UntrackedRepo, create_missing_repo_configs, find_untracked_repos,
+        github_token_env_var_name,
+    };
     use crate::schema::Repo;
     use std::collections::HashSet;
 
@@ -440,6 +461,20 @@ mod tests {
         );
     }
 
+    #[test]
+    fn creates_github_token_environment_variable_name() {
+        for (org, env_var) in [
+            ("rust-lang", "GITHUB_TOKEN_RUST_LANG"),
+            ("rust-lang-nursery", "GITHUB_TOKEN_RUST_LANG_NURSERY"),
+            ("rust-lang-deprecated", "GITHUB_TOKEN_RUST_LANG_DEPRECATED"),
+            ("rust-dev-tools", "GITHUB_TOKEN_RUST_DEV_TOOLS"),
+            ("rust-embedded", "GITHUB_TOKEN_RUST_EMBEDDED"),
+            ("rust-analyzer", "GITHUB_TOKEN_RUST_ANALYZER"),
+        ] {
+            assert_eq!(github_token_env_var_name(org), env_var);
+        }
+    }
+
     #[test]
     fn creates_parseable_repository_config() {
         let dir = tempfile::tempdir().unwrap();