oops

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "git-repo-manager"
-version = "0.6.2"
+version = "0.6.0"
 edition = "2021"
 authors = [
   "Hannes Körber <hannes@hkoerber.de>",
@@ -37,33 +37,43 @@ path = "src/grm/main.rs"
 [dependencies]
 
 [dependencies.toml]
-version = "=0.5.9"
+version = "=0.5.8"
 
 [dependencies.serde]
-version = "=1.0.137"
+version = "=1.0.135"
 features = ["derive"]
 
 [dependencies.git2]
-version = "=0.14.3"
+version = "=0.13.25"
 
 [dependencies.shellexpand]
 version = "=2.1.0"
 
 [dependencies.clap]
-version = "=3.1.17"
+version = "=3.0.10"
 features = ["derive", "cargo"]
 
 [dependencies.console]
 version = "=0.15.0"
 
 [dependencies.regex]
-version = "=1.5.5"
+version = "=1.5.4"
 
 [dependencies.comfy-table]
-version = "=5.0.1"
+version = "=5.0.0"
 
 [dependencies.serde_yaml]
-version = "=0.8.24"
+version = "=0.8.23"
+
+[dependencies.serde_json]
+version = "=1.0.78"
+
+[dependencies.isahc]
+version = "=1.6.0"
+features = ["json"]
+
+[dependencies.parse_link_header]
+version = "=0.3.2"
 
 [dev-dependencies.tempdir]
 version = "=0.3.7"

depcheck/update-cargo-dependencies.py

@@ -61,11 +61,10 @@ for tier in ["dependencies", "dev-dependencies"]:
         latest_version = None
         for version_entry in open(info_file, "r").readlines():
             version = semver.VersionInfo.parse(json.loads(version_entry)["vers"])
-            if latest_version is None or version > latest_version:
+            if current_version.prerelease == "" and version.prerelease != "":
-                if current_version.prerelease is None and version.prerelease is not None:
                 # skip prereleases, except when we are on a prerelease already
-                    print(f"{name}: Skipping prerelease version {version}")
                 continue
+            if latest_version is None or version > latest_version:
                 latest_version = version
 
         if latest_version != current_version:

e2e_tests/requirements.txt

@@ -1,14 +1,13 @@
 attrs==21.4.0
 gitdb==4.0.9
-GitPython==3.1.27
+GitPython==3.1.26
 iniconfig==1.1.1
 packaging==21.3
 pluggy==1.0.0
 py==1.11.0
-pyparsing==3.0.8
+pyparsing==3.0.7
-pytest==7.1.2
+pytest==6.2.5
 PyYAML==6.0
 smmap==5.0.0
 toml==0.10.2
-tomli==2.0.1
+typing_extensions==4.0.1
-typing_extensions==4.2.0

src/grm/cmd.rs

@@ -7,8 +7,8 @@ use clap::{AppSettings, Parser};
     author = clap::crate_authors!("\n"),
     about = clap::crate_description!(),
     long_version = clap::crate_version!(),
-    global_setting(AppSettings::DeriveDisplayOrder),
+    setting = AppSettings::DeriveDisplayOrder,
-    propagate_version = true,
+    setting = AppSettings::PropagateVersion,
 )]
 pub struct Opts {
     #[clap(subcommand)]
@@ -31,20 +31,94 @@ pub struct Repos {
 
 #[derive(Parser)]
 pub enum ReposAction {
-    #[clap(
+    #[clap(subcommand)]
-        visible_alias = "run",
+    Sync(SyncAction),
-        about = "Synchronize the repositories to the configured values"
+    #[clap(subcommand)]
-    )]
+    Find(FindAction),
-    Sync(Sync),
-    #[clap(about = "Generate a repository configuration from an existing file tree")]
-    Find(Find),
     #[clap(about = "Show status of configured repositories")]
     Status(OptionalConfig),
 }
 
+#[derive(Parser)]
+#[clap(about = "Sync local repositories with a configured list")]
+pub enum SyncAction {
+    #[clap(
+        visible_alias = "run",
+        about = "Synchronize the repositories to the configured values"
+    )]
+    Config(Config),
+    #[clap(about = "Synchronize the repositories from a remote provider")]
+    Remote(Remote),
+}
+
+#[derive(Parser)]
+#[clap(about = "Generate a repository configuration from existing repositories")]
+pub enum FindAction {
+    #[clap(about = "Find local repositories")]
+    Local(FindLocalArgs),
+    #[clap(about = "Find repositories on remote provider")]
+    Remote(FindRemoteArgs),
+}
+
+#[derive(Parser)]
+pub struct FindLocalArgs {
+    #[clap(help = "The path to search through")]
+    pub path: String,
+
+    #[clap(
+        arg_enum,
+        short,
+        long,
+        help = "Format to produce",
+        default_value_t = ConfigFormat::Toml,
+    )]
+    pub format: ConfigFormat,
+}
+
 #[derive(Parser)]
 #[clap()]
-pub struct Sync {
+pub struct FindRemoteArgs {
+    #[clap(arg_enum, short, long, help = "Remote provider to use")]
+    pub provider: RemoteProvider,
+
+    #[clap(
+        multiple_occurrences = true,
+        name = "user",
+        long,
+        help = "Users to get repositories from"
+    )]
+    pub users: Vec<String>,
+
+    #[clap(
+        multiple_occurrences = true,
+        name = "group",
+        long,
+        help = "Groups to get repositories from"
+    )]
+    pub groups: Vec<String>,
+
+    #[clap(long, help = "Get repositories that belong to the requesting user")]
+    pub owner: bool,
+
+    #[clap(long, help = "Command to get API token")]
+    pub token_command: String,
+
+    #[clap(
+        arg_enum,
+        short,
+        long,
+        help = "Format to produce",
+        default_value_t = ConfigFormat::Toml,
+    )]
+    pub format: ConfigFormat,
+
+    #[clap(long, help = "Root of the repo tree to produce")]
+    pub root: String,
+}
+
+#[derive(Parser)]
+#[clap()]
+pub struct Config {
     #[clap(
         short,
         long,
@@ -54,6 +128,40 @@ pub struct Sync {
     pub config: String,
 }
 
+#[derive(clap::ArgEnum, Clone)]
+pub enum RemoteProvider {
+    Github,
+}
+
+#[derive(Parser)]
+#[clap()]
+pub struct Remote {
+    #[clap(arg_enum, short, long, help = "Remote provider to use")]
+    pub provider: RemoteProvider,
+
+    #[clap(
+        multiple_occurrences = true,
+        name = "user",
+        long,
+        help = "Users to get repositories from"
+    )]
+    pub users: Vec<String>,
+
+    #[clap(
+        multiple_occurrences = true,
+        name = "group",
+        long,
+        help = "Groups to get repositories from"
+    )]
+    pub groups: Vec<String>,
+
+    #[clap(long, help = "Get repositories that belong to the requesting user")]
+    pub owner: bool,
+
+    #[clap(long, help = "Command to get API token")]
+    pub token_command: String,
+}
+
 #[derive(Parser)]
 #[clap()]
 pub struct OptionalConfig {
@@ -67,21 +175,6 @@ pub enum ConfigFormat {
     Toml,
 }
 
-#[derive(Parser)]
-pub struct Find {
-    #[clap(help = "The path to search through")]
-    pub path: String,
-
-    #[clap(
-        arg_enum,
-        short,
-        long,
-        help = "Format to produce",
-        default_value_t = ConfigFormat::Toml,
-    )]
-    pub format: ConfigFormat,
-}
-
 #[derive(Parser)]
 pub struct Worktree {
     #[clap(subcommand, name = "action")]

src/grm/main.rs

@@ -5,6 +5,7 @@ mod cmd;
 
 use grm::config;
 use grm::output::*;
+use grm::provider::Provider;
 use grm::repo;
 
 fn main() {
@@ -12,8 +13,9 @@ fn main() {
 
     match opts.subcmd {
         cmd::SubCommand::Repos(repos) => match repos.action {
-            cmd::ReposAction::Sync(sync) => {
+            cmd::ReposAction::Sync(sync) => match sync {
-                let config = match config::read_config(&sync.config) {
+                cmd::SyncAction::Config(args) => {
+                    let config = match config::read_config(&args.config) {
                         Ok(config) => config,
                         Err(error) => {
                             print_error(&error);
@@ -32,6 +34,67 @@ fn main() {
                         }
                     }
                 }
+                cmd::SyncAction::Remote(args) => {
+                    let users = if args.users.is_empty() {
+                        None
+                    } else {
+                        Some(args.users)
+                    };
+
+                    let groups = if args.groups.is_empty() {
+                        None
+                    } else {
+                        Some(args.groups)
+                    };
+
+                    let token_process = std::process::Command::new("/usr/bin/env")
+                        .arg("sh")
+                        .arg("-c")
+                        .arg(args.token_command)
+                        .output();
+
+                    let token: String = match token_process {
+                        Err(error) => {
+                            print_error(&format!("Failed to run token-command: {}", error));
+                            process::exit(1);
+                        }
+                        Ok(output) => {
+                            let stderr = String::from_utf8(output.stderr).unwrap();
+                            let stdout = String::from_utf8(output.stdout).unwrap();
+
+                            if !output.status.success() {
+                                if !stderr.is_empty() {
+                                    print_error(&format!("Token command failed: {}", stderr));
+                                } else {
+                                    print_error("Token command failed.");
+                                }
+                            }
+                            if !stderr.is_empty() {
+                                print_error(&format!("Token command produced stderr: {}", stderr));
+                            }
+
+                            if stdout.is_empty() {
+                                print_error("Token command did not produce output");
+                            }
+
+                            let token = stdout.split('\n').next().unwrap();
+
+                            token.to_string()
+                        }
+                    };
+
+                    let filter = grm::provider::Filter::new(users, groups, args.owner);
+                    let github = grm::provider::Github::new(filter, token);
+
+                    match github.get_repos() {
+                        Ok(repos) => println!("{:?}", repos),
+                        Err(error) => {
+                            print_error(&format!("Error: {}", error));
+                            process::exit(1);
+                        }
+                    }
+                }
+            },
             cmd::ReposAction::Status(args) => match &args.config {
                 Some(config_path) => {
                     let config = match config::read_config(config_path) {
@@ -79,8 +142,9 @@ fn main() {
                     }
                 }
             },
-            cmd::ReposAction::Find(find) => {
+            cmd::ReposAction::Find(find) => match find {
-                let path = Path::new(&find.path);
+                cmd::FindAction::Local(args) => {
+                    let path = Path::new(&args.path);
                     if !path.exists() {
                         print_error(&format!("Path \"{}\" does not exist", path.display()));
                         process::exit(1);
@@ -119,7 +183,7 @@ fn main() {
                     } else {
                         let config = trees.to_config();
 
-                    match find.format {
+                        match args.format {
                             cmd::ConfigFormat::Toml => {
                                 let toml = match config.as_toml() {
                                     Ok(toml) => toml,
@@ -152,6 +216,113 @@ fn main() {
                         print_warning(&warning);
                     }
                 }
+                cmd::FindAction::Remote(args) => {
+                    let users = if args.users.is_empty() {
+                        None
+                    } else {
+                        Some(args.users)
+                    };
+
+                    let groups = if args.groups.is_empty() {
+                        None
+                    } else {
+                        Some(args.groups)
+                    };
+
+                    let token_process = std::process::Command::new("/usr/bin/env")
+                        .arg("sh")
+                        .arg("-c")
+                        .arg(args.token_command)
+                        .output();
+
+                    let token: String = match token_process {
+                        Err(error) => {
+                            print_error(&format!("Failed to run token-command: {}", error));
+                            process::exit(1);
+                        }
+                        Ok(output) => {
+                            let stderr = String::from_utf8(output.stderr).unwrap();
+                            let stdout = String::from_utf8(output.stdout).unwrap();
+
+                            if !output.status.success() {
+                                if !stderr.is_empty() {
+                                    print_error(&format!("Token command failed: {}", stderr));
+                                } else {
+                                    print_error("Token command failed.");
+                                }
+                            }
+                            if !stderr.is_empty() {
+                                print_error(&format!("Token command produced stderr: {}", stderr));
+                            }
+
+                            if stdout.is_empty() {
+                                print_error("Token command did not produce output");
+                            }
+
+                            let token = stdout.split('\n').next().unwrap();
+
+                            token.to_string()
+                        }
+                    };
+
+                    let filter = grm::provider::Filter::new(users, groups, args.owner);
+                    let github = grm::provider::Github::new(filter, token);
+
+                    match github.get_repos() {
+                        Ok(repos) => {
+                            let mut trees: Vec<config::Tree> = vec![];
+
+                            for (namespace, repolist) in repos {
+                                let tree = config::Tree {
+                                    root: Path::new(&args.root)
+                                        .join(namespace)
+                                        .display()
+                                        .to_string(),
+                                    repos: Some(repolist),
+                                };
+                                trees.push(tree);
+                            }
+
+                            let config = config::Config {
+                                trees: config::Trees::from_vec(trees),
+                            };
+
+                            match args.format {
+                                cmd::ConfigFormat::Toml => {
+                                    let toml = match config.as_toml() {
+                                        Ok(toml) => toml,
+                                        Err(error) => {
+                                            print_error(&format!(
+                                                "Failed converting config to TOML: {}",
+                                                &error
+                                            ));
+                                            process::exit(1);
+                                        }
+                                    };
+                                    print!("{}", toml);
+                                }
+                                cmd::ConfigFormat::Yaml => {
+                                    let yaml = match config.as_yaml() {
+                                        Ok(yaml) => yaml,
+                                        Err(error) => {
+                                            print_error(&format!(
+                                                "Failed converting config to YAML: {}",
+                                                &error
+                                            ));
+                                            process::exit(1);
+                                        }
+                                    };
+                                    print!("{}", yaml);
+                                }
+                            }
+                        }
+                        Err(error) => {
+                            print_error(&format!("Error: {}", error));
+                            process::exit(1);
+                        }
+                    }
+                }
+            },
         },
         cmd::SubCommand::Worktree(args) => {
             let cwd = std::env::current_dir().unwrap_or_else(|error| {

src/lib.rs

@@ -6,13 +6,14 @@ use std::process;
 
 pub mod config;
 pub mod output;
+pub mod provider;
 pub mod repo;
 pub mod table;
 
 use config::{Config, Tree};
 use output::*;
 
-use repo::{clone_repo, detect_remote_type, Remote, RepoConfig};
+use repo::{clone_repo, detect_remote_type, Remote, RemoteType, RepoConfig};
 
 pub use repo::{RemoteTrackingStatus, Repo, RepoErrorKind, WorktreeRemoveFailureReason};
 

src/provider/github.rs

@@ -0,0 +1,199 @@
+use std::collections::HashMap;
+
+use isahc::prelude::*;
+use serde::Deserialize;
+
+use crate::{Remote, RemoteType, RepoConfig};
+
+use super::Filter;
+use super::Provider;
+use super::SecretToken;
+
+#[derive(Deserialize)]
+#[serde(untagged)]
+enum GithubUserProjectResponse {
+    Success(Vec<GithubProject>),
+    Failure(GithubFailureResponse),
+}
+
+#[derive(Deserialize)]
+struct GithubProject {
+    pub name: String,
+    pub full_name: String,
+    pub clone_url: String,
+    pub ssh_url: String,
+    pub private: bool,
+}
+
+impl GithubProject {
+    fn into_repo_config(self) -> RepoConfig {
+        RepoConfig {
+            name: self.name,
+            worktree_setup: false,
+            remotes: Some(vec![Remote {
+                name: String::from("github"),
+                url: match self.private {
+                    true => self.ssh_url,
+                    false => self.clone_url,
+                },
+                remote_type: match self.private {
+                    true => RemoteType::Ssh,
+                    false => RemoteType::Https,
+                },
+            }]),
+        }
+    }
+}
+
+#[derive(Deserialize)]
+struct GithubFailureResponse {
+    pub message: String,
+}
+
+pub struct Github {
+    filter: Filter,
+    secret_token: SecretToken,
+}
+
+impl Github {
+    fn get_repo_list_from_uri(
+        uri: &str,
+        secret_token: &SecretToken,
+    ) -> Result<Vec<(String, GithubProject)>, String> {
+        let mut repos: Vec<(String, GithubProject)> = vec![];
+
+        let client = isahc::HttpClient::new().map_err(|error| error.to_string())?;
+
+        let request = isahc::Request::builder()
+            .uri(uri)
+            .header("accept", " application/vnd.github.v3+json")
+            .header("authorization", format!("token {}", secret_token))
+            .body(())
+            .map_err(|error| error.to_string())?;
+
+        let mut response = client.send(request).map_err(|error| error.to_string())?;
+
+        let success = response.status().is_success();
+
+        {
+            let response: GithubUserProjectResponse = response
+                .json()
+                .map_err(|error| format!("Failed deserializing response: {}", error))?;
+
+            if !success {
+                match response {
+                    GithubUserProjectResponse::Failure(error) => return Err(error.message),
+                    _ => return Err(String::from("Unknown response error")),
+                }
+            }
+
+            match response {
+                GithubUserProjectResponse::Failure(error) => {
+                    return Err(format!(
+                        "Received error response but no error code: {}",
+                        error.message
+                    ))
+                }
+                GithubUserProjectResponse::Success(repo_list) => {
+                    for repo in repo_list {
+                        let (namespace, _name) = repo
+                            .full_name
+                            .rsplit_once('/')
+                            .unwrap_or(("", &repo.full_name));
+                        repos.push((namespace.to_string(), repo));
+                    }
+                }
+            }
+        }
+
+        let headers = response.headers();
+
+        if let Some(link_header) = headers.get("link") {
+            let link_header = link_header.to_str().map_err(|error| error.to_string())?;
+
+            let link_header =
+                parse_link_header::parse(link_header).map_err(|error| error.to_string())?;
+
+            let next_page = link_header.get(&Some(String::from("next")));
+
+            if let Some(page) = next_page {
+                let following_repos = Github::get_repo_list_from_uri(&page.raw_uri, secret_token)?;
+                repos.extend(following_repos);
+            }
+        }
+
+        Ok(repos)
+    }
+}
+
+impl Provider for Github {
+    fn new(filter: Filter, secret_token: SecretToken) -> Self {
+        Github {
+            filter,
+            secret_token,
+        }
+    }
+
+    fn get_repos(&self) -> Result<HashMap<String, Vec<RepoConfig>>, String> {
+        let mut namespaces: HashMap<String, HashMap<String, RepoConfig>> = HashMap::new();
+
+        let mut register = |namespace: String, repo: GithubProject| {
+            let name = repo.name.clone();
+            let repo_config = repo.into_repo_config();
+            match namespaces.get_mut(&namespace) {
+                Some(ns) => match ns.get_mut(&name) {
+                    Some(_entry) => {}
+                    None => {
+                        ns.insert(name, repo_config);
+                    }
+                },
+                None => {
+                    let mut ns = HashMap::new();
+                    ns.insert(name, repo_config);
+                    namespaces.insert(namespace, ns);
+                }
+            }
+        };
+
+        if let Some(users) = &self.filter.users {
+            for user in users {
+                let repos = Github::get_repo_list_from_uri(
+                    &format!("https://api.github.com/users/{}/repos", user),
+                    &self.secret_token,
+                )?;
+                for (namespace, repo) in repos {
+                    register(namespace, repo);
+                }
+            }
+        }
+
+        if let Some(groups) = &self.filter.groups {
+            for group in groups {
+                let repos = Github::get_repo_list_from_uri(
+                    &format!("https://api.github.com/orgs/{}/repos", group),
+                    &self.secret_token,
+                )?;
+                for (namespace, repo) in repos {
+                    register(namespace, repo);
+                }
+            }
+        }
+
+        if self.filter.owner {
+            let repos = Github::get_repo_list_from_uri(
+                "https://api.github.com/user/repos?affiliation=owner",
+                &self.secret_token,
+            )?;
+            for (namespace, repo) in repos {
+                register(namespace, repo);
+            }
+        }
+
+        let mut ret: HashMap<String, Vec<RepoConfig>> = HashMap::new();
+        for (namespace, repos) in namespaces {
+            ret.insert(namespace, repos.into_values().collect());
+        }
+
+        Ok(ret)
+    }
+}

src/provider/mod.rs

@@ -0,0 +1,30 @@
+pub mod github;
+
+pub use github::Github;
+
+use super::RepoConfig;
+
+use std::collections::HashMap;
+
+pub struct Filter {
+    users: Option<Vec<String>>,
+    groups: Option<Vec<String>>,
+    owner: bool,
+}
+
+type SecretToken = String;
+
+impl Filter {
+    pub fn new(users: Option<Vec<String>>, groups: Option<Vec<String>>, owner: bool) -> Self {
+        Filter {
+            users,
+            groups,
+            owner,
+        }
+    }
+}
+
+pub trait Provider {
+    fn new(filter: Filter, secret_token: SecretToken) -> Self;
+    fn get_repos(&self) -> Result<HashMap<String, Vec<RepoConfig>>, String>;
+}