2 Commits

Author SHA1 Message Date
fc4266bf67 oops 2022-01-23 22:17:43 +01:00
936e2bdba8 WIP: Add github integration 2022-01-23 22:16:22 +01:00
9 changed files with 1089 additions and 298 deletions

643
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
[package]
name = "git-repo-manager"
version = "0.6.2"
version = "0.6.0"
edition = "2021"
authors = [
"Hannes Körber <hannes@hkoerber.de>",
@@ -37,33 +37,43 @@ path = "src/grm/main.rs"
[dependencies]
[dependencies.toml]
version = "=0.5.9"
version = "=0.5.8"
[dependencies.serde]
version = "=1.0.137"
version = "=1.0.135"
features = ["derive"]
[dependencies.git2]
version = "=0.14.3"
version = "=0.13.25"
[dependencies.shellexpand]
version = "=2.1.0"
[dependencies.clap]
version = "=3.1.17"
version = "=3.0.10"
features = ["derive", "cargo"]
[dependencies.console]
version = "=0.15.0"
[dependencies.regex]
version = "=1.5.5"
version = "=1.5.4"
[dependencies.comfy-table]
version = "=5.0.1"
version = "=5.0.0"
[dependencies.serde_yaml]
version = "=0.8.24"
version = "=0.8.23"
[dependencies.serde_json]
version = "=1.0.78"
[dependencies.isahc]
version = "=1.6.0"
features = ["json"]
[dependencies.parse_link_header]
version = "=0.3.2"
[dev-dependencies.tempdir]
version = "=0.3.7"

View File

@@ -61,11 +61,10 @@ for tier in ["dependencies", "dev-dependencies"]:
latest_version = None
for version_entry in open(info_file, "r").readlines():
version = semver.VersionInfo.parse(json.loads(version_entry)["vers"])
if current_version.prerelease == "" and version.prerelease != "":
# skip prereleases, except when we are on a prerelease already
continue
if latest_version is None or version > latest_version:
if current_version.prerelease is None and version.prerelease is not None:
# skip prereleases, except when we are on a prerelease already
print(f"{name}: Skipping prerelease version {version}")
continue
latest_version = version
if latest_version != current_version:

View File

@@ -1,14 +1,13 @@
attrs==21.4.0
gitdb==4.0.9
GitPython==3.1.27
GitPython==3.1.26
iniconfig==1.1.1
packaging==21.3
pluggy==1.0.0
py==1.11.0
pyparsing==3.0.8
pytest==7.1.2
pyparsing==3.0.7
pytest==6.2.5
PyYAML==6.0
smmap==5.0.0
toml==0.10.2
tomli==2.0.1
typing_extensions==4.2.0
typing_extensions==4.0.1

View File

@@ -7,8 +7,8 @@ use clap::{AppSettings, Parser};
author = clap::crate_authors!("\n"),
about = clap::crate_description!(),
long_version = clap::crate_version!(),
global_setting(AppSettings::DeriveDisplayOrder),
propagate_version = true,
setting = AppSettings::DeriveDisplayOrder,
setting = AppSettings::PropagateVersion,
)]
pub struct Opts {
#[clap(subcommand)]
@@ -31,20 +31,94 @@ pub struct Repos {
#[derive(Parser)]
pub enum ReposAction {
#[clap(
visible_alias = "run",
about = "Synchronize the repositories to the configured values"
)]
Sync(Sync),
#[clap(about = "Generate a repository configuration from an existing file tree")]
Find(Find),
#[clap(subcommand)]
Sync(SyncAction),
#[clap(subcommand)]
Find(FindAction),
#[clap(about = "Show status of configured repositories")]
Status(OptionalConfig),
}
#[derive(Parser)]
#[clap(about = "Sync local repositories with a configured list")]
pub enum SyncAction {
#[clap(
visible_alias = "run",
about = "Synchronize the repositories to the configured values"
)]
Config(Config),
#[clap(about = "Synchronize the repositories from a remote provider")]
Remote(Remote),
}
#[derive(Parser)]
#[clap(about = "Generate a repository configuration from existing repositories")]
pub enum FindAction {
#[clap(about = "Find local repositories")]
Local(FindLocalArgs),
#[clap(about = "Find repositories on remote provider")]
Remote(FindRemoteArgs),
}
#[derive(Parser)]
pub struct FindLocalArgs {
#[clap(help = "The path to search through")]
pub path: String,
#[clap(
arg_enum,
short,
long,
help = "Format to produce",
default_value_t = ConfigFormat::Toml,
)]
pub format: ConfigFormat,
}
#[derive(Parser)]
#[clap()]
pub struct Sync {
pub struct FindRemoteArgs {
#[clap(arg_enum, short, long, help = "Remote provider to use")]
pub provider: RemoteProvider,
#[clap(
multiple_occurrences = true,
name = "user",
long,
help = "Users to get repositories from"
)]
pub users: Vec<String>,
#[clap(
multiple_occurrences = true,
name = "group",
long,
help = "Groups to get repositories from"
)]
pub groups: Vec<String>,
#[clap(long, help = "Get repositories that belong to the requesting user")]
pub owner: bool,
#[clap(long, help = "Command to get API token")]
pub token_command: String,
#[clap(
arg_enum,
short,
long,
help = "Format to produce",
default_value_t = ConfigFormat::Toml,
)]
pub format: ConfigFormat,
#[clap(long, help = "Root of the repo tree to produce")]
pub root: String,
}
#[derive(Parser)]
#[clap()]
pub struct Config {
#[clap(
short,
long,
@@ -54,6 +128,40 @@ pub struct Sync {
pub config: String,
}
#[derive(clap::ArgEnum, Clone)]
pub enum RemoteProvider {
Github,
}
#[derive(Parser)]
#[clap()]
pub struct Remote {
#[clap(arg_enum, short, long, help = "Remote provider to use")]
pub provider: RemoteProvider,
#[clap(
multiple_occurrences = true,
name = "user",
long,
help = "Users to get repositories from"
)]
pub users: Vec<String>,
#[clap(
multiple_occurrences = true,
name = "group",
long,
help = "Groups to get repositories from"
)]
pub groups: Vec<String>,
#[clap(long, help = "Get repositories that belong to the requesting user")]
pub owner: bool,
#[clap(long, help = "Command to get API token")]
pub token_command: String,
}
#[derive(Parser)]
#[clap()]
pub struct OptionalConfig {
@@ -67,21 +175,6 @@ pub enum ConfigFormat {
Toml,
}
#[derive(Parser)]
pub struct Find {
#[clap(help = "The path to search through")]
pub path: String,
#[clap(
arg_enum,
short,
long,
help = "Format to produce",
default_value_t = ConfigFormat::Toml,
)]
pub format: ConfigFormat,
}
#[derive(Parser)]
pub struct Worktree {
#[clap(subcommand, name = "action")]

View File

@@ -5,6 +5,7 @@ mod cmd;
use grm::config;
use grm::output::*;
use grm::provider::Provider;
use grm::repo;
fn main() {
@@ -12,26 +13,88 @@ fn main() {
match opts.subcmd {
cmd::SubCommand::Repos(repos) => match repos.action {
cmd::ReposAction::Sync(sync) => {
let config = match config::read_config(&sync.config) {
Ok(config) => config,
Err(error) => {
print_error(&error);
process::exit(1);
}
};
match grm::sync_trees(config) {
Ok(success) => {
if !success {
process::exit(1)
cmd::ReposAction::Sync(sync) => match sync {
cmd::SyncAction::Config(args) => {
let config = match config::read_config(&args.config) {
Ok(config) => config,
Err(error) => {
print_error(&error);
process::exit(1);
}
};
match grm::sync_trees(config) {
Ok(success) => {
if !success {
process::exit(1)
}
}
Err(error) => {
print_error(&format!("Error syncing trees: {}", error));
process::exit(1);
}
}
Err(error) => {
print_error(&format!("Error syncing trees: {}", error));
process::exit(1);
}
cmd::SyncAction::Remote(args) => {
let users = if args.users.is_empty() {
None
} else {
Some(args.users)
};
let groups = if args.groups.is_empty() {
None
} else {
Some(args.groups)
};
let token_process = std::process::Command::new("/usr/bin/env")
.arg("sh")
.arg("-c")
.arg(args.token_command)
.output();
let token: String = match token_process {
Err(error) => {
print_error(&format!("Failed to run token-command: {}", error));
process::exit(1);
}
Ok(output) => {
let stderr = String::from_utf8(output.stderr).unwrap();
let stdout = String::from_utf8(output.stdout).unwrap();
if !output.status.success() {
if !stderr.is_empty() {
print_error(&format!("Token command failed: {}", stderr));
} else {
print_error("Token command failed.");
}
}
if !stderr.is_empty() {
print_error(&format!("Token command produced stderr: {}", stderr));
}
if stdout.is_empty() {
print_error("Token command did not produce output");
}
let token = stdout.split('\n').next().unwrap();
token.to_string()
}
};
let filter = grm::provider::Filter::new(users, groups, args.owner);
let github = grm::provider::Github::new(filter, token);
match github.get_repos() {
Ok(repos) => println!("{:?}", repos),
Err(error) => {
print_error(&format!("Error: {}", error));
process::exit(1);
}
}
}
}
},
cmd::ReposAction::Status(args) => match &args.config {
Some(config_path) => {
let config = match config::read_config(config_path) {
@@ -79,79 +142,187 @@ fn main() {
}
}
},
cmd::ReposAction::Find(find) => {
let path = Path::new(&find.path);
if !path.exists() {
print_error(&format!("Path \"{}\" does not exist", path.display()));
process::exit(1);
}
if !path.is_dir() {
print_error(&format!("Path \"{}\" is not a directory", path.display()));
process::exit(1);
}
let path = match path.canonicalize() {
Ok(path) => path,
Err(error) => {
print_error(&format!(
"Failed to canonicalize path \"{}\". This is a bug. Error message: {}",
&path.display(),
error
));
cmd::ReposAction::Find(find) => match find {
cmd::FindAction::Local(args) => {
let path = Path::new(&args.path);
if !path.exists() {
print_error(&format!("Path \"{}\" does not exist", path.display()));
process::exit(1);
}
};
let (found_repos, warnings) = match grm::find_in_tree(&path) {
Ok((repos, warnings)) => (repos, warnings),
Err(error) => {
print_error(&error);
if !path.is_dir() {
print_error(&format!("Path \"{}\" is not a directory", path.display()));
process::exit(1);
}
};
let trees = grm::config::Trees::from_vec(vec![found_repos]);
if trees.as_vec_ref().iter().all(|t| match &t.repos {
None => false,
Some(r) => r.is_empty(),
}) {
print_warning("No repositories found");
} else {
let config = trees.to_config();
match find.format {
cmd::ConfigFormat::Toml => {
let toml = match config.as_toml() {
Ok(toml) => toml,
Err(error) => {
print_error(&format!(
"Failed converting config to TOML: {}",
&error
));
process::exit(1);
}
};
print!("{}", toml);
let path = match path.canonicalize() {
Ok(path) => path,
Err(error) => {
print_error(&format!(
"Failed to canonicalize path \"{}\". This is a bug. Error message: {}",
&path.display(),
error
));
process::exit(1);
}
cmd::ConfigFormat::Yaml => {
let yaml = match config.as_yaml() {
Ok(yaml) => yaml,
Err(error) => {
print_error(&format!(
"Failed converting config to YAML: {}",
&error
));
process::exit(1);
};
let (found_repos, warnings) = match grm::find_in_tree(&path) {
Ok((repos, warnings)) => (repos, warnings),
Err(error) => {
print_error(&error);
process::exit(1);
}
};
let trees = grm::config::Trees::from_vec(vec![found_repos]);
if trees.as_vec_ref().iter().all(|t| match &t.repos {
None => false,
Some(r) => r.is_empty(),
}) {
print_warning("No repositories found");
} else {
let config = trees.to_config();
match args.format {
cmd::ConfigFormat::Toml => {
let toml = match config.as_toml() {
Ok(toml) => toml,
Err(error) => {
print_error(&format!(
"Failed converting config to TOML: {}",
&error
));
process::exit(1);
}
};
print!("{}", toml);
}
cmd::ConfigFormat::Yaml => {
let yaml = match config.as_yaml() {
Ok(yaml) => yaml,
Err(error) => {
print_error(&format!(
"Failed converting config to YAML: {}",
&error
));
process::exit(1);
}
};
print!("{}", yaml);
}
}
}
for warning in warnings {
print_warning(&warning);
}
}
cmd::FindAction::Remote(args) => {
let users = if args.users.is_empty() {
None
} else {
Some(args.users)
};
let groups = if args.groups.is_empty() {
None
} else {
Some(args.groups)
};
let token_process = std::process::Command::new("/usr/bin/env")
.arg("sh")
.arg("-c")
.arg(args.token_command)
.output();
let token: String = match token_process {
Err(error) => {
print_error(&format!("Failed to run token-command: {}", error));
process::exit(1);
}
Ok(output) => {
let stderr = String::from_utf8(output.stderr).unwrap();
let stdout = String::from_utf8(output.stdout).unwrap();
if !output.status.success() {
if !stderr.is_empty() {
print_error(&format!("Token command failed: {}", stderr));
} else {
print_error("Token command failed.");
}
}
if !stderr.is_empty() {
print_error(&format!("Token command produced stderr: {}", stderr));
}
if stdout.is_empty() {
print_error("Token command did not produce output");
}
let token = stdout.split('\n').next().unwrap();
token.to_string()
}
};
let filter = grm::provider::Filter::new(users, groups, args.owner);
let github = grm::provider::Github::new(filter, token);
match github.get_repos() {
Ok(repos) => {
let mut trees: Vec<config::Tree> = vec![];
for (namespace, repolist) in repos {
let tree = config::Tree {
root: Path::new(&args.root)
.join(namespace)
.display()
.to_string(),
repos: Some(repolist),
};
trees.push(tree);
}
let config = config::Config {
trees: config::Trees::from_vec(trees),
};
print!("{}", yaml);
match args.format {
cmd::ConfigFormat::Toml => {
let toml = match config.as_toml() {
Ok(toml) => toml,
Err(error) => {
print_error(&format!(
"Failed converting config to TOML: {}",
&error
));
process::exit(1);
}
};
print!("{}", toml);
}
cmd::ConfigFormat::Yaml => {
let yaml = match config.as_yaml() {
Ok(yaml) => yaml,
Err(error) => {
print_error(&format!(
"Failed converting config to YAML: {}",
&error
));
process::exit(1);
}
};
print!("{}", yaml);
}
}
}
Err(error) => {
print_error(&format!("Error: {}", error));
process::exit(1);
}
}
}
for warning in warnings {
print_warning(&warning);
}
}
},
},
cmd::SubCommand::Worktree(args) => {
let cwd = std::env::current_dir().unwrap_or_else(|error| {

View File

@@ -6,13 +6,14 @@ use std::process;
pub mod config;
pub mod output;
pub mod provider;
pub mod repo;
pub mod table;
use config::{Config, Tree};
use output::*;
use repo::{clone_repo, detect_remote_type, Remote, RepoConfig};
use repo::{clone_repo, detect_remote_type, Remote, RemoteType, RepoConfig};
pub use repo::{RemoteTrackingStatus, Repo, RepoErrorKind, WorktreeRemoveFailureReason};

199
src/provider/github.rs Normal file
View File

@@ -0,0 +1,199 @@
use std::collections::HashMap;
use isahc::prelude::*;
use serde::Deserialize;
use crate::{Remote, RemoteType, RepoConfig};
use super::Filter;
use super::Provider;
use super::SecretToken;
#[derive(Deserialize)]
#[serde(untagged)]
enum GithubUserProjectResponse {
Success(Vec<GithubProject>),
Failure(GithubFailureResponse),
}
#[derive(Deserialize)]
struct GithubProject {
pub name: String,
pub full_name: String,
pub clone_url: String,
pub ssh_url: String,
pub private: bool,
}
impl GithubProject {
fn into_repo_config(self) -> RepoConfig {
RepoConfig {
name: self.name,
worktree_setup: false,
remotes: Some(vec![Remote {
name: String::from("github"),
url: match self.private {
true => self.ssh_url,
false => self.clone_url,
},
remote_type: match self.private {
true => RemoteType::Ssh,
false => RemoteType::Https,
},
}]),
}
}
}
#[derive(Deserialize)]
struct GithubFailureResponse {
pub message: String,
}
pub struct Github {
filter: Filter,
secret_token: SecretToken,
}
impl Github {
fn get_repo_list_from_uri(
uri: &str,
secret_token: &SecretToken,
) -> Result<Vec<(String, GithubProject)>, String> {
let mut repos: Vec<(String, GithubProject)> = vec![];
let client = isahc::HttpClient::new().map_err(|error| error.to_string())?;
let request = isahc::Request::builder()
.uri(uri)
.header("accept", " application/vnd.github.v3+json")
.header("authorization", format!("token {}", secret_token))
.body(())
.map_err(|error| error.to_string())?;
let mut response = client.send(request).map_err(|error| error.to_string())?;
let success = response.status().is_success();
{
let response: GithubUserProjectResponse = response
.json()
.map_err(|error| format!("Failed deserializing response: {}", error))?;
if !success {
match response {
GithubUserProjectResponse::Failure(error) => return Err(error.message),
_ => return Err(String::from("Unknown response error")),
}
}
match response {
GithubUserProjectResponse::Failure(error) => {
return Err(format!(
"Received error response but no error code: {}",
error.message
))
}
GithubUserProjectResponse::Success(repo_list) => {
for repo in repo_list {
let (namespace, _name) = repo
.full_name
.rsplit_once('/')
.unwrap_or(("", &repo.full_name));
repos.push((namespace.to_string(), repo));
}
}
}
}
let headers = response.headers();
if let Some(link_header) = headers.get("link") {
let link_header = link_header.to_str().map_err(|error| error.to_string())?;
let link_header =
parse_link_header::parse(link_header).map_err(|error| error.to_string())?;
let next_page = link_header.get(&Some(String::from("next")));
if let Some(page) = next_page {
let following_repos = Github::get_repo_list_from_uri(&page.raw_uri, secret_token)?;
repos.extend(following_repos);
}
}
Ok(repos)
}
}
impl Provider for Github {
fn new(filter: Filter, secret_token: SecretToken) -> Self {
Github {
filter,
secret_token,
}
}
fn get_repos(&self) -> Result<HashMap<String, Vec<RepoConfig>>, String> {
let mut namespaces: HashMap<String, HashMap<String, RepoConfig>> = HashMap::new();
let mut register = |namespace: String, repo: GithubProject| {
let name = repo.name.clone();
let repo_config = repo.into_repo_config();
match namespaces.get_mut(&namespace) {
Some(ns) => match ns.get_mut(&name) {
Some(_entry) => {}
None => {
ns.insert(name, repo_config);
}
},
None => {
let mut ns = HashMap::new();
ns.insert(name, repo_config);
namespaces.insert(namespace, ns);
}
}
};
if let Some(users) = &self.filter.users {
for user in users {
let repos = Github::get_repo_list_from_uri(
&format!("https://api.github.com/users/{}/repos", user),
&self.secret_token,
)?;
for (namespace, repo) in repos {
register(namespace, repo);
}
}
}
if let Some(groups) = &self.filter.groups {
for group in groups {
let repos = Github::get_repo_list_from_uri(
&format!("https://api.github.com/orgs/{}/repos", group),
&self.secret_token,
)?;
for (namespace, repo) in repos {
register(namespace, repo);
}
}
}
if self.filter.owner {
let repos = Github::get_repo_list_from_uri(
"https://api.github.com/user/repos?affiliation=owner",
&self.secret_token,
)?;
for (namespace, repo) in repos {
register(namespace, repo);
}
}
let mut ret: HashMap<String, Vec<RepoConfig>> = HashMap::new();
for (namespace, repos) in namespaces {
ret.insert(namespace, repos.into_values().collect());
}
Ok(ret)
}
}

30
src/provider/mod.rs Normal file
View File

@@ -0,0 +1,30 @@
pub mod github;
pub use github::Github;
use super::RepoConfig;
use std::collections::HashMap;
pub struct Filter {
users: Option<Vec<String>>,
groups: Option<Vec<String>>,
owner: bool,
}
type SecretToken = String;
impl Filter {
pub fn new(users: Option<Vec<String>>, groups: Option<Vec<String>>, owner: bool) -> Self {
Filter {
users,
groups,
owner,
}
}
}
pub trait Provider {
fn new(filter: Filter, secret_token: SecretToken) -> Self;
fn get_repos(&self) -> Result<HashMap<String, Vec<RepoConfig>>, String>;
}