From 14c95f270460927e24fe16c97c5450edab432e55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Mon, 24 Jan 2022 17:03:37 +0100 Subject: [PATCH 01/36] Fix worktree creation handling --- src/lib.rs | 11 ++++++----- src/repo.rs | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 3bc321b..0facb54 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -474,15 +474,16 @@ pub fn add_worktree( let config = repo::read_worktree_root_config(directory)?; - let path = match subdirectory { - Some(dir) => dir.join(name), - None => Path::new(name).to_path_buf(), - }; - if repo.find_worktree(&path).is_ok() { + if repo.find_worktree(&name).is_ok() { return Err(format!("Worktree {} already exists", &name)); } + let path = match subdirectory { + Some(dir) => directory.join(dir).join(name), + None => directory.join(Path::new(name).to_path_buf()), + }; + let mut remote_branch_exists = false; let default_checkout = || repo.default_branch()?.to_commit(); diff --git a/src/repo.rs b/src/repo.rs index a680014..33a0899 100644 --- a/src/repo.rs +++ b/src/repo.rs @@ -672,9 +672,9 @@ impl Repo { self.0.config().map_err(convert_libgit2_error) } - pub fn find_worktree(&self, path: &Path) -> Result<(), String> { + pub fn find_worktree(&self, name: &str) -> Result<(), String> { self.0 - .find_worktree(path.to_str().expect("Worktree path is not valid utf-8")) + .find_worktree(&name) .map_err(convert_libgit2_error)?; Ok(()) } From 1cf4e85014bc1768f43de4c6ba92291a0dc32ac4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Tue, 10 May 2022 18:23:56 +0200 Subject: [PATCH 02/36] Fix non-worktree directory detection for status --- e2e_tests/test_worktree_status.py | 37 ++++++++++++++++++++++++++++++- src/table.rs | 26 ++++++---------------- 2 files changed, 43 insertions(+), 20 deletions(-) diff --git a/e2e_tests/test_worktree_status.py b/e2e_tests/test_worktree_status.py index d55fa66..68b86e7 100644 --- a/e2e_tests/test_worktree_status.py +++ b/e2e_tests/test_worktree_status.py @@ -1,10 +1,18 @@ #!/usr/bin/env python3 +import re + from helpers import * +import pytest -def test_worktree_status(): + +@pytest.mark.parametrize("has_config", [True, False]) +def test_worktree_status(has_config): with TempGitRepositoryWorktree() as (base_dir, _commit): + if has_config: + with open(os.path.join(base_dir, "grm.toml"), "w") as f: + f.write("") cmd = grm(["wt", "add", "test"], cwd=base_dir) assert cmd.returncode == 0 @@ -40,3 +48,30 @@ def test_worktree_status_non_git(): assert cmd.returncode != 0 assert len(cmd.stdout) == 0 assert len(cmd.stderr) != 0 + + +def test_worktree_status_warn_with_non_worktree_dir(): + with TempGitRepositoryWorktree() as (base_dir, _commit): + cmd = grm(["wt", "add", "test"], cwd=base_dir) + assert cmd.returncode == 0 + + shell( + f""" + cd {base_dir} + mkdir not_a_worktree + """ + ) + + cmd = grm(["wt", "status"], cwd=base_dir) + + assert cmd.returncode == 0 + assert len(cmd.stdout) != 0 + assert len(cmd.stderr) != 0 + assert ( + re.match( + ".*error.*not_a_worktree.*not a valid worktree directory", + cmd.stderr, + re.IGNORECASE, + ) + is not None + ) diff --git a/src/table.rs b/src/table.rs index 3ea818d..b4a9814 100644 --- a/src/table.rs +++ b/src/table.rs @@ -1,3 +1,5 @@ +use crate::Repo; + use comfy_table::{Cell, Table}; use std::path::Path; @@ -130,25 +132,11 @@ pub fn get_worktree_status_table( )); } } - for entry in std::fs::read_dir(&directory).map_err(|error| error.to_string())? { - let dirname = crate::path_as_string( - entry - .map_err(|error| error.to_string())? - .path() - .strip_prefix(&directory) - // this unwrap is safe, as we can be sure that each subentry of - // &directory also has the prefix &dir - .unwrap(), - ); - if dirname == crate::GIT_MAIN_WORKTREE_DIRECTORY { - continue; - } - if !&worktrees.iter().any(|worktree| worktree.name() == dirname) { - errors.push(format!( - "Found {}, which is not a valid worktree directory!", - &dirname - )); - } + for worktree in Repo::find_unmanaged_worktrees(&repo, &directory).unwrap() { + errors.push(format!( + "Found {}, which is not a valid worktree directory!", + &worktree + )); } Ok((table, errors)) } From e940ab69fb46bcdc13dc601b6ad856fc6cb688d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Tue, 10 May 2022 18:25:45 +0200 Subject: [PATCH 03/36] Accept clippy suggestions --- src/lib.rs | 4 ++-- src/repo.rs | 2 +- src/table.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 0facb54..ccb84c9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -475,13 +475,13 @@ pub fn add_worktree( let config = repo::read_worktree_root_config(directory)?; - if repo.find_worktree(&name).is_ok() { + if repo.find_worktree(name).is_ok() { return Err(format!("Worktree {} already exists", &name)); } let path = match subdirectory { Some(dir) => directory.join(dir).join(name), - None => directory.join(Path::new(name).to_path_buf()), + None => directory.join(Path::new(name)), }; let mut remote_branch_exists = false; diff --git a/src/repo.rs b/src/repo.rs index 33a0899..b56a0f1 100644 --- a/src/repo.rs +++ b/src/repo.rs @@ -674,7 +674,7 @@ impl Repo { pub fn find_worktree(&self, name: &str) -> Result<(), String> { self.0 - .find_worktree(&name) + .find_worktree(name) .map_err(convert_libgit2_error)?; Ok(()) } diff --git a/src/table.rs b/src/table.rs index b4a9814..fcea35f 100644 --- a/src/table.rs +++ b/src/table.rs @@ -132,7 +132,7 @@ pub fn get_worktree_status_table( )); } } - for worktree in Repo::find_unmanaged_worktrees(&repo, &directory).unwrap() { + for worktree in Repo::find_unmanaged_worktrees(repo, directory).unwrap() { errors.push(format!( "Found {}, which is not a valid worktree directory!", &worktree From c3c1c989137ff23a7379a5826886ccb81d39c9c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Tue, 10 May 2022 18:26:06 +0200 Subject: [PATCH 04/36] Run cargo fmt --- src/lib.rs | 1 - src/repo.rs | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index ccb84c9..d598aac 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -474,7 +474,6 @@ pub fn add_worktree( let config = repo::read_worktree_root_config(directory)?; - if repo.find_worktree(name).is_ok() { return Err(format!("Worktree {} already exists", &name)); } diff --git a/src/repo.rs b/src/repo.rs index b56a0f1..a632cb4 100644 --- a/src/repo.rs +++ b/src/repo.rs @@ -673,9 +673,7 @@ impl Repo { } pub fn find_worktree(&self, name: &str) -> Result<(), String> { - self.0 - .find_worktree(name) - .map_err(convert_libgit2_error)?; + self.0.find_worktree(name).map_err(convert_libgit2_error)?; Ok(()) } From 908094f48bf7258716f4e565c25c0d6bf2902cc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Thu, 26 May 2022 17:24:35 +0200 Subject: [PATCH 05/36] dependencies: Update git2 to 0.14.4 --- Cargo.lock | 12 ++++++------ Cargo.toml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3180fb0..7c4ad14 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -212,9 +212,9 @@ dependencies = [ [[package]] name = "git2" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e77a14ffc6ba4ad5188d6cf428894c4fcfda725326b37558f35bb677e712cec" +checksum = "d0155506aab710a86160ddb504a480d2964d7ab5b9e62419be69e0032bc5931c" dependencies = [ "bitflags", "libc", @@ -293,15 +293,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b" +checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" [[package]] name = "libgit2-sys" -version = "0.13.3+1.4.2" +version = "0.13.4+1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c24d36c3ac9b9996a2418d6bf428cc0bc5d1a814a84303fc60986088c5ed60de" +checksum = "d0fa6563431ede25f5cc7f6d803c6afbc1c5d3ad3d4925d12c882bf2b526f5d1" dependencies = [ "cc", "libc", diff --git a/Cargo.toml b/Cargo.toml index 17168ab..4458681 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,7 +44,7 @@ version = "=1.0.137" features = ["derive"] [dependencies.git2] -version = "=0.14.3" +version = "=0.14.4" [dependencies.shellexpand] version = "=2.1.0" From 3557dd26869183226687798a463bb4c6171ea56d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Thu, 26 May 2022 17:24:35 +0200 Subject: [PATCH 06/36] dependencies: Update clap to 3.1.18 --- Cargo.lock | 36 ++++++++++++++++++------------------ Cargo.toml | 2 +- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7c4ad14..480b7cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -51,9 +51,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "3.1.17" +version = "3.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47582c09be7c8b32c0ab3a6181825ababb713fde6fff20fc573a3870dd45c6a0" +checksum = "d2dbdf4bdacb33466e854ce889eee8dfd5729abf7ccd7664d0a2d60cd384440b" dependencies = [ "atty", "bitflags", @@ -68,9 +68,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "3.1.7" +version = "3.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3aab4734e083b809aaf5794e14e756d1c798d2c69c7f7de7a09a2f5214993c1" +checksum = "25320346e922cffe59c0bbc5410c8d8784509efb321488971081313cb1e1a33c" dependencies = [ "heck 0.4.0", "proc-macro-error", @@ -433,9 +433,9 @@ dependencies = [ [[package]] name = "os_str_bytes" -version = "6.0.0" +version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64" +checksum = "21326818e99cfe6ce1e524c2a805c189a99b5ae555a35d19f9a284b427d86afa" [[package]] name = "parking_lot" @@ -498,11 +498,11 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.38" +version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9027b48e9d4c9175fa2218adf3557f91c1137021739951d4932f5f8268ac48aa" +checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f" dependencies = [ - "unicode-xid", + "unicode-ident", ] [[package]] @@ -719,13 +719,13 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.92" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ff7c592601f11445996a06f8ad0c27f094a58857c2f89e97974ab9235b92c52" +checksum = "fbaf6116ab8924f39d52792136fb74fd60a80194cf1b1c6ffa6453eef1c3f942" dependencies = [ "proc-macro2", "quote", - "unicode-xid", + "unicode-ident", ] [[package]] @@ -813,6 +813,12 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" +[[package]] +name = "unicode-ident" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee" + [[package]] name = "unicode-normalization" version = "0.1.19" @@ -834,12 +840,6 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" -[[package]] -name = "unicode-xid" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04" - [[package]] name = "url" version = "2.2.2" diff --git a/Cargo.toml b/Cargo.toml index 4458681..f89a8b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,7 +50,7 @@ version = "=0.14.4" version = "=2.1.0" [dependencies.clap] -version = "=3.1.17" +version = "=3.1.18" features = ["derive", "cargo"] [dependencies.console] From be085e9b0f585b7e342136eec4ada798ca09e35c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Thu, 26 May 2022 17:24:35 +0200 Subject: [PATCH 07/36] dependencies: Update regex to 1.5.6 --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 480b7cd..fbc881d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -573,9 +573,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.5.5" +version = "1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" +checksum = "d83f127d94bdbcda4c8cc2e50f6f84f4b611f69c902699ca385a39c3a75f9ff1" dependencies = [ "aho-corasick", "memchr", @@ -584,9 +584,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.25" +version = "0.6.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64" [[package]] name = "remove_dir_all" diff --git a/Cargo.toml b/Cargo.toml index f89a8b8..974bbe0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,7 +57,7 @@ features = ["derive", "cargo"] version = "=0.15.0" [dependencies.regex] -version = "=1.5.5" +version = "=1.5.6" [dependencies.comfy-table] version = "=5.0.1" From f01568a695789d8a3d57739c4a7046f69a681a33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Thu, 26 May 2022 17:24:36 +0200 Subject: [PATCH 08/36] Cargo.lock: Updating mio v0.8.2 -> v0.8.3 --- Cargo.lock | 30 +++++------------------------- 1 file changed, 5 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fbc881d..3504790 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -376,34 +376,14 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "mio" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9" +checksum = "713d550d9b44d89174e066b7a6217ae06234c10cb47819a88290d2b353c31799" dependencies = [ "libc", "log", - "miow", - "ntapi", "wasi 0.11.0+wasi-snapshot-preview1", - "winapi", -] - -[[package]] -name = "miow" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" -dependencies = [ - "winapi", -] - -[[package]] -name = "ntapi" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f" -dependencies = [ - "winapi", + "windows-sys", ] [[package]] @@ -658,9 +638,9 @@ dependencies = [ [[package]] name = "signal-hook" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "647c97df271007dcea485bb74ffdb57f2e683f1306c854f468a0c244badabf2d" +checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d" dependencies = [ "libc", "signal-hook-registry", From dd65f2cd813de7d651518a08d3e608463fca71aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Thu, 26 May 2022 17:24:38 +0200 Subject: [PATCH 09/36] Cargo.lock: Updating once_cell v1.10.0 -> v1.12.0 --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3504790..0b1b80f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -388,9 +388,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.10.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" +checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225" [[package]] name = "openssl-probe" From 7ad51ccb474398dab029be44b375c48535ee05f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Thu, 26 May 2022 17:24:39 +0200 Subject: [PATCH 10/36] Cargo.lock: Updating ryu v1.0.9 -> v1.0.10 --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0b1b80f..b45a897 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -585,9 +585,9 @@ checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" [[package]] name = "ryu" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" +checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" [[package]] name = "scopeguard" From 38c66cad623be3f14708edaf5728bee7a50b3887 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Sun, 23 Jan 2022 16:33:10 +0100 Subject: [PATCH 11/36] Add git forge integration --- Cargo.lock | 378 +++++++ Cargo.toml | 13 + Justfile | 53 +- docs/src/SUMMARY.md | 1 + docs/src/forge_integration.md | 205 ++++ docs/src/repos.md | 6 +- e2e_tests/.gitignore | 2 - e2e_tests/conftest.py | 8 + e2e_tests/docker-compose.yml | 34 + e2e_tests/docker-rest/Dockerfile | 19 + e2e_tests/docker-rest/flask/app.py | 7 + e2e_tests/docker-rest/flask/github.py | 103 ++ .../flask/github_api_page_1.json.j2 | 228 +++++ .../flask/github_api_page_2.json.j2 | 115 +++ .../flask/github_api_page_3.json.j2 | 115 +++ .../flask/github_api_page_4.json.j2 | 115 +++ .../docker-rest/flask/github_api_user.json | 46 + e2e_tests/docker-rest/flask/gitlab.py | 106 ++ .../docker-rest/flask/gitlab_api_page_1.json | 236 +++++ .../docker-rest/flask/gitlab_api_page_2.json | 119 +++ .../docker-rest/flask/gitlab_api_page_3.json | 119 +++ .../docker-rest/flask/gitlab_api_page_4.json | 119 +++ .../docker-rest/flask/gitlab_api_user.json | 42 + e2e_tests/docker/Dockerfile | 14 + e2e_tests/helpers.py | 6 +- e2e_tests/requirements.txt | 14 - e2e_tests/test_repos_find.py | 21 +- e2e_tests/test_repos_find_remote.py | 950 ++++++++++++++++++ e2e_tests/test_repos_sync.py | 51 +- e2e_tests/update_requirementstxt.sh | 20 - src/config.rs | 128 ++- src/grm/cmd.rs | 205 +++- src/grm/main.rs | 391 ++++++- src/lib.rs | 88 +- src/provider/github.rs | 144 +++ src/provider/gitlab.rs | 165 +++ src/provider/mod.rs | 340 +++++++ src/table.rs | 2 +- 38 files changed, 4522 insertions(+), 206 deletions(-) create mode 100644 docs/src/forge_integration.md delete mode 100644 e2e_tests/.gitignore create mode 100644 e2e_tests/conftest.py create mode 100644 e2e_tests/docker-compose.yml create mode 100644 e2e_tests/docker-rest/Dockerfile create mode 100644 e2e_tests/docker-rest/flask/app.py create mode 100644 e2e_tests/docker-rest/flask/github.py create mode 100644 e2e_tests/docker-rest/flask/github_api_page_1.json.j2 create mode 100644 e2e_tests/docker-rest/flask/github_api_page_2.json.j2 create mode 100644 e2e_tests/docker-rest/flask/github_api_page_3.json.j2 create mode 100644 e2e_tests/docker-rest/flask/github_api_page_4.json.j2 create mode 100644 e2e_tests/docker-rest/flask/github_api_user.json create mode 100644 e2e_tests/docker-rest/flask/gitlab.py create mode 100644 e2e_tests/docker-rest/flask/gitlab_api_page_1.json create mode 100644 e2e_tests/docker-rest/flask/gitlab_api_page_2.json create mode 100644 e2e_tests/docker-rest/flask/gitlab_api_page_3.json create mode 100644 e2e_tests/docker-rest/flask/gitlab_api_page_4.json create mode 100644 e2e_tests/docker-rest/flask/gitlab_api_user.json create mode 100644 e2e_tests/docker/Dockerfile delete mode 100644 e2e_tests/requirements.txt create mode 100644 e2e_tests/test_repos_find_remote.py delete mode 100755 e2e_tests/update_requirementstxt.sh create mode 100644 src/provider/github.rs create mode 100644 src/provider/gitlab.rs create mode 100644 src/provider/mod.rs diff --git a/Cargo.lock b/Cargo.lock index b45a897..c356d4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,17 @@ dependencies = [ "memchr", ] +[[package]] +name = "async-channel" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2114d64672151c0c5eaa5e131ec84a74f06e1e559830dabba01ca30605d66319" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + [[package]] name = "atty" version = "0.2.14" @@ -34,6 +45,24 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bytes" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" + +[[package]] +name = "cache-padded" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1db59621ec70f09c5e9b597b220c7a2b43611f4710dc03ceb8748637775692c" + +[[package]] +name = "castaway" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2698f953def977c68f935bb0dfa959375ad4638570e969e2f1e9f433cbf1af6" + [[package]] name = "cc" version = "1.0.73" @@ -100,6 +129,15 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "concurrent-queue" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ed07550be01594c6026cff2a1d7fe9c8f683caa798e12b68694ac9e88286a3" +dependencies = [ + "cache-padded", +] + [[package]] name = "console" version = "0.15.0" @@ -115,6 +153,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" +dependencies = [ + "cfg-if", + "lazy_static", +] + [[package]] name = "crossterm" version = "0.23.2" @@ -140,6 +188,37 @@ dependencies = [ "winapi", ] +[[package]] +name = "curl" +version = "0.4.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d855aeef205b43f65a5001e0997d81f8efca7badad4fad7d897aa7f0d0651f" +dependencies = [ + "curl-sys", + "libc", + "openssl-probe", + "openssl-sys", + "schannel", + "socket2", + "winapi", +] + +[[package]] +name = "curl-sys" +version = "0.4.55+curl-7.83.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23734ec77368ec583c2e61dd3f0b0e5c98b93abe6d2a004ca06b91dd7e3e2762" +dependencies = [ + "cc", + "libc", + "libnghttp2-sys", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", + "winapi", +] + [[package]] name = "dirs-next" version = "2.0.0" @@ -167,6 +246,36 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +[[package]] +name = "encoding_rs" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "event-listener" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77f3309417938f28bf8228fcff79a4a37103981e3e186d2ccd19c74b38f4eb71" + +[[package]] +name = "fastrand" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf" +dependencies = [ + "instant", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "form_urlencoded" version = "1.0.1" @@ -183,6 +292,33 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" +[[package]] +name = "futures-core" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3" + +[[package]] +name = "futures-io" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b" + +[[package]] +name = "futures-lite" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + [[package]] name = "getrandom" version = "0.2.6" @@ -202,8 +338,11 @@ dependencies = [ "comfy-table", "console", "git2", + "isahc", + "parse_link_header", "regex", "serde", + "serde_json", "serde_yaml", "shellexpand", "tempdir", @@ -255,6 +394,17 @@ dependencies = [ "libc", ] +[[package]] +name = "http" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff8670570af52249509a86f5e3e18a08c60b177071826898fde8997cf5f6bfbb" +dependencies = [ + "bytes", + "fnv", + "itoa 1.0.2", +] + [[package]] name = "idna" version = "0.2.3" @@ -276,6 +426,56 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "isahc" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "480d9158c9977bff0bc024a11dcad04efcd3955c1e55301092b13fc439d41720" +dependencies = [ + "async-channel", + "castaway", + "crossbeam-utils", + "curl", + "curl-sys", + "encoding_rs", + "event-listener", + "futures-lite", + "http", + "log", + "mime", + "once_cell", + "polling", + "serde", + "serde_json", + "slab", + "sluice", + "tracing", + "tracing-futures", + "url", + "waker-fn", +] + +[[package]] +name = "itoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" + +[[package]] +name = "itoa" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" + [[package]] name = "jobserver" version = "0.1.24" @@ -311,6 +511,16 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "libnghttp2-sys" +version = "0.1.7+1.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57ed28aba195b38d5ff02b9170cbff627e336a20925e43b4945390401c5dc93f" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "libssh2-sys" version = "0.2.23" @@ -374,6 +584,12 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + [[package]] name = "mio" version = "0.8.3" @@ -417,6 +633,12 @@ version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21326818e99cfe6ce1e524c2a805c189a99b5ae555a35d19f9a284b427d86afa" +[[package]] +name = "parking" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" + [[package]] name = "parking_lot" version = "0.12.0" @@ -440,18 +662,68 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "parse_link_header" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40728c9c01de984c45f49385ab054fdc31cd3322658a6934347887e72cb48df9" +dependencies = [ + "http", + "lazy_static", + "regex", +] + [[package]] name = "percent-encoding" version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +[[package]] +name = "pin-project" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58ad3879ad3baf4e44784bc6a718a8698867bb991f8ce24d1bcbe2cfb4c3a75e" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "744b6f092ba29c3650faf274db506afd39944f48420f6c86b17cfe0ee1cb36bb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + [[package]] name = "pkg-config" version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" +[[package]] +name = "polling" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "685404d509889fade3e86fe3a5803bca2ec09b0c0778d5ada6ec8bf7a8de5259" +dependencies = [ + "cfg-if", + "libc", + "log", + "wepoll-ffi", + "winapi", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -589,6 +861,16 @@ version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" +[[package]] +name = "schannel" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" +dependencies = [ + "lazy_static", + "windows-sys", +] + [[package]] name = "scopeguard" version = "1.1.0" @@ -615,6 +897,17 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_json" +version = "1.0.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcac07dbffa1c65e7f816ab9eba78eb142c6d44410f4eeba1e26e4f5dfa56b95" +dependencies = [ + "itoa 0.4.8", + "ryu", + "serde", +] + [[package]] name = "serde_yaml" version = "0.8.24" @@ -666,12 +959,39 @@ dependencies = [ "libc", ] +[[package]] +name = "slab" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" + +[[package]] +name = "sluice" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7400c0eff44aa2fcb5e31a5f24ba9716ed90138769e4977a2ba6014ae63eb5" +dependencies = [ + "async-channel", + "futures-core", + "futures-io", +] + [[package]] name = "smallvec" version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" +[[package]] +name = "socket2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "strsim" version = "0.10.0" @@ -787,6 +1107,49 @@ dependencies = [ "serde", ] +[[package]] +name = "tracing" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09" +dependencies = [ + "cfg-if", + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + [[package]] name = "unicode-bidi" version = "0.3.8" @@ -844,6 +1207,12 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "waker-fn" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" + [[package]] name = "wasi" version = "0.10.2+wasi-snapshot-preview1" @@ -856,6 +1225,15 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wepoll-ffi" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d743fdedc5c64377b5fc2bc036b01c7fd642205a0d96356034ae3404d49eb7fb" +dependencies = [ + "cc", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 974bbe0..25fed11 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,9 @@ rust-version = "1.57" license = "GPL-3.0-only" +[profile.e2e-tests] +inherits = "release" + [lib] name = "grm" path = "src/lib.rs" @@ -65,5 +68,15 @@ version = "=5.0.1" [dependencies.serde_yaml] version = "=0.8.24" +[dependencies.serde_json] +version = "=1.0.59" + +[dependencies.isahc] +version = "=1.7.1" +features = ["json"] + +[dependencies.parse_link_header] +version = "=0.3.2" + [dev-dependencies.tempdir] version = "=0.3.7" diff --git a/Justfile b/Justfile index bf74f55..84e7886 100644 --- a/Justfile +++ b/Justfile @@ -1,4 +1,6 @@ -check: check-cargo-lock check-pip-requirements test +set positional-arguments + +check: check-cargo-lock test cargo check cargo fmt --check cargo clippy --no-deps -- -Dwarnings @@ -12,6 +14,18 @@ lint-fix: release: cargo build --release +test-binary-docker: + env \ + GITHUB_API_BASEURL=http://rest:5000/github \ + GITLAB_API_BASEURL=http://rest:5000/gitlab \ + cargo build --profile e2e-tests + +test-binary: + env \ + GITHUB_API_BASEURL=http://localhost:5000/github \ + GITLAB_API_BASEURL=http://localhost:5000/gitlab \ + cargo build --profile e2e-tests + install: cargo install --path . @@ -23,19 +37,26 @@ test-unit: test-integration: cargo test --test "*" -e2e-venv: +test-e2e-docker +tests=".": test-binary-docker cd ./e2e_tests \ - && python3 -m venv venv \ - && . ./venv/bin/activate \ - && pip --disable-pip-version-check install -r ./requirements.txt >/dev/null + && docker-compose rm --stop -f \ + && docker-compose build \ + && docker-compose run \ + --rm \ + -v $PWD/../target/e2e-tests/grm:/grm \ + pytest \ + "GRM_BINARY=/grm python3 ALTERNATE_DOMAIN=alternate-rest -m pytest -p no:cacheprovider --color=yes "$@"" \ + && docker-compose rm --stop -f - -test-e2e +tests=".": e2e-venv release +test-e2e +tests=".": test-binary cd ./e2e_tests \ - && . ./venv/bin/activate \ - && TMPDIR=/dev/shm python -m pytest --color=yes {{tests}} + && docker-compose rm --stop -f \ + && docker-compose build \ + && docker-compose up -d rest \ + && GRM_BINARY={{justfile_directory()}}/target/e2e-tests/grm ALTERNATE_DOMAIN=127.0.0.1 python3 -m pytest -p no:cacheprovider --color=yes {{tests}} \ + && docker-compose rm --stop -f -update-dependencies: update-cargo-dependencies update-pip-requirements +update-dependencies: update-cargo-dependencies update-cargo-dependencies: @cd ./depcheck \ @@ -43,15 +64,3 @@ update-cargo-dependencies: && . ./venv/bin/activate \ && pip --disable-pip-version-check install -r ./requirements.txt > /dev/null \ && ./update-cargo-dependencies.py - -update-pip-requirements: e2e-venv - @cd ./e2e_tests \ - && ./update_requirementstxt.sh - -check-pip-requirements: e2e-venv - @cd ./e2e_tests \ - && . ./venv/bin/activate \ - && pip list --outdated | grep -q '.' && exit 1 || exit 0 - -clean: - cargo clean diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 5af1148..cd7b635 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -4,5 +4,6 @@ - [Getting started](./getting_started.md) - [Repository trees](./repos.md) - [Git Worktrees](./worktrees.md) +- [Forge Integrations](./forge_integration.md) - [FAQ](./faq.md) - [Contributing](./contributing.md) diff --git a/docs/src/forge_integration.md b/docs/src/forge_integration.md new file mode 100644 index 0000000..f9382a7 --- /dev/null +++ b/docs/src/forge_integration.md @@ -0,0 +1,205 @@ +# Forge Integrations + +In addition to manging repositories locally, `grm` also integrates with source +code hosting platforms. Right now, the following platforms are supported: + +* [GitHub](https://github.com/) +* [GitLab](https://gitlab.com/) + +Imagine you are just starting out with `grm` and want to clone all your repositories +from GitHub. This is as simple as: + +```bash +$ grm repos sync remote --provider github --owner --token-command "pass show github_grm_access_token --path ~/projects" +``` + +You will end up with your projects cloned into `~/projects/{your_github_username}/` + +## Authentication + +The only currently supported authentication option is using personal access +token. + +### GitHub + +See the GitHub documentation for personal access tokens: +[Link](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token). + +The only required permission is the "repo" scope. + +### GitHub + +See the GitLab documentation for personal access tokens: +[Link](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html). + +The required scopes are a bit weird. Actually, the following should suffice: + +* * `read_user` to get user information (required to get the current authenticated + user name for the `--owner` filter. +* A scope that allows reading private repositories. (`read_repository` is just + for *cloning* private repos). This unfortunately does not exist. + +So currently, you'll need to select the `read_api` scope. + +## Filters + +By default, `grm` will sync **nothing**. This is quite boring, so you have to +tell the command what repositories to include. They are all inclusive (i.e. act +as a logical OR), so you can easily chain many filters to clone a bunch of +repositories. It's quite simple: + +* `--user ` syncs all repositories of that remote user +* `--group ` syncs all repositories of that remote group/organization +* `--owner` syncs all repositories of the user that is used for authentication. + This is effectively a shortcut for `--user $YOUR_USER` +* `--access` syncs all repositories that the current user has access to + +Easiest to see in an example: + +```bash +$ grm repos sync remote --provider github --user torvals --owner --group zalando [...] +``` + +This would sync all of Torvald's repositories, all of my own repositories and +all (public) repositories in the "zalando" group. + +## Strategies + +There are generally three ways how you can use `grm` with forges: + +### Ad-hoc cloning + +This is the easiest, there are no local files involved. You just run the +command, `grm` clones the repos, that's it. If you run the command again, `grm` +will figure out the differences between local and remote repositories and +resolve them locally. + +### Create a file + +This is effectively `grm repos find local`, but using the forge instead of the +local file system. You will end up with a normal repository file that you can +commit to git. To update the list of repositories, just run the command again +and commit the new file. + +### Define options in a file + +This is a hybrid approach: You define filtering options in a file that you can +commit to source control. Effectively, you are persisting the options you gave +to `grm` on the command line with the ad-hoc approach. Similarly, `grm` will +figure out differences between local and remote and resolve them. + +A file would look like this: + +```toml +provider = "github" +token_command = "cat ~/.github_token" +root = "~/projects" + +[filters] +owner = true +groups = [ + "zalando" +] +``` + +The options in the file map to the command line options of the `grm repos sync +remote` command. + +You'd then run the `grm repos sync` command the same way as with a list of +repositories in a config: + +```bash +$ grm repos sync --config example.config.toml +``` + +You can even use that file to generate a repository list that you can feed into +`grm repos sync`: + +```bash +$ grm repos find config --config example.config.toml > repos.toml +$ grm repos sync config --config repos.toml +``` + +## Using with selfhosted GitLab + +By default, `grm` uses the default GitLab API endpoint +([https://gitlab.com](https://gitlab.com)). You can override the +endpoint by specifying the `--api-url` parameter. Like this: + +```bash +$ grm repos sync remote --provider gitlab --api-url https://gitlab.example.com [...] +``` + +## The cloning protocol + +By default, `grm` will use HTTPS for public repositories and SSH otherwise. This +can be overridden with the `--force-ssh` switch. + +## About the token command + +To ensure maximum flexibility, `grm` has a single way to get the token it uses +to authenticate: Specify a command that returns the token via stdout. This easily +integrates with password managers like [`pass`](https://www.passwordstore.org/). + +Of course, you are also free to specify something like `echo mytoken` as the +command, as long as you are ok with the security implications (like having the +token in cleartext in your shell history). It may be better to have the token +in a file instead and read it: `cat ~/.gitlab_token`. + +Generally, use whatever you want. The command just has to return sucessfully and +return the token as the first line of stdout. + +## Examples + +Maybe you just want to locally clone all repos from your github user? + +```bash +$ grm repos sync remote --provider github --owner --root ~/github_projects --token-command "pass show github_grm_access_token" +``` + +This will clone all repositories into `~/github_projects/{your_github_username}`. + +If instead you want to clone **all** repositories you have access to (e.g. via +organizations or other users' private repos you have access to), just change the +filter a little bit: + +```bash +$ grm repos sync remote --provider github --access --root ~/github_projects --token-command "pass show github_grm_access_token" +``` + +## Limitations + +### GitHub + +Unfortunately, GitHub does not have a nice API endpoint to get **private** +repositories for a certain user ([`/users/{user}/repos/`](https://docs.github.com/en/rest/repos/repos#list-repositories-for-a-user) only returns public +repositories). + +Therefore, using `--user {user}` will only show public repositories for GitHub. +Note that this does not apply to `--access`: If you have access to another user's +private repository, it will be listed. + +## Adding integrations + +Adding a new integration involves writing some Rust code. Most of the logic is +generic, so you will not have to reinvent the wheel. Generally, you will need to +gather the following information: + +* A list of repositories for a single user +* A list of repositories for a group (or any similar concept if applicable) +* A list of repositories for the user that the API token belongs to +* The username of the currently authenticated user + +Authentication currently only works via a bearer token passed via the +`Authorization` HTTP header. + +Each repo has to have the following properties: + +* A name (which also acts as the identifier for diff between local and remote + repositories) +* An SSH url to push to +* An HTTPS url to clone and fetch from +* A flag that marks the repository as private + +If you plan to implement another forge, please first open an issue so we can +go through the required setup. I'm happy to help! diff --git a/docs/src/repos.md b/docs/src/repos.md index 69004f8..cd8fe10 100644 --- a/docs/src/repos.md +++ b/docs/src/repos.md @@ -17,7 +17,7 @@ Then, you're ready to run the first sync. This will clone all configured reposit and set up the remotes. ```bash -$ grm repos sync --config example.config.toml +$ grm repos sync config --config example.config.toml [⚙] Cloning into "/home/me/projects/git-repo-manager" from "https://code.hkoerber.de/hannes/git-repo-manager.git" [✔] git-repo-manager: Repository successfully cloned [⚙] git-repo-manager: Setting up new remote "github" to "https://github.com/hakoerber/git-repo-manager.git" @@ -30,7 +30,7 @@ $ grm repos sync --config example.config.toml If you run it again, it will report no changes: ``` -$ grm repos sync --config example.config.toml +$ grm repos sync config -c example.config.toml [✔] git-repo-manager: OK [✔] dotfiles: OK ``` @@ -42,7 +42,7 @@ a configuration from scratch. Luckily, GRM has a way to generate a configuration from an existing file tree: ```bash -$ grm repos find ~/your/project/root > config.toml +$ grm repos find local ~/your/project/root > config.toml ``` This will detect all repositories and remotes and write them to `config.toml`. diff --git a/e2e_tests/.gitignore b/e2e_tests/.gitignore deleted file mode 100644 index e79509f..0000000 --- a/e2e_tests/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/venv/ -/__pycache__/ diff --git a/e2e_tests/conftest.py b/e2e_tests/conftest.py new file mode 100644 index 0000000..ac3ba25 --- /dev/null +++ b/e2e_tests/conftest.py @@ -0,0 +1,8 @@ +import os + + +def pytest_configure(config): + os.environ["GIT_AUTHOR_NAME"] = "Example user" + os.environ["GIT_AUTHOR_EMAIL"] = "user@example.com" + os.environ["GIT_COMMITTER_NAME"] = "Example user" + os.environ["GIT_COMMITTER_EMAIL"] = "user@example.com" diff --git a/e2e_tests/docker-compose.yml b/e2e_tests/docker-compose.yml new file mode 100644 index 0000000..f75f12a --- /dev/null +++ b/e2e_tests/docker-compose.yml @@ -0,0 +1,34 @@ +version: "3.7" + +services: + pytest: + build: ./docker + volumes: + - type: bind + source: ./ + target: /tests + read_only: true + - type: tmpfs + target: /tmp + environment: + TMPDIR: /tmp + depends_on: + - rest + command: + - "true" + networks: + main: + + rest: + build: ./docker-rest/ + expose: + - "5000" + ports: + - "5000:5000" + networks: + main: + aliases: + - alternate-rest + +networks: + main: diff --git a/e2e_tests/docker-rest/Dockerfile b/e2e_tests/docker-rest/Dockerfile new file mode 100644 index 0000000..b4abb91 --- /dev/null +++ b/e2e_tests/docker-rest/Dockerfile @@ -0,0 +1,19 @@ +FROM docker.io/debian:11.3 + +WORKDIR /app + +ENV FLASK_APP=app.py + +RUN apt-get update \ + && apt-get install -y \ + dumb-init \ + python3-flask \ + python3-jinja2 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +EXPOSE 5000 + +COPY flask . + +CMD ["/usr/bin/dumb-init", "--", "flask", "run", "--port", "5000", "--host", "0.0.0.0"] diff --git a/e2e_tests/docker-rest/flask/app.py b/e2e_tests/docker-rest/flask/app.py new file mode 100644 index 0000000..e22c533 --- /dev/null +++ b/e2e_tests/docker-rest/flask/app.py @@ -0,0 +1,7 @@ +from flask import Flask + +app = Flask(__name__) +app.url_map.strict_slashes = False + +import github +import gitlab diff --git a/e2e_tests/docker-rest/flask/github.py b/e2e_tests/docker-rest/flask/github.py new file mode 100644 index 0000000..6a1f29d --- /dev/null +++ b/e2e_tests/docker-rest/flask/github.py @@ -0,0 +1,103 @@ +import os.path + +from app import app + +from flask import Flask, request, abort, jsonify, make_response + +import jinja2 + + +def check_headers(): + if request.headers.get("accept") != "application/vnd.github.v3+json": + app.logger.error("Invalid accept header") + abort(500) + auth_header = request.headers.get("authorization") + if auth_header != "token authtoken": + app.logger.error("Invalid authorization header: %s", auth_header) + abort( + make_response( + jsonify( + { + "message": "Bad credentials", + "documentation_url": "https://docs.example.com/rest", + } + ), + 401, + ) + ) + + +def add_pagination(response, page, last_page): + host = request.headers["host"] + link_header = "" + + def args(page): + args = request.args.copy() + args["page"] = page + return "&".join([f"{k}={v}" for k, v in args.items()]) + + if page < last_page: + link_header += ( + f'<{request.scheme}://{host}{request.path}?{args(page+1)}>; rel="next", ' + ) + link_header += ( + f'<{request.scheme}://{host}{request.path}?{args(last_page)}>; rel="last"' + ) + response.headers["link"] = link_header + + +def read_project_files(namespaces=[]): + last_page = 4 + page = username = int(request.args.get("page", "1")) + response_file = f"./github_api_page_{page}.json.j2" + if not os.path.exists(response_file): + return jsonify([]) + + response = make_response( + jinja2.Template(open(response_file).read()).render( + namespace=namespaces[page - 1] + ) + ) + add_pagination(response, page, last_page) + response.headers["content-type"] = "application/json" + return response + + +def single_namespaced_projects(namespace): + return read_project_files([namespace] * 4) + + +def mixed_projects(namespaces): + return read_project_files(namespaces) + + +@app.route("/github/users//repos/") +def github_user_repos(user): + check_headers() + if user == "myuser1": + return single_namespaced_projects("myuser1") + return jsonify([]) + + +@app.route("/github/orgs//repos/") +def github_group_repos(group): + check_headers() + if not (request.args.get("type") == "all"): + abort(500, "wrong arguments") + if group == "mygroup1": + return single_namespaced_projects("mygroup1") + return jsonify([]) + + +@app.route("/github/user/repos/") +def github_own_repos(): + check_headers() + return mixed_projects(["myuser1", "myuser2", "mygroup1", "mygroup2"]) + + +@app.route("/github/user/") +def github_user(): + check_headers() + response = make_response(open("./github_api_user.json").read()) + response.headers["content-type"] = "application/json" + return response diff --git a/e2e_tests/docker-rest/flask/github_api_page_1.json.j2 b/e2e_tests/docker-rest/flask/github_api_page_1.json.j2 new file mode 100644 index 0000000..9606852 --- /dev/null +++ b/e2e_tests/docker-rest/flask/github_api_page_1.json.j2 @@ -0,0 +1,228 @@ +[ + { + "id": 1, + "node_id": "MDEwOlJlcG9zaXRvcnk0OTIzNDY2Ng==", + "name": "myproject1", + "full_name": "{{ namespace }}/myproject1", + "private": true, + "owner": { + "login": "someuser", + "id": 1, + "node_id": "MDQ6VXNlcjM3NDg2OTY=", + "avatar_url": "https://example.com/u/3748696?v=4", + "gravatar_id": "", + "url": "https://api.example.com/users/{{ namespace }}", + "html_url": "https://example.com/{{ namespace }}", + "followers_url": "https://api.example.com/users/{{ namespace }}/followers", + "following_url": "https://api.example.com/users/{{ namespace }}/following{/other_user}", + "gists_url": "https://api.example.com/users/{{ namespace }}/gists{/gist_id}", + "starred_url": "https://api.example.com/users/{{ namespace }}/starred{/owner}{/repo}", + "subscriptions_url": "https://api.example.com/users/{{ namespace }}/subscriptions", + "organizations_url": "https://api.example.com/users/{{ namespace }}/orgs", + "repos_url": "https://api.example.com/users/{{ namespace }}/repos", + "events_url": "https://api.example.com/users/{{ namespace }}/events{/privacy}", + "received_events_url": "https://api.example.com/users/{{ namespace }}/received_events", + "type": "User", + "site_admin": false + }, + "html_url": "https://example.com/{{ namespace }}/myproject1", + "description": "Shell script for automatically building ACI containers from scratch using acbuild.", + "fork": false, + "url": "https://api.example.com/repos/{{ namespace }}/myproject1", + "forks_url": "https://api.example.com/repos/{{ namespace }}/myproject1/forks", + "keys_url": "https://api.example.com/repos/{{ namespace }}/myproject1/keys{/key_id}", + "collaborators_url": "https://api.example.com/repos/{{ namespace }}/myproject1/collaborators{/collaborator}", + "teams_url": "https://api.example.com/repos/{{ namespace }}/myproject1/teams", + "hooks_url": "https://api.example.com/repos/{{ namespace }}/myproject1/hooks", + "issue_events_url": "https://api.example.com/repos/{{ namespace }}/myproject1/issues/events{/number}", + "events_url": "https://api.example.com/repos/{{ namespace }}/myproject1/events", + "assignees_url": "https://api.example.com/repos/{{ namespace }}/myproject1/assignees{/user}", + "branches_url": "https://api.example.com/repos/{{ namespace }}/myproject1/branches{/branch}", + "tags_url": "https://api.example.com/repos/{{ namespace }}/myproject1/tags", + "blobs_url": "https://api.example.com/repos/{{ namespace }}/myproject1/git/blobs{/sha}", + "git_tags_url": "https://api.example.com/repos/{{ namespace }}/myproject1/git/tags{/sha}", + "git_refs_url": "https://api.example.com/repos/{{ namespace }}/myproject1/git/refs{/sha}", + "trees_url": "https://api.example.com/repos/{{ namespace }}/myproject1/git/trees{/sha}", + "statuses_url": "https://api.example.com/repos/{{ namespace }}/myproject1/statuses/{sha}", + "languages_url": "https://api.example.com/repos/{{ namespace }}/myproject1/languages", + "stargazers_url": "https://api.example.com/repos/{{ namespace }}/myproject1/stargazers", + "contributors_url": "https://api.example.com/repos/{{ namespace }}/myproject1/contributors", + "subscribers_url": "https://api.example.com/repos/{{ namespace }}/myproject1/subscribers", + "subscription_url": "https://api.example.com/repos/{{ namespace }}/myproject1/subscription", + "commits_url": "https://api.example.com/repos/{{ namespace }}/myproject1/commits{/sha}", + "git_commits_url": "https://api.example.com/repos/{{ namespace }}/myproject1/git/commits{/sha}", + "comments_url": "https://api.example.com/repos/{{ namespace }}/myproject1/comments{/number}", + "issue_comment_url": "https://api.example.com/repos/{{ namespace }}/myproject1/issues/comments{/number}", + "contents_url": "https://api.example.com/repos/{{ namespace }}/myproject1/contents/{+path}", + "compare_url": "https://api.example.com/repos/{{ namespace }}/myproject1/compare/{base}...{head}", + "merges_url": "https://api.example.com/repos/{{ namespace }}/myproject1/merges", + "archive_url": "https://api.example.com/repos/{{ namespace }}/myproject1/{archive_format}{/ref}", + "downloads_url": "https://api.example.com/repos/{{ namespace }}/myproject1/downloads", + "issues_url": "https://api.example.com/repos/{{ namespace }}/myproject1/issues{/number}", + "pulls_url": "https://api.example.com/repos/{{ namespace }}/myproject1/pulls{/number}", + "milestones_url": "https://api.example.com/repos/{{ namespace }}/myproject1/milestones{/number}", + "notifications_url": "https://api.example.com/repos/{{ namespace }}/myproject1/notifications{?since,all,participating}", + "labels_url": "https://api.example.com/repos/{{ namespace }}/myproject1/labels{/name}", + "releases_url": "https://api.example.com/repos/{{ namespace }}/myproject1/releases{/id}", + "deployments_url": "https://api.example.com/repos/{{ namespace }}/myproject1/deployments", + "created_at": "2016-01-07T22:27:54Z", + "updated_at": "2021-11-20T16:15:37Z", + "pushed_at": "2021-11-20T16:15:34Z", + "git_url": "git://example.com/{{ namespace }}/myproject1.git", + "ssh_url": "ssh://git@example.com/{{ namespace }}/myproject1.git", + "clone_url": "https://example.com/{{ namespace }}/myproject1.git", + "svn_url": "https://example.com/{{ namespace }}/myproject1", + "homepage": null, + "size": 12, + "stargazers_count": 0, + "watchers_count": 0, + "language": "Shell", + "has_issues": true, + "has_projects": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": false, + "forks_count": 0, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 0, + "license": { + "key": "apache-2.0", + "name": "Apache License 2.0", + "spdx_id": "Apache-2.0", + "url": "https://api.example.com/licenses/apache-2.0", + "node_id": "MDc6TGljZW5zZTI=" + }, + "allow_forking": true, + "is_template": false, + "topics": [ + + ], + "visibility": "public", + "forks": 0, + "open_issues": 0, + "watchers": 0, + "default_branch": "master", + "permissions": { + "admin": true, + "maintain": true, + "push": true, + "triage": true, + "pull": true + } + }, + { + "id": 2, + "node_id": "MDEwOlJlcG9zaXRvcnk0OTIzNDY2Ng==", + "name": "myproject2", + "full_name": "{{ namespace }}/myproject2", + "private": false, + "owner": { + "login": "someuser", + "id": 1, + "node_id": "MDQ6VXNlcjM3NDg2OTY=", + "avatar_url": "https://example.com/u/3748696?v=4", + "gravatar_id": "", + "url": "https://api.example.com/users/{{ namespace }}", + "html_url": "https://example.com/{{ namespace }}", + "followers_url": "https://api.example.com/users/{{ namespace }}/followers", + "following_url": "https://api.example.com/users/{{ namespace }}/following{/other_user}", + "gists_url": "https://api.example.com/users/{{ namespace }}/gists{/gist_id}", + "starred_url": "https://api.example.com/users/{{ namespace }}/starred{/owner}{/repo}", + "subscriptions_url": "https://api.example.com/users/{{ namespace }}/subscriptions", + "organizations_url": "https://api.example.com/users/{{ namespace }}/orgs", + "repos_url": "https://api.example.com/users/{{ namespace }}/repos", + "events_url": "https://api.example.com/users/{{ namespace }}/events{/privacy}", + "received_events_url": "https://api.example.com/users/{{ namespace }}/received_events", + "type": "User", + "site_admin": false + }, + "html_url": "https://example.com/{{ namespace }}/myproject2", + "description": "Shell script for automatically building ACI containers from scratch using acbuild.", + "fork": false, + "url": "https://api.example.com/repos/{{ namespace }}/myproject2", + "forks_url": "https://api.example.com/repos/{{ namespace }}/myproject2/forks", + "keys_url": "https://api.example.com/repos/{{ namespace }}/myproject2/keys{/key_id}", + "collaborators_url": "https://api.example.com/repos/{{ namespace }}/myproject2/collaborators{/collaborator}", + "teams_url": "https://api.example.com/repos/{{ namespace }}/myproject2/teams", + "hooks_url": "https://api.example.com/repos/{{ namespace }}/myproject2/hooks", + "issue_events_url": "https://api.example.com/repos/{{ namespace }}/myproject2/issues/events{/number}", + "events_url": "https://api.example.com/repos/{{ namespace }}/myproject2/events", + "assignees_url": "https://api.example.com/repos/{{ namespace }}/myproject2/assignees{/user}", + "branches_url": "https://api.example.com/repos/{{ namespace }}/myproject2/branches{/branch}", + "tags_url": "https://api.example.com/repos/{{ namespace }}/myproject2/tags", + "blobs_url": "https://api.example.com/repos/{{ namespace }}/myproject2/git/blobs{/sha}", + "git_tags_url": "https://api.example.com/repos/{{ namespace }}/myproject2/git/tags{/sha}", + "git_refs_url": "https://api.example.com/repos/{{ namespace }}/myproject2/git/refs{/sha}", + "trees_url": "https://api.example.com/repos/{{ namespace }}/myproject2/git/trees{/sha}", + "statuses_url": "https://api.example.com/repos/{{ namespace }}/myproject2/statuses/{sha}", + "languages_url": "https://api.example.com/repos/{{ namespace }}/myproject2/languages", + "stargazers_url": "https://api.example.com/repos/{{ namespace }}/myproject2/stargazers", + "contributors_url": "https://api.example.com/repos/{{ namespace }}/myproject2/contributors", + "subscribers_url": "https://api.example.com/repos/{{ namespace }}/myproject2/subscribers", + "subscription_url": "https://api.example.com/repos/{{ namespace }}/myproject2/subscription", + "commits_url": "https://api.example.com/repos/{{ namespace }}/myproject2/commits{/sha}", + "git_commits_url": "https://api.example.com/repos/{{ namespace }}/myproject2/git/commits{/sha}", + "comments_url": "https://api.example.com/repos/{{ namespace }}/myproject2/comments{/number}", + "issue_comment_url": "https://api.example.com/repos/{{ namespace }}/myproject2/issues/comments{/number}", + "contents_url": "https://api.example.com/repos/{{ namespace }}/myproject2/contents/{+path}", + "compare_url": "https://api.example.com/repos/{{ namespace }}/myproject2/compare/{base}...{head}", + "merges_url": "https://api.example.com/repos/{{ namespace }}/myproject2/merges", + "archive_url": "https://api.example.com/repos/{{ namespace }}/myproject2/{archive_format}{/ref}", + "downloads_url": "https://api.example.com/repos/{{ namespace }}/myproject2/downloads", + "issues_url": "https://api.example.com/repos/{{ namespace }}/myproject2/issues{/number}", + "pulls_url": "https://api.example.com/repos/{{ namespace }}/myproject2/pulls{/number}", + "milestones_url": "https://api.example.com/repos/{{ namespace }}/myproject2/milestones{/number}", + "notifications_url": "https://api.example.com/repos/{{ namespace }}/myproject2/notifications{?since,all,participating}", + "labels_url": "https://api.example.com/repos/{{ namespace }}/myproject2/labels{/name}", + "releases_url": "https://api.example.com/repos/{{ namespace }}/myproject2/releases{/id}", + "deployments_url": "https://api.example.com/repos/{{ namespace }}/myproject2/deployments", + "created_at": "2016-01-07T22:27:54Z", + "updated_at": "2021-11-20T16:15:37Z", + "pushed_at": "2021-11-20T16:15:34Z", + "git_url": "git://example.com/{{ namespace }}/myproject2.git", + "ssh_url": "ssh://git@example.com/{{ namespace }}/myproject2.git", + "clone_url": "https://example.com/{{ namespace }}/myproject2.git", + "svn_url": "https://example.com/{{ namespace }}/myproject2", + "homepage": null, + "size": 12, + "stargazers_count": 0, + "watchers_count": 0, + "language": "Shell", + "has_issues": true, + "has_projects": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": false, + "forks_count": 0, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 0, + "license": { + "key": "apache-2.0", + "name": "Apache License 2.0", + "spdx_id": "Apache-2.0", + "url": "https://api.example.com/licenses/apache-2.0", + "node_id": "MDc6TGljZW5zZTI=" + }, + "allow_forking": true, + "is_template": false, + "topics": [ + + ], + "visibility": "public", + "forks": 0, + "open_issues": 0, + "watchers": 0, + "default_branch": "master", + "permissions": { + "admin": true, + "maintain": true, + "push": true, + "triage": true, + "pull": true + } + } +] diff --git a/e2e_tests/docker-rest/flask/github_api_page_2.json.j2 b/e2e_tests/docker-rest/flask/github_api_page_2.json.j2 new file mode 100644 index 0000000..c195453 --- /dev/null +++ b/e2e_tests/docker-rest/flask/github_api_page_2.json.j2 @@ -0,0 +1,115 @@ +[ + { + "id": 3, + "node_id": "MDEwOlJlcG9zaXRvcnk0OTIzNDY2Ng==", + "name": "myproject3", + "full_name": "{{ namespace }}/myproject3", + "private": false, + "owner": { + "login": "someuser", + "id": 1, + "node_id": "MDQ6VXNlcjM3NDg2OTY=", + "avatar_url": "https://example.com/u/3748696?v=4", + "gravatar_id": "", + "url": "https://api.example.com/users/{{ namespace }}", + "html_url": "https://example.com/{{ namespace }}", + "followers_url": "https://api.example.com/users/{{ namespace }}/followers", + "following_url": "https://api.example.com/users/{{ namespace }}/following{/other_user}", + "gists_url": "https://api.example.com/users/{{ namespace }}/gists{/gist_id}", + "starred_url": "https://api.example.com/users/{{ namespace }}/starred{/owner}{/repo}", + "subscriptions_url": "https://api.example.com/users/{{ namespace }}/subscriptions", + "organizations_url": "https://api.example.com/users/{{ namespace }}/orgs", + "repos_url": "https://api.example.com/users/{{ namespace }}/repos", + "events_url": "https://api.example.com/users/{{ namespace }}/events{/privacy}", + "received_events_url": "https://api.example.com/users/{{ namespace }}/received_events", + "type": "User", + "site_admin": false + }, + "html_url": "https://example.com/{{ namespace }}/myproject3", + "description": "Shell script for automatically building ACI containers from scratch using acbuild.", + "fork": false, + "url": "https://api.example.com/repos/{{ namespace }}/myproject3", + "forks_url": "https://api.example.com/repos/{{ namespace }}/myproject3/forks", + "keys_url": "https://api.example.com/repos/{{ namespace }}/myproject3/keys{/key_id}", + "collaborators_url": "https://api.example.com/repos/{{ namespace }}/myproject3/collaborators{/collaborator}", + "teams_url": "https://api.example.com/repos/{{ namespace }}/myproject3/teams", + "hooks_url": "https://api.example.com/repos/{{ namespace }}/myproject3/hooks", + "issue_events_url": "https://api.example.com/repos/{{ namespace }}/myproject3/issues/events{/number}", + "events_url": "https://api.example.com/repos/{{ namespace }}/myproject3/events", + "assignees_url": "https://api.example.com/repos/{{ namespace }}/myproject3/assignees{/user}", + "branches_url": "https://api.example.com/repos/{{ namespace }}/myproject3/branches{/branch}", + "tags_url": "https://api.example.com/repos/{{ namespace }}/myproject3/tags", + "blobs_url": "https://api.example.com/repos/{{ namespace }}/myproject3/git/blobs{/sha}", + "git_tags_url": "https://api.example.com/repos/{{ namespace }}/myproject3/git/tags{/sha}", + "git_refs_url": "https://api.example.com/repos/{{ namespace }}/myproject3/git/refs{/sha}", + "trees_url": "https://api.example.com/repos/{{ namespace }}/myproject3/git/trees{/sha}", + "statuses_url": "https://api.example.com/repos/{{ namespace }}/myproject3/statuses/{sha}", + "languages_url": "https://api.example.com/repos/{{ namespace }}/myproject3/languages", + "stargazers_url": "https://api.example.com/repos/{{ namespace }}/myproject3/stargazers", + "contributors_url": "https://api.example.com/repos/{{ namespace }}/myproject3/contributors", + "subscribers_url": "https://api.example.com/repos/{{ namespace }}/myproject3/subscribers", + "subscription_url": "https://api.example.com/repos/{{ namespace }}/myproject3/subscription", + "commits_url": "https://api.example.com/repos/{{ namespace }}/myproject3/commits{/sha}", + "git_commits_url": "https://api.example.com/repos/{{ namespace }}/myproject3/git/commits{/sha}", + "comments_url": "https://api.example.com/repos/{{ namespace }}/myproject3/comments{/number}", + "issue_comment_url": "https://api.example.com/repos/{{ namespace }}/myproject3/issues/comments{/number}", + "contents_url": "https://api.example.com/repos/{{ namespace }}/myproject3/contents/{+path}", + "compare_url": "https://api.example.com/repos/{{ namespace }}/myproject3/compare/{base}...{head}", + "merges_url": "https://api.example.com/repos/{{ namespace }}/myproject3/merges", + "archive_url": "https://api.example.com/repos/{{ namespace }}/myproject3/{archive_format}{/ref}", + "downloads_url": "https://api.example.com/repos/{{ namespace }}/myproject3/downloads", + "issues_url": "https://api.example.com/repos/{{ namespace }}/myproject3/issues{/number}", + "pulls_url": "https://api.example.com/repos/{{ namespace }}/myproject3/pulls{/number}", + "milestones_url": "https://api.example.com/repos/{{ namespace }}/myproject3/milestones{/number}", + "notifications_url": "https://api.example.com/repos/{{ namespace }}/myproject3/notifications{?since,all,participating}", + "labels_url": "https://api.example.com/repos/{{ namespace }}/myproject3/labels{/name}", + "releases_url": "https://api.example.com/repos/{{ namespace }}/myproject3/releases{/id}", + "deployments_url": "https://api.example.com/repos/{{ namespace }}/myproject3/deployments", + "created_at": "2016-01-07T22:27:54Z", + "updated_at": "2021-11-20T16:15:37Z", + "pushed_at": "2021-11-20T16:15:34Z", + "git_url": "git://example.com/{{ namespace }}/myproject3.git", + "ssh_url": "ssh://git@example.com/{{ namespace }}/myproject3.git", + "clone_url": "https://example.com/{{ namespace }}/myproject3.git", + "svn_url": "https://example.com/{{ namespace }}/myproject3", + "homepage": null, + "size": 12, + "stargazers_count": 0, + "watchers_count": 0, + "language": "Shell", + "has_issues": true, + "has_projects": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": false, + "forks_count": 0, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 0, + "license": { + "key": "apache-2.0", + "name": "Apache License 2.0", + "spdx_id": "Apache-2.0", + "url": "https://api.example.com/licenses/apache-2.0", + "node_id": "MDc6TGljZW5zZTI=" + }, + "allow_forking": true, + "is_template": false, + "topics": [ + + ], + "visibility": "public", + "forks": 0, + "open_issues": 0, + "watchers": 0, + "default_branch": "master", + "permissions": { + "admin": true, + "maintain": true, + "push": true, + "triage": true, + "pull": true + } + } +] diff --git a/e2e_tests/docker-rest/flask/github_api_page_3.json.j2 b/e2e_tests/docker-rest/flask/github_api_page_3.json.j2 new file mode 100644 index 0000000..003f331 --- /dev/null +++ b/e2e_tests/docker-rest/flask/github_api_page_3.json.j2 @@ -0,0 +1,115 @@ +[ + { + "id": 3, + "node_id": "MDEwOlJlcG9zaXRvcnk0OTIzNDY2Ng==", + "name": "myproject4", + "full_name": "{{ namespace }}/myproject4", + "private": false, + "owner": { + "login": "someuser", + "id": 1, + "node_id": "MDQ6VXNlcjM3NDg2OTY=", + "avatar_url": "https://example.com/u/3748696?v=4", + "gravatar_id": "", + "url": "https://api.example.com/users/{{ namespace }}", + "html_url": "https://example.com/{{ namespace }}", + "followers_url": "https://api.example.com/users/{{ namespace }}/followers", + "following_url": "https://api.example.com/users/{{ namespace }}/following{/other_user}", + "gists_url": "https://api.example.com/users/{{ namespace }}/gists{/gist_id}", + "starred_url": "https://api.example.com/users/{{ namespace }}/starred{/owner}{/repo}", + "subscriptions_url": "https://api.example.com/users/{{ namespace }}/subscriptions", + "organizations_url": "https://api.example.com/users/{{ namespace }}/orgs", + "repos_url": "https://api.example.com/users/{{ namespace }}/repos", + "events_url": "https://api.example.com/users/{{ namespace }}/events{/privacy}", + "received_events_url": "https://api.example.com/users/{{ namespace }}/received_events", + "type": "User", + "site_admin": false + }, + "html_url": "https://example.com/{{ namespace }}/myproject4", + "description": "Shell script for automatically building ACI containers from scratch using acbuild.", + "fork": false, + "url": "https://api.example.com/repos/{{ namespace }}/myproject4", + "forks_url": "https://api.example.com/repos/{{ namespace }}/myproject4/forks", + "keys_url": "https://api.example.com/repos/{{ namespace }}/myproject4/keys{/key_id}", + "collaborators_url": "https://api.example.com/repos/{{ namespace }}/myproject4/collaborators{/collaborator}", + "teams_url": "https://api.example.com/repos/{{ namespace }}/myproject4/teams", + "hooks_url": "https://api.example.com/repos/{{ namespace }}/myproject4/hooks", + "issue_events_url": "https://api.example.com/repos/{{ namespace }}/myproject4/issues/events{/number}", + "events_url": "https://api.example.com/repos/{{ namespace }}/myproject4/events", + "assignees_url": "https://api.example.com/repos/{{ namespace }}/myproject4/assignees{/user}", + "branches_url": "https://api.example.com/repos/{{ namespace }}/myproject4/branches{/branch}", + "tags_url": "https://api.example.com/repos/{{ namespace }}/myproject4/tags", + "blobs_url": "https://api.example.com/repos/{{ namespace }}/myproject4/git/blobs{/sha}", + "git_tags_url": "https://api.example.com/repos/{{ namespace }}/myproject4/git/tags{/sha}", + "git_refs_url": "https://api.example.com/repos/{{ namespace }}/myproject4/git/refs{/sha}", + "trees_url": "https://api.example.com/repos/{{ namespace }}/myproject4/git/trees{/sha}", + "statuses_url": "https://api.example.com/repos/{{ namespace }}/myproject4/statuses/{sha}", + "languages_url": "https://api.example.com/repos/{{ namespace }}/myproject4/languages", + "stargazers_url": "https://api.example.com/repos/{{ namespace }}/myproject4/stargazers", + "contributors_url": "https://api.example.com/repos/{{ namespace }}/myproject4/contributors", + "subscribers_url": "https://api.example.com/repos/{{ namespace }}/myproject4/subscribers", + "subscription_url": "https://api.example.com/repos/{{ namespace }}/myproject4/subscription", + "commits_url": "https://api.example.com/repos/{{ namespace }}/myproject4/commits{/sha}", + "git_commits_url": "https://api.example.com/repos/{{ namespace }}/myproject4/git/commits{/sha}", + "comments_url": "https://api.example.com/repos/{{ namespace }}/myproject4/comments{/number}", + "issue_comment_url": "https://api.example.com/repos/{{ namespace }}/myproject4/issues/comments{/number}", + "contents_url": "https://api.example.com/repos/{{ namespace }}/myproject4/contents/{+path}", + "compare_url": "https://api.example.com/repos/{{ namespace }}/myproject4/compare/{base}...{head}", + "merges_url": "https://api.example.com/repos/{{ namespace }}/myproject4/merges", + "archive_url": "https://api.example.com/repos/{{ namespace }}/myproject4/{archive_format}{/ref}", + "downloads_url": "https://api.example.com/repos/{{ namespace }}/myproject4/downloads", + "issues_url": "https://api.example.com/repos/{{ namespace }}/myproject4/issues{/number}", + "pulls_url": "https://api.example.com/repos/{{ namespace }}/myproject4/pulls{/number}", + "milestones_url": "https://api.example.com/repos/{{ namespace }}/myproject4/milestones{/number}", + "notifications_url": "https://api.example.com/repos/{{ namespace }}/myproject4/notifications{?since,all,participating}", + "labels_url": "https://api.example.com/repos/{{ namespace }}/myproject4/labels{/name}", + "releases_url": "https://api.example.com/repos/{{ namespace }}/myproject4/releases{/id}", + "deployments_url": "https://api.example.com/repos/{{ namespace }}/myproject4/deployments", + "created_at": "2016-01-07T22:27:54Z", + "updated_at": "2021-11-20T16:15:37Z", + "pushed_at": "2021-11-20T16:15:34Z", + "git_url": "git://example.com/{{ namespace }}/myproject4.git", + "ssh_url": "ssh://git@example.com/{{ namespace }}/myproject4.git", + "clone_url": "https://example.com/{{ namespace }}/myproject4.git", + "svn_url": "https://example.com/{{ namespace }}/myproject4", + "homepage": null, + "size": 12, + "stargazers_count": 0, + "watchers_count": 0, + "language": "Shell", + "has_issues": true, + "has_projects": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": false, + "forks_count": 0, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 0, + "license": { + "key": "apache-2.0", + "name": "Apache License 2.0", + "spdx_id": "Apache-2.0", + "url": "https://api.example.com/licenses/apache-2.0", + "node_id": "MDc6TGljZW5zZTI=" + }, + "allow_forking": true, + "is_template": false, + "topics": [ + + ], + "visibility": "public", + "forks": 0, + "open_issues": 0, + "watchers": 0, + "default_branch": "master", + "permissions": { + "admin": true, + "maintain": true, + "push": true, + "triage": true, + "pull": true + } + } +] diff --git a/e2e_tests/docker-rest/flask/github_api_page_4.json.j2 b/e2e_tests/docker-rest/flask/github_api_page_4.json.j2 new file mode 100644 index 0000000..e755be2 --- /dev/null +++ b/e2e_tests/docker-rest/flask/github_api_page_4.json.j2 @@ -0,0 +1,115 @@ +[ + { + "id": 3, + "node_id": "MDEwOlJlcG9zaXRvcnk0OTIzNDY2Ng==", + "name": "myproject5", + "full_name": "{{ namespace }}/myproject5", + "private": false, + "owner": { + "login": "someuser", + "id": 1, + "node_id": "MDQ6VXNlcjM3NDg2OTY=", + "avatar_url": "https://example.com/u/3748696?v=4", + "gravatar_id": "", + "url": "https://api.example.com/users/{{ namespace }}", + "html_url": "https://example.com/{{ namespace }}", + "followers_url": "https://api.example.com/users/{{ namespace }}/followers", + "following_url": "https://api.example.com/users/{{ namespace }}/following{/other_user}", + "gists_url": "https://api.example.com/users/{{ namespace }}/gists{/gist_id}", + "starred_url": "https://api.example.com/users/{{ namespace }}/starred{/owner}{/repo}", + "subscriptions_url": "https://api.example.com/users/{{ namespace }}/subscriptions", + "organizations_url": "https://api.example.com/users/{{ namespace }}/orgs", + "repos_url": "https://api.example.com/users/{{ namespace }}/repos", + "events_url": "https://api.example.com/users/{{ namespace }}/events{/privacy}", + "received_events_url": "https://api.example.com/users/{{ namespace }}/received_events", + "type": "User", + "site_admin": false + }, + "html_url": "https://example.com/{{ namespace }}/myproject5", + "description": "Shell script for automatically building ACI containers from scratch using acbuild.", + "fork": false, + "url": "https://api.example.com/repos/{{ namespace }}/myproject5", + "forks_url": "https://api.example.com/repos/{{ namespace }}/myproject5/forks", + "keys_url": "https://api.example.com/repos/{{ namespace }}/myproject5/keys{/key_id}", + "collaborators_url": "https://api.example.com/repos/{{ namespace }}/myproject5/collaborators{/collaborator}", + "teams_url": "https://api.example.com/repos/{{ namespace }}/myproject5/teams", + "hooks_url": "https://api.example.com/repos/{{ namespace }}/myproject5/hooks", + "issue_events_url": "https://api.example.com/repos/{{ namespace }}/myproject5/issues/events{/number}", + "events_url": "https://api.example.com/repos/{{ namespace }}/myproject5/events", + "assignees_url": "https://api.example.com/repos/{{ namespace }}/myproject5/assignees{/user}", + "branches_url": "https://api.example.com/repos/{{ namespace }}/myproject5/branches{/branch}", + "tags_url": "https://api.example.com/repos/{{ namespace }}/myproject5/tags", + "blobs_url": "https://api.example.com/repos/{{ namespace }}/myproject5/git/blobs{/sha}", + "git_tags_url": "https://api.example.com/repos/{{ namespace }}/myproject5/git/tags{/sha}", + "git_refs_url": "https://api.example.com/repos/{{ namespace }}/myproject5/git/refs{/sha}", + "trees_url": "https://api.example.com/repos/{{ namespace }}/myproject5/git/trees{/sha}", + "statuses_url": "https://api.example.com/repos/{{ namespace }}/myproject5/statuses/{sha}", + "languages_url": "https://api.example.com/repos/{{ namespace }}/myproject5/languages", + "stargazers_url": "https://api.example.com/repos/{{ namespace }}/myproject5/stargazers", + "contributors_url": "https://api.example.com/repos/{{ namespace }}/myproject5/contributors", + "subscribers_url": "https://api.example.com/repos/{{ namespace }}/myproject5/subscribers", + "subscription_url": "https://api.example.com/repos/{{ namespace }}/myproject5/subscription", + "commits_url": "https://api.example.com/repos/{{ namespace }}/myproject5/commits{/sha}", + "git_commits_url": "https://api.example.com/repos/{{ namespace }}/myproject5/git/commits{/sha}", + "comments_url": "https://api.example.com/repos/{{ namespace }}/myproject5/comments{/number}", + "issue_comment_url": "https://api.example.com/repos/{{ namespace }}/myproject5/issues/comments{/number}", + "contents_url": "https://api.example.com/repos/{{ namespace }}/myproject5/contents/{+path}", + "compare_url": "https://api.example.com/repos/{{ namespace }}/myproject5/compare/{base}...{head}", + "merges_url": "https://api.example.com/repos/{{ namespace }}/myproject5/merges", + "archive_url": "https://api.example.com/repos/{{ namespace }}/myproject5/{archive_format}{/ref}", + "downloads_url": "https://api.example.com/repos/{{ namespace }}/myproject5/downloads", + "issues_url": "https://api.example.com/repos/{{ namespace }}/myproject5/issues{/number}", + "pulls_url": "https://api.example.com/repos/{{ namespace }}/myproject5/pulls{/number}", + "milestones_url": "https://api.example.com/repos/{{ namespace }}/myproject5/milestones{/number}", + "notifications_url": "https://api.example.com/repos/{{ namespace }}/myproject5/notifications{?since,all,participating}", + "labels_url": "https://api.example.com/repos/{{ namespace }}/myproject5/labels{/name}", + "releases_url": "https://api.example.com/repos/{{ namespace }}/myproject5/releases{/id}", + "deployments_url": "https://api.example.com/repos/{{ namespace }}/myproject5/deployments", + "created_at": "2016-01-07T22:27:54Z", + "updated_at": "2021-11-20T16:15:37Z", + "pushed_at": "2021-11-20T16:15:34Z", + "git_url": "git://example.com/{{ namespace }}/myproject5.git", + "ssh_url": "ssh://git@example.com/{{ namespace }}/myproject5.git", + "clone_url": "https://example.com/{{ namespace }}/myproject5.git", + "svn_url": "https://example.com/{{ namespace }}/myproject5", + "homepage": null, + "size": 12, + "stargazers_count": 0, + "watchers_count": 0, + "language": "Shell", + "has_issues": true, + "has_projects": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": false, + "forks_count": 0, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 0, + "license": { + "key": "apache-2.0", + "name": "Apache License 2.0", + "spdx_id": "Apache-2.0", + "url": "https://api.example.com/licenses/apache-2.0", + "node_id": "MDc6TGljZW5zZTI=" + }, + "allow_forking": true, + "is_template": false, + "topics": [ + + ], + "visibility": "public", + "forks": 0, + "open_issues": 0, + "watchers": 0, + "default_branch": "master", + "permissions": { + "admin": true, + "maintain": true, + "push": true, + "triage": true, + "pull": true + } + } +] diff --git a/e2e_tests/docker-rest/flask/github_api_user.json b/e2e_tests/docker-rest/flask/github_api_user.json new file mode 100644 index 0000000..03df406 --- /dev/null +++ b/e2e_tests/docker-rest/flask/github_api_user.json @@ -0,0 +1,46 @@ +{ + "login": "myuser1", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://example.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.example.com/users/octocat", + "html_url": "https://example.com/octocat", + "followers_url": "https://api.example.com/users/octocat/followers", + "following_url": "https://api.example.com/users/octocat/following{/other_user}", + "gists_url": "https://api.example.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.example.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.example.com/users/octocat/subscriptions", + "organizations_url": "https://api.example.com/users/octocat/orgs", + "repos_url": "https://api.example.com/users/octocat/repos", + "events_url": "https://api.example.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.example.com/users/octocat/received_events", + "type": "User", + "site_admin": false, + "name": "monalisa octocat", + "company": "GitHub", + "blog": "https://example.com/blog", + "location": "San Francisco", + "email": "octocat@example.com", + "hireable": false, + "bio": "There once was...", + "twitter_username": "monatheoctocat", + "public_repos": 2, + "public_gists": 1, + "followers": 20, + "following": 0, + "created_at": "2008-01-14T04:33:35Z", + "updated_at": "2008-01-14T04:33:35Z", + "private_gists": 81, + "total_private_repos": 100, + "owned_private_repos": 100, + "disk_usage": 10000, + "collaborators": 8, + "two_factor_authentication": true, + "plan": { + "name": "Medium", + "space": 400, + "private_repos": 20, + "collaborators": 0 + } +} diff --git a/e2e_tests/docker-rest/flask/gitlab.py b/e2e_tests/docker-rest/flask/gitlab.py new file mode 100644 index 0000000..12f070c --- /dev/null +++ b/e2e_tests/docker-rest/flask/gitlab.py @@ -0,0 +1,106 @@ +import os.path + +from app import app + +from flask import Flask, request, abort, jsonify, make_response + +import jinja2 + + +def check_headers(): + if request.headers.get("accept") != "application/json": + app.logger.error("Invalid accept header") + abort(500) + auth_header = request.headers.get("authorization") + if auth_header != "bearer authtoken": + app.logger.error("Invalid authorization header: %s", auth_header) + abort( + make_response( + jsonify( + { + "message": "Bad credentials", + "documentation_url": "https://docs.example.com/rest", + } + ), + 401, + ) + ) + + +def add_pagination(response, page, last_page): + host = request.headers["host"] + link_header = "" + + def args(page): + args = request.args.copy() + args["page"] = page + return "&".join([f"{k}={v}" for k, v in args.items()]) + + if page < last_page: + link_header += ( + f'<{request.scheme}://{host}{request.path}?{args(page+1)}>; rel="next", ' + ) + link_header += ( + f'<{request.scheme}://{host}{request.path}?{args(last_page)}>; rel="last"' + ) + response.headers["link"] = link_header + + +def read_project_files(namespaces=[]): + last_page = 4 + page = username = int(request.args.get("page", "1")) + response_file = f"./gitlab_api_page_{page}.json" + if not os.path.exists(response_file): + return jsonify([]) + + response = make_response( + jinja2.Template(open(response_file).read()).render( + namespace=namespaces[page - 1] + ) + ) + add_pagination(response, page, last_page) + response.headers["content-type"] = "application/json" + return response + + +def single_namespaced_projects(namespace): + return read_project_files([namespace] * 4) + + +def mixed_projects(namespaces): + return read_project_files(namespaces) + + +@app.route("/gitlab/api/v4/users//projects") +def gitlab_user_repos(user): + check_headers() + if user == "myuser1": + return single_namespaced_projects("myuser1") + return jsonify([]) + + +@app.route("/gitlab/api/v4/groups//projects") +def gitlab_group_repos(group): + check_headers() + if not ( + request.args.get("include_subgroups") == "true" + and request.args.get("archived") == "false" + ): + abort(500, "wrong arguments") + if group == "mygroup1": + return single_namespaced_projects("mygroup1") + return jsonify([]) + + +@app.route("/gitlab/api/v4/projects/") +def gitlab_own_repos(): + check_headers() + return mixed_projects(["myuser1", "myuser2", "mygroup1", "mygroup2"]) + + +@app.route("/gitlab/api/v4/user/") +def gitlab_user(): + check_headers() + response = make_response(open("./gitlab_api_user.json").read()) + response.headers["content-type"] = "application/json" + return response diff --git a/e2e_tests/docker-rest/flask/gitlab_api_page_1.json b/e2e_tests/docker-rest/flask/gitlab_api_page_1.json new file mode 100644 index 0000000..1ea538f --- /dev/null +++ b/e2e_tests/docker-rest/flask/gitlab_api_page_1.json @@ -0,0 +1,236 @@ +[ + { + "id": 1, + "description": "", + "name": "myproject1", + "name_with_namespace": "{{ namespace }} / myproject1", + "path": "myproject1", + "path_with_namespace": "{{ namespace }}/myproject1", + "created_at": "2020-11-26T17:23:39.904Z", + "default_branch": "master", + "tag_list": [], + "topics": [], + "ssh_url_to_repo": "ssh://git@example.com/{{ namespace }}/myproject1.git", + "http_url_to_repo": "https://example.com/{{ namespace }}/myproject1.git", + "web_url": "https://example.com/{{ namespace }}/myproject1", + "readme_url": null, + "avatar_url": null, + "forks_count": 0, + "star_count": 0, + "last_activity_at": "2020-11-26T17:23:39.904Z", + "namespace": { + "id": 3, + "name": "{{ namespace }}", + "path": "{{ namespace }}", + "kind": "group", + "full_path": "{{ namespace }}", + "parent_id": null, + "avatar_url": "/uploads/-/system/group/avatar/5/x.png", + "web_url": "https://example.com/groups/{{ namespace }}" + }, + "container_registry_image_prefix": "registry.example.com/{{ namespace }}/myproject1", + "_links": { + "self": "https://example.com/api/v4/projects/2", + "issues": "https://example.com/api/v4/projects/2/issues", + "merge_requests": "https://example.com/api/v4/projects/2/merge_requests", + "repo_branches": "https://example.com/api/v4/projects/2/repository/branches", + "labels": "https://example.com/api/v4/projects/2/labels", + "events": "https://example.com/api/v4/projects/2/events", + "members": "https://example.com/api/v4/projects/2/members", + "cluster_agents": "https://example.com/api/v4/projects/2/cluster_agents" + }, + "packages_enabled": true, + "empty_repo": false, + "archived": false, + "visibility": "private", + "resolve_outdated_diff_discussions": false, + "container_expiration_policy": { + "cadence": "1d", + "enabled": false, + "keep_n": 10, + "older_than": "90d", + "name_regex": ".*", + "name_regex_keep": null, + "next_run_at": "2020-11-27T17:23:39.927Z" + }, + "issues_enabled": true, + "merge_requests_enabled": true, + "wiki_enabled": true, + "jobs_enabled": true, + "snippets_enabled": true, + "container_registry_enabled": true, + "service_desk_enabled": true, + "service_desk_address": "contact-for-myproject1-2-issue-@incoming.example.com", + "can_create_merge_request_in": true, + "issues_access_level": "enabled", + "repository_access_level": "enabled", + "merge_requests_access_level": "enabled", + "forking_access_level": "enabled", + "wiki_access_level": "enabled", + "builds_access_level": "enabled", + "snippets_access_level": "enabled", + "pages_access_level": "private", + "operations_access_level": "enabled", + "analytics_access_level": "enabled", + "container_registry_access_level": "enabled", + "security_and_compliance_access_level": "private", + "emails_disabled": null, + "shared_runners_enabled": true, + "lfs_enabled": true, + "creator_id": 1803951, + "import_url": null, + "import_type": null, + "import_status": "none", + "open_issues_count": 0, + "ci_default_git_depth": 50, + "ci_forward_deployment_enabled": true, + "ci_job_token_scope_enabled": false, + "ci_separated_caches": true, + "public_jobs": true, + "build_timeout": 3600, + "auto_cancel_pending_pipelines": "enabled", + "build_coverage_regex": null, + "ci_config_path": "", + "shared_with_groups": [], + "only_allow_merge_if_pipeline_succeeds": false, + "allow_merge_on_skipped_pipeline": null, + "restrict_user_defined_variables": false, + "request_access_enabled": true, + "only_allow_merge_if_all_discussions_are_resolved": false, + "remove_source_branch_after_merge": true, + "printing_merge_request_link_enabled": true, + "merge_method": "merge", + "squash_option": "default_off", + "enforce_auth_checks_on_uploads": true, + "suggestion_commit_message": null, + "merge_commit_template": null, + "squash_commit_template": null, + "auto_devops_enabled": false, + "auto_devops_deploy_strategy": "continuous", + "autoclose_referenced_issues": true, + "keep_latest_artifact": true, + "runner_token_expiration_interval": null, + "external_authorization_classification_label": "", + "requirements_enabled": false, + "requirements_access_level": "enabled", + "security_and_compliance_enabled": true, + "compliance_frameworks": [] + }, + { + "id": 2, + "description": "", + "name": "myproject2", + "name_with_namespace": "{{ namespace }} / myproject2", + "path": "myproject2", + "path_with_namespace": "{{ namespace }}/myproject2", + "created_at": "2020-11-26T17:23:39.904Z", + "default_branch": "master", + "tag_list": [], + "topics": [], + "ssh_url_to_repo": "ssh://git@example.com/{{ namespace }}/myproject2.git", + "http_url_to_repo": "https://example.com/{{ namespace }}/myproject2.git", + "web_url": "https://example.com/{{ namespace }}/myproject2", + "readme_url": null, + "avatar_url": null, + "forks_count": 0, + "star_count": 0, + "last_activity_at": "2020-11-26T17:23:39.904Z", + "namespace": { + "id": 3, + "name": "{{ namespace }}", + "path": "{{ namespace }}", + "kind": "group", + "full_path": "{{ namespace }}", + "parent_id": null, + "avatar_url": "/uploads/-/system/group/avatar/5/x.png", + "web_url": "https://example.com/groups/{{ namespace }}" + }, + "container_registry_image_prefix": "registry.example.com/{{ namespace }}/myproject2", + "_links": { + "self": "https://example.com/api/v4/projects/2", + "issues": "https://example.com/api/v4/projects/2/issues", + "merge_requests": "https://example.com/api/v4/projects/2/merge_requests", + "repo_branches": "https://example.com/api/v4/projects/2/repository/branches", + "labels": "https://example.com/api/v4/projects/2/labels", + "events": "https://example.com/api/v4/projects/2/events", + "members": "https://example.com/api/v4/projects/2/members", + "cluster_agents": "https://example.com/api/v4/projects/2/cluster_agents" + }, + "packages_enabled": true, + "empty_repo": false, + "archived": false, + "visibility": "public", + "resolve_outdated_diff_discussions": false, + "container_expiration_policy": { + "cadence": "1d", + "enabled": false, + "keep_n": 10, + "older_than": "90d", + "name_regex": ".*", + "name_regex_keep": null, + "next_run_at": "2020-11-27T17:23:39.927Z" + }, + "issues_enabled": true, + "merge_requests_enabled": true, + "wiki_enabled": true, + "jobs_enabled": true, + "snippets_enabled": true, + "container_registry_enabled": true, + "service_desk_enabled": true, + "service_desk_address": "contact-for-myproject2-2-issue-@incoming.example.com", + "can_create_merge_request_in": true, + "issues_access_level": "enabled", + "repository_access_level": "enabled", + "merge_requests_access_level": "enabled", + "forking_access_level": "enabled", + "wiki_access_level": "enabled", + "builds_access_level": "enabled", + "snippets_access_level": "enabled", + "pages_access_level": "private", + "operations_access_level": "enabled", + "analytics_access_level": "enabled", + "container_registry_access_level": "enabled", + "security_and_compliance_access_level": "private", + "emails_disabled": null, + "shared_runners_enabled": true, + "lfs_enabled": true, + "creator_id": 1803951, + "import_url": null, + "import_type": null, + "import_status": "none", + "open_issues_count": 0, + "ci_default_git_depth": 50, + "ci_forward_deployment_enabled": true, + "ci_job_token_scope_enabled": false, + "ci_separated_caches": true, + "public_jobs": true, + "build_timeout": 3600, + "auto_cancel_pending_pipelines": "enabled", + "build_coverage_regex": null, + "ci_config_path": "", + "shared_with_groups": [], + "only_allow_merge_if_pipeline_succeeds": false, + "allow_merge_on_skipped_pipeline": null, + "restrict_user_defined_variables": false, + "request_access_enabled": true, + "only_allow_merge_if_all_discussions_are_resolved": false, + "remove_source_branch_after_merge": true, + "printing_merge_request_link_enabled": true, + "merge_method": "merge", + "squash_option": "default_off", + "enforce_auth_checks_on_uploads": true, + "suggestion_commit_message": null, + "merge_commit_template": null, + "squash_commit_template": null, + "auto_devops_enabled": false, + "auto_devops_deploy_strategy": "continuous", + "autoclose_referenced_issues": true, + "keep_latest_artifact": true, + "runner_token_expiration_interval": null, + "external_authorization_classification_label": "", + "requirements_enabled": false, + "requirements_access_level": "enabled", + "security_and_compliance_enabled": true, + "compliance_frameworks": [] + } +] diff --git a/e2e_tests/docker-rest/flask/gitlab_api_page_2.json b/e2e_tests/docker-rest/flask/gitlab_api_page_2.json new file mode 100644 index 0000000..8964151 --- /dev/null +++ b/e2e_tests/docker-rest/flask/gitlab_api_page_2.json @@ -0,0 +1,119 @@ +[ + { + "id": 3, + "description": "", + "name": "myproject3", + "name_with_namespace": "{{ namespace }} / myproject3", + "path": "myproject3", + "path_with_namespace": "{{ namespace }}/myproject3", + "created_at": "2020-11-26T17:23:39.904Z", + "default_branch": "master", + "tag_list": [], + "topics": [], + "ssh_url_to_repo": "ssh://git@example.com/{{ namespace }}/myproject3.git", + "http_url_to_repo": "https://example.com/{{ namespace }}/myproject3.git", + "web_url": "https://example.com/{{ namespace }}/myproject3", + "readme_url": null, + "avatar_url": null, + "forks_count": 0, + "star_count": 0, + "last_activity_at": "2020-11-26T17:23:39.904Z", + "namespace": { + "id": 3, + "name": "{{ namespace }}", + "path": "{{ namespace }}", + "kind": "group", + "full_path": "{{ namespace }}", + "parent_id": null, + "avatar_url": "/uploads/-/system/group/avatar/5/x.png", + "web_url": "https://example.com/groups/{{ namespace }}" + }, + "container_registry_image_prefix": "registry.example.com/{{ namespace }}/myproject3", + "_links": { + "self": "https://example.com/api/v4/projects/2", + "issues": "https://example.com/api/v4/projects/2/issues", + "merge_requests": "https://example.com/api/v4/projects/2/merge_requests", + "repo_branches": "https://example.com/api/v4/projects/2/repository/branches", + "labels": "https://example.com/api/v4/projects/2/labels", + "events": "https://example.com/api/v4/projects/2/events", + "members": "https://example.com/api/v4/projects/2/members", + "cluster_agents": "https://example.com/api/v4/projects/2/cluster_agents" + }, + "packages_enabled": true, + "empty_repo": false, + "archived": false, + "visibility": "public", + "resolve_outdated_diff_discussions": false, + "container_expiration_policy": { + "cadence": "1d", + "enabled": false, + "keep_n": 10, + "older_than": "90d", + "name_regex": ".*", + "name_regex_keep": null, + "next_run_at": "2020-11-27T17:23:39.927Z" + }, + "issues_enabled": true, + "merge_requests_enabled": true, + "wiki_enabled": true, + "jobs_enabled": true, + "snippets_enabled": true, + "container_registry_enabled": true, + "service_desk_enabled": true, + "service_desk_address": "contact-for-myproject3-2-issue-@incoming.example.com", + "can_create_merge_request_in": true, + "issues_access_level": "enabled", + "repository_access_level": "enabled", + "merge_requests_access_level": "enabled", + "forking_access_level": "enabled", + "wiki_access_level": "enabled", + "builds_access_level": "enabled", + "snippets_access_level": "enabled", + "pages_access_level": "private", + "operations_access_level": "enabled", + "analytics_access_level": "enabled", + "container_registry_access_level": "enabled", + "security_and_compliance_access_level": "private", + "emails_disabled": null, + "shared_runners_enabled": true, + "lfs_enabled": true, + "creator_id": 1803951, + "import_url": null, + "import_type": null, + "import_status": "none", + "open_issues_count": 0, + "ci_default_git_depth": 50, + "ci_forward_deployment_enabled": true, + "ci_job_token_scope_enabled": false, + "ci_separated_caches": true, + "public_jobs": true, + "build_timeout": 3600, + "auto_cancel_pending_pipelines": "enabled", + "build_coverage_regex": null, + "ci_config_path": "", + "shared_with_groups": [], + "only_allow_merge_if_pipeline_succeeds": false, + "allow_merge_on_skipped_pipeline": null, + "restrict_user_defined_variables": false, + "request_access_enabled": true, + "only_allow_merge_if_all_discussions_are_resolved": false, + "remove_source_branch_after_merge": true, + "printing_merge_request_link_enabled": true, + "merge_method": "merge", + "squash_option": "default_off", + "enforce_auth_checks_on_uploads": true, + "suggestion_commit_message": null, + "merge_commit_template": null, + "squash_commit_template": null, + "auto_devops_enabled": false, + "auto_devops_deploy_strategy": "continuous", + "autoclose_referenced_issues": true, + "keep_latest_artifact": true, + "runner_token_expiration_interval": null, + "external_authorization_classification_label": "", + "requirements_enabled": false, + "requirements_access_level": "enabled", + "security_and_compliance_enabled": true, + "compliance_frameworks": [] + } +] diff --git a/e2e_tests/docker-rest/flask/gitlab_api_page_3.json b/e2e_tests/docker-rest/flask/gitlab_api_page_3.json new file mode 100644 index 0000000..33e36be --- /dev/null +++ b/e2e_tests/docker-rest/flask/gitlab_api_page_3.json @@ -0,0 +1,119 @@ +[ + { + "id": 4, + "description": "", + "name": "myproject4", + "name_with_namespace": "{{ namespace }} / myproject4", + "path": "myproject4", + "path_with_namespace": "{{ namespace }}/myproject4", + "created_at": "2020-11-26T17:23:39.904Z", + "default_branch": "master", + "tag_list": [], + "topics": [], + "ssh_url_to_repo": "ssh://git@example.com/{{ namespace }}/myproject4.git", + "http_url_to_repo": "https://example.com/{{ namespace }}/myproject4.git", + "web_url": "https://example.com/{{ namespace }}/myproject4", + "readme_url": null, + "avatar_url": null, + "forks_count": 0, + "star_count": 0, + "last_activity_at": "2020-11-26T17:23:39.904Z", + "namespace": { + "id": 3, + "name": "{{ namespace }}", + "path": "{{ namespace }}", + "kind": "group", + "full_path": "{{ namespace }}", + "parent_id": null, + "avatar_url": "/uploads/-/system/group/avatar/5/x.png", + "web_url": "https://example.com/groups/{{ namespace }}" + }, + "container_registry_image_prefix": "registry.example.com/{{ namespace }}/myproject4", + "_links": { + "self": "https://example.com/api/v4/projects/2", + "issues": "https://example.com/api/v4/projects/2/issues", + "merge_requests": "https://example.com/api/v4/projects/2/merge_requests", + "repo_branches": "https://example.com/api/v4/projects/2/repository/branches", + "labels": "https://example.com/api/v4/projects/2/labels", + "events": "https://example.com/api/v4/projects/2/events", + "members": "https://example.com/api/v4/projects/2/members", + "cluster_agents": "https://example.com/api/v4/projects/2/cluster_agents" + }, + "packages_enabled": true, + "empty_repo": false, + "archived": false, + "visibility": "public", + "resolve_outdated_diff_discussions": false, + "container_expiration_policy": { + "cadence": "1d", + "enabled": false, + "keep_n": 10, + "older_than": "90d", + "name_regex": ".*", + "name_regex_keep": null, + "next_run_at": "2020-11-27T17:23:39.927Z" + }, + "issues_enabled": true, + "merge_requests_enabled": true, + "wiki_enabled": true, + "jobs_enabled": true, + "snippets_enabled": true, + "container_registry_enabled": true, + "service_desk_enabled": true, + "service_desk_address": "contact-for-myproject4-2-issue-@incoming.example.com", + "can_create_merge_request_in": true, + "issues_access_level": "enabled", + "repository_access_level": "enabled", + "merge_requests_access_level": "enabled", + "forking_access_level": "enabled", + "wiki_access_level": "enabled", + "builds_access_level": "enabled", + "snippets_access_level": "enabled", + "pages_access_level": "private", + "operations_access_level": "enabled", + "analytics_access_level": "enabled", + "container_registry_access_level": "enabled", + "security_and_compliance_access_level": "private", + "emails_disabled": null, + "shared_runners_enabled": true, + "lfs_enabled": true, + "creator_id": 1803951, + "import_url": null, + "import_type": null, + "import_status": "none", + "open_issues_count": 0, + "ci_default_git_depth": 50, + "ci_forward_deployment_enabled": true, + "ci_job_token_scope_enabled": false, + "ci_separated_caches": true, + "public_jobs": true, + "build_timeout": 3600, + "auto_cancel_pending_pipelines": "enabled", + "build_coverage_regex": null, + "ci_config_path": "", + "shared_with_groups": [], + "only_allow_merge_if_pipeline_succeeds": false, + "allow_merge_on_skipped_pipeline": null, + "restrict_user_defined_variables": false, + "request_access_enabled": true, + "only_allow_merge_if_all_discussions_are_resolved": false, + "remove_source_branch_after_merge": true, + "printing_merge_request_link_enabled": true, + "merge_method": "merge", + "squash_option": "default_off", + "enforce_auth_checks_on_uploads": true, + "suggestion_commit_message": null, + "merge_commit_template": null, + "squash_commit_template": null, + "auto_devops_enabled": false, + "auto_devops_deploy_strategy": "continuous", + "autoclose_referenced_issues": true, + "keep_latest_artifact": true, + "runner_token_expiration_interval": null, + "external_authorization_classification_label": "", + "requirements_enabled": false, + "requirements_access_level": "enabled", + "security_and_compliance_enabled": true, + "compliance_frameworks": [] + } +] diff --git a/e2e_tests/docker-rest/flask/gitlab_api_page_4.json b/e2e_tests/docker-rest/flask/gitlab_api_page_4.json new file mode 100644 index 0000000..dc7e17c --- /dev/null +++ b/e2e_tests/docker-rest/flask/gitlab_api_page_4.json @@ -0,0 +1,119 @@ +[ + { + "id": 5, + "description": "", + "name": "myproject5", + "name_with_namespace": "{{ namespace }} / myproject5", + "path": "myproject5", + "path_with_namespace": "{{ namespace }}/myproject5", + "created_at": "2020-11-26T17:23:39.904Z", + "default_branch": "master", + "tag_list": [], + "topics": [], + "ssh_url_to_repo": "ssh://git@example.com/{{ namespace }}/myproject5.git", + "http_url_to_repo": "https://example.com/{{ namespace }}/myproject5.git", + "web_url": "https://example.com/{{ namespace }}/myproject5", + "readme_url": null, + "avatar_url": null, + "forks_count": 0, + "star_count": 0, + "last_activity_at": "2020-11-26T17:23:39.904Z", + "namespace": { + "id": 3, + "name": "{{ namespace }}", + "path": "{{ namespace }}", + "kind": "group", + "full_path": "{{ namespace }}", + "parent_id": null, + "avatar_url": "/uploads/-/system/group/avatar/5/x.png", + "web_url": "https://example.com/groups/{{ namespace }}" + }, + "container_registry_image_prefix": "registry.example.com/{{ namespace }}/myproject5", + "_links": { + "self": "https://example.com/api/v4/projects/2", + "issues": "https://example.com/api/v4/projects/2/issues", + "merge_requests": "https://example.com/api/v4/projects/2/merge_requests", + "repo_branches": "https://example.com/api/v4/projects/2/repository/branches", + "labels": "https://example.com/api/v4/projects/2/labels", + "events": "https://example.com/api/v4/projects/2/events", + "members": "https://example.com/api/v4/projects/2/members", + "cluster_agents": "https://example.com/api/v4/projects/2/cluster_agents" + }, + "packages_enabled": true, + "empty_repo": false, + "archived": false, + "visibility": "public", + "resolve_outdated_diff_discussions": false, + "container_expiration_policy": { + "cadence": "1d", + "enabled": false, + "keep_n": 10, + "older_than": "90d", + "name_regex": ".*", + "name_regex_keep": null, + "next_run_at": "2020-11-27T17:23:39.927Z" + }, + "issues_enabled": true, + "merge_requests_enabled": true, + "wiki_enabled": true, + "jobs_enabled": true, + "snippets_enabled": true, + "container_registry_enabled": true, + "service_desk_enabled": true, + "service_desk_address": "contact-for-myproject5-2-issue-@incoming.example.com", + "can_create_merge_request_in": true, + "issues_access_level": "enabled", + "repository_access_level": "enabled", + "merge_requests_access_level": "enabled", + "forking_access_level": "enabled", + "wiki_access_level": "enabled", + "builds_access_level": "enabled", + "snippets_access_level": "enabled", + "pages_access_level": "private", + "operations_access_level": "enabled", + "analytics_access_level": "enabled", + "container_registry_access_level": "enabled", + "security_and_compliance_access_level": "private", + "emails_disabled": null, + "shared_runners_enabled": true, + "lfs_enabled": true, + "creator_id": 1803951, + "import_url": null, + "import_type": null, + "import_status": "none", + "open_issues_count": 0, + "ci_default_git_depth": 50, + "ci_forward_deployment_enabled": true, + "ci_job_token_scope_enabled": false, + "ci_separated_caches": true, + "public_jobs": true, + "build_timeout": 3600, + "auto_cancel_pending_pipelines": "enabled", + "build_coverage_regex": null, + "ci_config_path": "", + "shared_with_groups": [], + "only_allow_merge_if_pipeline_succeeds": false, + "allow_merge_on_skipped_pipeline": null, + "restrict_user_defined_variables": false, + "request_access_enabled": true, + "only_allow_merge_if_all_discussions_are_resolved": false, + "remove_source_branch_after_merge": true, + "printing_merge_request_link_enabled": true, + "merge_method": "merge", + "squash_option": "default_off", + "enforce_auth_checks_on_uploads": true, + "suggestion_commit_message": null, + "merge_commit_template": null, + "squash_commit_template": null, + "auto_devops_enabled": false, + "auto_devops_deploy_strategy": "continuous", + "autoclose_referenced_issues": true, + "keep_latest_artifact": true, + "runner_token_expiration_interval": null, + "external_authorization_classification_label": "", + "requirements_enabled": false, + "requirements_access_level": "enabled", + "security_and_compliance_enabled": true, + "compliance_frameworks": [] + } +] diff --git a/e2e_tests/docker-rest/flask/gitlab_api_user.json b/e2e_tests/docker-rest/flask/gitlab_api_user.json new file mode 100644 index 0000000..b09e810 --- /dev/null +++ b/e2e_tests/docker-rest/flask/gitlab_api_user.json @@ -0,0 +1,42 @@ +{ + "id": 1, + "username": "myuser1", + "name": "My User", + "state": "active", + "avatar_url": "https://example.com/avatar", + "web_url": "https://example.com/myuser1", + "created_at": "2016-12-10T10:09:11.585Z", + "bio": "", + "location": "", + "public_email": "", + "skype": "", + "linkedin": "", + "twitter": "", + "website_url": "", + "organization": "", + "job_title": "", + "pronouns": "", + "bot": false, + "work_information": null, + "followers": 0, + "following": 0, + "is_followed": false, + "local_time": "11:59 PM", + "last_sign_in_at": "2020-03-14T09:13:44.977Z", + "confirmed_at": "2022-05-19T23:48:47.033Z", + "last_activity_on": "2022-05-19", + "email": "myuser1@example.com", + "theme_id": null, + "color_scheme_id": 1, + "projects_limit": 100000, + "current_sign_in_at": "2022-05-19T23:45:49.661Z", + "identities": [], + "can_create_group": true, + "can_create_project": true, + "two_factor_enabled": false, + "external": false, + "private_profile": false, + "commit_email": "myuser1@example.com", + "shared_runners_minutes_limit": 2000, + "extra_shared_runners_minutes_limit": null +} diff --git a/e2e_tests/docker/Dockerfile b/e2e_tests/docker/Dockerfile new file mode 100644 index 0000000..b6e75ce --- /dev/null +++ b/e2e_tests/docker/Dockerfile @@ -0,0 +1,14 @@ +FROM docker.io/debian:11.3 + +RUN apt-get update \ + && apt-get install -y \ + python3-pytest \ + python3-toml \ + python3-git \ + python3-yaml \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /tests + +ENTRYPOINT ["/bin/sh", "-c", "--"] diff --git a/e2e_tests/helpers.py b/e2e_tests/helpers.py index a5b837f..c66289b 100644 --- a/e2e_tests/helpers.py +++ b/e2e_tests/helpers.py @@ -8,9 +8,7 @@ import hashlib import git -binary = os.path.join( - os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "target/release/grm" -) +binary = os.environ["GRM_BINARY"] def grm(args, cwd=None, is_invalid=False): @@ -208,7 +206,7 @@ class RepoTree: """ ) - cmd = grm(["repos", "sync", "--config", self.config.name]) + cmd = grm(["repos", "sync", "config", "--config", self.config.name]) assert cmd.returncode == 0 return (self.root.name, self.config.name, ["test", "test_worktree"]) diff --git a/e2e_tests/requirements.txt b/e2e_tests/requirements.txt deleted file mode 100644 index 6f444a8..0000000 --- a/e2e_tests/requirements.txt +++ /dev/null @@ -1,14 +0,0 @@ -attrs==21.4.0 -gitdb==4.0.9 -GitPython==3.1.27 -iniconfig==1.1.1 -packaging==21.3 -pluggy==1.0.0 -py==1.11.0 -pyparsing==3.0.8 -pytest==7.1.2 -PyYAML==6.0 -smmap==5.0.0 -toml==0.10.2 -tomli==2.0.1 -typing_extensions==4.2.0 diff --git a/e2e_tests/test_repos_find.py b/e2e_tests/test_repos_find.py index cf6e692..9f97c6d 100644 --- a/e2e_tests/test_repos_find.py +++ b/e2e_tests/test_repos_find.py @@ -11,7 +11,7 @@ from helpers import * def test_repos_find_nonexistent(): with NonExistentPath() as nonexistent_dir: - cmd = grm(["repos", "find", nonexistent_dir]) + cmd = grm(["repos", "find", "local", nonexistent_dir]) assert "does not exist" in cmd.stderr.lower() assert cmd.returncode != 0 assert not os.path.exists(nonexistent_dir) @@ -19,14 +19,14 @@ def test_repos_find_nonexistent(): def test_repos_find_file(): with tempfile.NamedTemporaryFile() as tmpfile: - cmd = grm(["repos", "find", tmpfile.name]) + cmd = grm(["repos", "find", "local", tmpfile.name]) assert "not a directory" in cmd.stderr.lower() assert cmd.returncode != 0 def test_repos_find_empty(): with tempfile.TemporaryDirectory() as tmpdir: - cmd = grm(["repos", "find", tmpdir]) + cmd = grm(["repos", "find", "local", tmpdir]) assert cmd.returncode == 0 assert len(cmd.stdout) == 0 assert len(cmd.stderr) != 0 @@ -35,7 +35,8 @@ def test_repos_find_empty(): def test_repos_find_invalid_format(): with tempfile.TemporaryDirectory() as tmpdir: cmd = grm( - ["repos", "find", tmpdir, "--format", "invalidformat"], is_invalid=True + ["repos", "find", "local", tmpdir, "--format", "invalidformat"], + is_invalid=True, ) assert cmd.returncode != 0 assert len(cmd.stdout) == 0 @@ -55,7 +56,7 @@ def test_repos_find_non_git_repos(): """ ) - cmd = grm(["repos", "find", tmpdir]) + cmd = grm(["repos", "find", "local", tmpdir]) assert cmd.returncode == 0 assert len(cmd.stdout) == 0 @@ -83,7 +84,7 @@ def test_repos_find(configtype, default): ( cd ./repo2 git init - git co -b main + git checkout -b main echo test > test git add test git commit -m "commit1" @@ -97,7 +98,7 @@ def test_repos_find(configtype, default): """ ) - args = ["repos", "find", tmpdir] + args = ["repos", "find", "local", tmpdir] if not default: args += ["--format", configtype] cmd = grm(args) @@ -152,7 +153,7 @@ def test_repos_find(configtype, default): def test_repos_find_in_root(configtype, default): with TempGitRepository() as repo_dir: - args = ["repos", "find", repo_dir] + args = ["repos", "find", "local", repo_dir] if not default: args += ["--format", configtype] cmd = grm(args) @@ -213,7 +214,7 @@ def test_repos_find_with_invalid_repo(configtype, default): ( cd ./repo2 git init - git co -b main + git checkout -b main echo test > test git add test git commit -m "commit1" @@ -227,7 +228,7 @@ def test_repos_find_with_invalid_repo(configtype, default): """ ) - args = ["repos", "find", tmpdir] + args = ["repos", "find", "local", tmpdir] if not default: args += ["--format", configtype] cmd = grm(args) diff --git a/e2e_tests/test_repos_find_remote.py b/e2e_tests/test_repos_find_remote.py new file mode 100644 index 0000000..40452a6 --- /dev/null +++ b/e2e_tests/test_repos_find_remote.py @@ -0,0 +1,950 @@ +#!/usr/bin/env python3 + +import re +import os + +import toml +import pytest +import yaml + +from helpers import * + + +ALTERNATE_DOMAIN = os.environ["ALTERNATE_DOMAIN"] +PROVIDERS = ["github", "gitlab"] + + +@pytest.mark.parametrize("use_config", [True, False]) +def test_repos_find_remote_invalid_provider(use_config): + if use_config: + with tempfile.NamedTemporaryFile() as config: + with open(config.name, "w") as f: + f.write( + """ + provider = "thisproviderdoesnotexist" + token_command = "true" + root = "/" + """ + ) + args = ["repos", "find", "config", "--config", config.name] + cmd = grm(args, is_invalid=True) + else: + args = [ + "repos", + "find", + "remote", + "--provider", + "thisproviderdoesnotexist", + "--token-command", + "true", + "--root", + "/", + ] + cmd = grm(args, is_invalid=True) + assert cmd.returncode != 0 + assert len(cmd.stdout) == 0 + if not use_config: + assert re.match(".*isn't a valid value for.*provider", cmd.stderr) + + +@pytest.mark.parametrize("provider", PROVIDERS) +def test_repos_find_remote_invalid_format(provider): + cmd = grm( + [ + "repos", + "find", + "remote", + "--provider", + provider, + "--format", + "invalidformat", + "--token-command", + "true", + "--root", + "/myroot", + ], + is_invalid=True, + ) + assert cmd.returncode != 0 + assert len(cmd.stdout) == 0 + assert "isn't a valid value" in cmd.stderr + + +@pytest.mark.parametrize("provider", PROVIDERS) +def test_repos_find_remote_token_command_failed(provider): + cmd = grm( + [ + "repos", + "find", + "remote", + "--provider", + provider, + "--format", + "yaml", + "--token-command", + "false", + "--root", + "/myroot", + ], + is_invalid=True, + ) + assert cmd.returncode != 0 + assert len(cmd.stdout) == 0 + assert "token command failed" in cmd.stderr.lower() + + +@pytest.mark.parametrize("provider", PROVIDERS) +@pytest.mark.parametrize("use_config", [True, False]) +def test_repos_find_remote_wrong_token(provider, use_config): + if use_config: + with tempfile.NamedTemporaryFile() as config: + with open(config.name, "w") as f: + f.write( + f""" + provider = "{provider}" + token_command = "echo wrongtoken" + root = "/myroot" + [filters] + access = true + """ + ) + args = ["repos", "find", "config", "--config", config.name] + cmd = grm(args, is_invalid=True) + else: + args = [ + "repos", + "find", + "remote", + "--provider", + provider, + "--token-command", + "echo wrongtoken", + "--root", + "/myroot", + "--access", + ] + cmd = grm(args, is_invalid=True) + + assert cmd.returncode != 0 + assert len(cmd.stdout) == 0 + assert "bad credentials" in cmd.stderr.lower() + + +@pytest.mark.parametrize("provider", PROVIDERS) +@pytest.mark.parametrize("default", [True, False]) +@pytest.mark.parametrize("configtype", ["toml", "yaml"]) +@pytest.mark.parametrize("use_config", [True, False]) +def test_repos_find_remote_no_filter(provider, configtype, default, use_config): + if use_config: + with tempfile.NamedTemporaryFile() as config: + with open(config.name, "w") as f: + f.write( + f""" + provider = "{provider}" + token_command = "echo authtoken" + root = "/myroot" + """ + ) + args = ["repos", "find", "config", "--config", config.name] + if not default: + args += ["--format", configtype] + cmd = grm(args) + else: + args = [ + "repos", + "find", + "remote", + "--provider", + provider, + "--token-command", + "echo authtoken", + "--root", + "/myroot", + ] + if not default: + args += ["--format", configtype] + cmd = grm(args) + + assert cmd.returncode == 0 + assert len(cmd.stderr) == 0 + + if default or configtype == "toml": + output = toml.loads(cmd.stdout) + elif configtype == "yaml": + output = yaml.safe_load(cmd.stdout) + else: + raise NotImplementedError() + + assert isinstance(output, dict) + assert set(output.keys()) == {"trees"} + assert isinstance(output["trees"], list) + assert len(output["trees"]) == 0 + + +@pytest.mark.parametrize("provider", PROVIDERS) +@pytest.mark.parametrize("configtype_default", [True, False]) +@pytest.mark.parametrize("configtype", ["toml", "yaml"]) +@pytest.mark.parametrize("use_config", [True, False]) +def test_repos_find_remote_user_empty( + provider, configtype, configtype_default, use_config +): + if use_config: + with tempfile.NamedTemporaryFile() as config: + with open(config.name, "w") as f: + cfg = f""" + provider = "{provider}" + token_command = "echo authtoken" + root = "/myroot" + + [filters] + users = ["someotheruser"] + """ + + f.write(cfg) + args = ["repos", "find", "config", "--config", config.name] + if not configtype_default: + args += ["--format", configtype] + cmd = grm(args) + else: + args = [ + "repos", + "find", + "remote", + "--provider", + provider, + "--token-command", + "echo authtoken", + "--root", + "/myroot", + "--user", + "someotheruser", + ] + + if not configtype_default: + args += ["--format", configtype] + cmd = grm(args) + assert cmd.returncode == 0 + assert len(cmd.stderr) == 0 + + if configtype_default or configtype == "toml": + output = toml.loads(cmd.stdout) + elif configtype == "yaml": + output = yaml.safe_load(cmd.stdout) + else: + raise NotImplementedError() + + assert isinstance(output, dict) + assert set(output.keys()) == {"trees"} + assert isinstance(output["trees"], list) + assert len(output["trees"]) == 0 + + +@pytest.mark.parametrize("provider", PROVIDERS) +@pytest.mark.parametrize("configtype_default", [True, False]) +@pytest.mark.parametrize("configtype", ["toml", "yaml"]) +@pytest.mark.parametrize("worktree_default", [True, False]) +@pytest.mark.parametrize("worktree", [True, False]) +@pytest.mark.parametrize("use_owner", [True, False]) +@pytest.mark.parametrize("force_ssh", [True, False]) +@pytest.mark.parametrize("use_alternate_endpoint", [True, False]) +@pytest.mark.parametrize("use_config", [True, False]) +def test_repos_find_remote_user( + provider, + configtype, + configtype_default, + worktree, + worktree_default, + use_owner, + force_ssh, + use_alternate_endpoint, + use_config, +): + if use_config: + with tempfile.NamedTemporaryFile() as config: + with open(config.name, "w") as f: + cfg = f""" + provider = "{provider}" + token_command = "echo authtoken" + root = "/myroot" + """ + + if use_alternate_endpoint: + cfg += f'api_url = "http://{ALTERNATE_DOMAIN}:5000/{provider}"\n' + if not worktree_default: + cfg += f"worktree = {str(worktree).lower()}\n" + if force_ssh: + cfg += f"force_ssh = true\n" + if use_owner: + cfg += """ + [filters] + owner = true\n + """ + else: + cfg += """ + [filters] + users = ["myuser1"]\n + """ + + print(cfg) + f.write(cfg) + + args = ["repos", "find", "config", "--config", config.name] + if not configtype_default: + args += ["--format", configtype] + cmd = grm(args) + else: + args = [ + "repos", + "find", + "remote", + "--provider", + provider, + "--token-command", + "echo authtoken", + "--root", + "/myroot", + ] + if use_owner: + args += ["--owner"] + else: + args += ["--user", "myuser1"] + if force_ssh: + args += ["--force-ssh"] + if not worktree_default: + args += ["--worktree", str(worktree).lower()] + if use_alternate_endpoint: + args += ["--api-url", f"http://{ALTERNATE_DOMAIN}:5000/{provider}"] + + if not configtype_default: + args += ["--format", configtype] + cmd = grm(args) + + if use_alternate_endpoint and provider == "github": + assert cmd.returncode != 0 + assert "overriding is not supported for github" in cmd.stderr.lower() + return + + assert cmd.returncode == 0 + assert len(cmd.stderr) == 0 + + if configtype_default or configtype == "toml": + output = toml.loads(cmd.stdout) + elif configtype == "yaml": + output = yaml.safe_load(cmd.stdout) + else: + raise NotImplementedError() + + assert isinstance(output, dict) + assert set(output.keys()) == {"trees"} + assert isinstance(output["trees"], list) + assert len(output["trees"]) == 1 + + assert set(output["trees"][0].keys()) == {"root", "repos"} + assert isinstance(output["trees"][0]["repos"], list) + assert len(output["trees"][0]["repos"]) == 5 + + for i in range(1, 6): + repo = [r for r in output["trees"][0]["repos"] if r["name"] == f"myproject{i}"][ + 0 + ] + assert repo["worktree_setup"] is (not worktree_default and worktree) + assert isinstance(repo["remotes"], list) + assert len(repo["remotes"]) == 1 + assert repo["remotes"][0]["name"] == provider + if force_ssh or i == 1: + assert ( + repo["remotes"][0]["url"] + == f"ssh://git@example.com/myuser1/myproject{i}.git" + ) + assert repo["remotes"][0]["type"] == "ssh" + else: + assert ( + repo["remotes"][0]["url"] + == f"https://example.com/myuser1/myproject{i}.git" + ) + assert repo["remotes"][0]["type"] == "https" + + +@pytest.mark.parametrize("provider", PROVIDERS) +@pytest.mark.parametrize("configtype_default", [False]) +@pytest.mark.parametrize("configtype", ["toml", "yaml"]) +@pytest.mark.parametrize("use_alternate_endpoint", [True, False]) +@pytest.mark.parametrize("use_config", [True, False]) +def test_repos_find_remote_group_empty( + provider, configtype, configtype_default, use_alternate_endpoint, use_config +): + if use_config: + with tempfile.NamedTemporaryFile() as config: + with open(config.name, "w") as f: + cfg = f""" + provider = "{provider}" + token_command = "echo authtoken" + root = "/myroot" + """ + + if use_alternate_endpoint: + cfg += f'api_url = "http://{ALTERNATE_DOMAIN}:5000/{provider}"\n' + cfg += """ + [filters] + groups = ["someothergroup"]\n + """ + + f.write(cfg) + + args = ["repos", "find", "config", "--config", config.name] + if not configtype_default: + args += ["--format", configtype] + cmd = grm(args) + else: + args = [ + "repos", + "find", + "remote", + "--provider", + provider, + "--token-command", + "echo authtoken", + "--root", + "/myroot", + "--group", + "someothergroup", + ] + if use_alternate_endpoint: + args += ["--api-url", f"http://{ALTERNATE_DOMAIN}:5000/{provider}"] + + if not configtype_default: + args += ["--format", configtype] + cmd = grm(args) + + if use_alternate_endpoint and provider == "github": + assert cmd.returncode != 0 + assert "overriding is not supported for github" in cmd.stderr.lower() + return + assert cmd.returncode == 0 + assert len(cmd.stderr) == 0 + + if configtype_default or configtype == "toml": + output = toml.loads(cmd.stdout) + elif configtype == "yaml": + output = yaml.safe_load(cmd.stdout) + else: + raise NotImplementedError() + + assert isinstance(output, dict) + assert set(output.keys()) == {"trees"} + assert isinstance(output["trees"], list) + assert len(output["trees"]) == 0 + + +@pytest.mark.parametrize("provider", PROVIDERS) +@pytest.mark.parametrize("configtype_default", [False]) +@pytest.mark.parametrize("configtype", ["toml", "yaml"]) +@pytest.mark.parametrize("worktree_default", [True, False]) +@pytest.mark.parametrize("worktree", [True, False]) +@pytest.mark.parametrize("force_ssh", [True, False]) +@pytest.mark.parametrize("use_alternate_endpoint", [True, False]) +@pytest.mark.parametrize("use_config", [True, False]) +def test_repos_find_remote_group( + provider, + configtype, + configtype_default, + worktree, + worktree_default, + force_ssh, + use_alternate_endpoint, + use_config, +): + if use_config: + with tempfile.NamedTemporaryFile() as config: + with open(config.name, "w") as f: + cfg = f""" + provider = "{provider}" + token_command = "echo authtoken" + root = "/myroot" + """ + + if not worktree_default: + cfg += f"worktree = {str(worktree).lower()}\n" + if force_ssh: + cfg += f"force_ssh = true\n" + if use_alternate_endpoint: + cfg += f'api_url = "http://{ALTERNATE_DOMAIN}:5000/{provider}"\n' + cfg += """ + [filters] + groups = ["mygroup1"]\n + """ + + f.write(cfg) + + args = ["repos", "find", "config", "--config", config.name] + if not configtype_default: + args += ["--format", configtype] + cmd = grm(args) + else: + args = [ + "repos", + "find", + "remote", + "--provider", + provider, + "--token-command", + "echo authtoken", + "--root", + "/myroot", + "--group", + "mygroup1", + ] + if not worktree_default: + args += ["--worktree", str(worktree).lower()] + if force_ssh: + args += ["--force-ssh"] + if use_alternate_endpoint: + args += ["--api-url", f"http://{ALTERNATE_DOMAIN}:5000/{provider}"] + + if not configtype_default: + args += ["--format", configtype] + cmd = grm(args) + if use_alternate_endpoint and provider == "github": + assert cmd.returncode != 0 + assert "overriding is not supported for github" in cmd.stderr.lower() + return + assert cmd.returncode == 0 + assert len(cmd.stderr) == 0 + + if configtype_default or configtype == "toml": + output = toml.loads(cmd.stdout) + elif configtype == "yaml": + output = yaml.safe_load(cmd.stdout) + else: + raise NotImplementedError() + + assert isinstance(output, dict) + assert set(output.keys()) == {"trees"} + assert isinstance(output["trees"], list) + assert len(output["trees"]) == 1 + + assert set(output["trees"][0].keys()) == {"root", "repos"} + assert isinstance(output["trees"][0]["repos"], list) + assert len(output["trees"][0]["repos"]) == 5 + + for i in range(1, 6): + repo = [r for r in output["trees"][0]["repos"] if r["name"] == f"myproject{i}"][ + 0 + ] + assert repo["worktree_setup"] is (not worktree_default and worktree) + assert isinstance(repo["remotes"], list) + assert len(repo["remotes"]) == 1 + if force_ssh or i == 1: + assert repo["remotes"][0]["name"] == provider + assert ( + repo["remotes"][0]["url"] + == f"ssh://git@example.com/mygroup1/myproject{i}.git" + ) + assert repo["remotes"][0]["type"] == "ssh" + else: + assert repo["remotes"][0]["name"] == provider + assert ( + repo["remotes"][0]["url"] + == f"https://example.com/mygroup1/myproject{i}.git" + ) + assert repo["remotes"][0]["type"] == "https" + + +@pytest.mark.parametrize("provider", PROVIDERS) +@pytest.mark.parametrize("configtype_default", [False]) +@pytest.mark.parametrize("configtype", ["toml", "yaml"]) +@pytest.mark.parametrize("worktree_default", [True, False]) +@pytest.mark.parametrize("worktree", [True, False]) +@pytest.mark.parametrize("use_owner", [True, False]) +@pytest.mark.parametrize("force_ssh", [True, False]) +@pytest.mark.parametrize("use_alternate_endpoint", [True, False]) +@pytest.mark.parametrize("use_config", [True, False]) +def test_repos_find_remote_user_and_group( + provider, + configtype, + configtype_default, + worktree, + worktree_default, + use_owner, + force_ssh, + use_alternate_endpoint, + use_config, +): + if use_config: + with tempfile.NamedTemporaryFile() as config: + with open(config.name, "w") as f: + cfg = f""" + provider = "{provider}" + token_command = "echo authtoken" + root = "/myroot" + """ + + if not worktree_default: + cfg += f"worktree = {str(worktree).lower()}\n" + if force_ssh: + cfg += f"force_ssh = true\n" + if use_alternate_endpoint: + cfg += f'api_url = "http://{ALTERNATE_DOMAIN}:5000/{provider}"\n' + cfg += """ + [filters] + groups = ["mygroup1"]\n + """ + + if use_owner: + cfg += "owner = true\n" + else: + cfg += 'users = ["myuser1"]\n' + + f.write(cfg) + + args = ["repos", "find", "config", "--config", config.name] + if not configtype_default: + args += ["--format", configtype] + cmd = grm(args) + else: + args = [ + "repos", + "find", + "remote", + "--provider", + provider, + "--token-command", + "echo authtoken", + "--root", + "/myroot", + "--group", + "mygroup1", + ] + if use_owner: + args += ["--owner"] + else: + args += ["--user", "myuser1"] + if not worktree_default: + args += ["--worktree", str(worktree).lower()] + if force_ssh: + args += ["--force-ssh"] + if use_alternate_endpoint: + args += ["--api-url", f"http://{ALTERNATE_DOMAIN}:5000/{provider}"] + + if not configtype_default: + args += ["--format", configtype] + cmd = grm(args) + if use_alternate_endpoint and provider == "github": + assert cmd.returncode != 0 + assert "overriding is not supported for github" in cmd.stderr.lower() + return + assert cmd.returncode == 0 + assert len(cmd.stderr) == 0 + + if configtype_default or configtype == "toml": + output = toml.loads(cmd.stdout) + elif configtype == "yaml": + output = yaml.safe_load(cmd.stdout) + else: + raise NotImplementedError() + + assert isinstance(output, dict) + assert set(output.keys()) == {"trees"} + assert isinstance(output["trees"], list) + assert len(output["trees"]) == 2 + + user_namespace = [t for t in output["trees"] if t["root"] == "/myroot/myuser1"][0] + + assert set(user_namespace.keys()) == {"root", "repos"} + assert isinstance(user_namespace["repos"], list) + assert len(user_namespace["repos"]) == 5 + + for i in range(1, 6): + repo = [r for r in user_namespace["repos"] if r["name"] == f"myproject{i}"][0] + assert repo["worktree_setup"] is (not worktree_default and worktree) + assert isinstance(repo["remotes"], list) + assert len(repo["remotes"]) == 1 + assert repo["remotes"][0]["name"] == provider + if force_ssh or i == 1: + assert ( + repo["remotes"][0]["url"] + == f"ssh://git@example.com/myuser1/myproject{i}.git" + ) + assert repo["remotes"][0]["type"] == "ssh" + else: + assert ( + repo["remotes"][0]["url"] + == f"https://example.com/myuser1/myproject{i}.git" + ) + assert repo["remotes"][0]["type"] == "https" + + group_namespace = [t for t in output["trees"] if t["root"] == "/myroot/mygroup1"][0] + + assert set(group_namespace.keys()) == {"root", "repos"} + assert isinstance(group_namespace["repos"], list) + assert len(group_namespace["repos"]) == 5 + + for i in range(1, 6): + repo = [r for r in group_namespace["repos"] if r["name"] == f"myproject{i}"][0] + assert repo["worktree_setup"] is (not worktree_default and worktree) + assert isinstance(repo["remotes"], list) + assert len(repo["remotes"]) == 1 + assert repo["remotes"][0]["name"] == provider + if force_ssh or i == 1: + assert ( + repo["remotes"][0]["url"] + == f"ssh://git@example.com/mygroup1/myproject{i}.git" + ) + assert repo["remotes"][0]["type"] == "ssh" + else: + assert ( + repo["remotes"][0]["url"] + == f"https://example.com/mygroup1/myproject{i}.git" + ) + assert repo["remotes"][0]["type"] == "https" + + +@pytest.mark.parametrize("provider", PROVIDERS) +@pytest.mark.parametrize("configtype_default", [False]) +@pytest.mark.parametrize("configtype", ["toml", "yaml"]) +@pytest.mark.parametrize("worktree_default", [True, False]) +@pytest.mark.parametrize("worktree", [True, False]) +@pytest.mark.parametrize("with_user_filter", [True, False]) +@pytest.mark.parametrize("with_group_filter", [True, False]) +@pytest.mark.parametrize("force_ssh", [True, False]) +@pytest.mark.parametrize("use_alternate_endpoint", [True, False]) +@pytest.mark.parametrize("use_config", [True, False]) +def test_repos_find_remote_owner( + provider, + configtype, + configtype_default, + worktree, + worktree_default, + with_user_filter, + with_group_filter, + force_ssh, + use_alternate_endpoint, + use_config, +): + if use_config: + with tempfile.NamedTemporaryFile() as config: + with open(config.name, "w") as f: + cfg = f""" + provider = "{provider}" + token_command = "echo authtoken" + root = "/myroot" + """ + + if not worktree_default: + cfg += f"worktree = {str(worktree).lower()}\n" + if force_ssh: + cfg += f"force_ssh = true\n" + if use_alternate_endpoint: + cfg += f'api_url = "http://{ALTERNATE_DOMAIN}:5000/{provider}"\n' + cfg += """ + [filters] + access = true\n + """ + + if with_user_filter: + cfg += 'users = ["myuser1"]\n' + if with_group_filter: + cfg += 'groups = ["mygroup1"]\n' + + f.write(cfg) + + args = ["repos", "find", "config", "--config", config.name] + if not configtype_default: + args += ["--format", configtype] + cmd = grm(args) + + else: + args = [ + "repos", + "find", + "remote", + "--provider", + provider, + "--token-command", + "echo authtoken", + "--root", + "/myroot", + "--access", + ] + if not worktree_default: + args += ["--worktree", str(worktree).lower()] + if with_user_filter: + args += ["--user", "myuser1"] + if with_group_filter: + args += ["--group", "mygroup1"] + if force_ssh: + args += ["--force-ssh"] + if use_alternate_endpoint: + args += ["--api-url", f"http://{ALTERNATE_DOMAIN}:5000/{provider}"] + + if not configtype_default: + args += ["--format", configtype] + cmd = grm(args) + if use_alternate_endpoint and provider == "github": + assert cmd.returncode != 0 + assert "overriding is not supported for github" in cmd.stderr.lower() + return + assert cmd.returncode == 0 + assert len(cmd.stderr) == 0 + + if configtype_default or configtype == "toml": + output = toml.loads(cmd.stdout) + elif configtype == "yaml": + output = yaml.safe_load(cmd.stdout) + else: + raise NotImplementedError() + + assert isinstance(output, dict) + assert set(output.keys()) == {"trees"} + assert isinstance(output["trees"], list) + assert len(output["trees"]) == 4 + + user_namespace_1 = [t for t in output["trees"] if t["root"] == "/myroot/myuser1"][0] + + assert set(user_namespace_1.keys()) == {"root", "repos"} + assert isinstance(user_namespace_1["repos"], list) + + if with_user_filter: + assert len(user_namespace_1["repos"]) == 5 + + for i in range(1, 6): + repo = [ + r for r in user_namespace_1["repos"] if r["name"] == f"myproject{i}" + ][0] + assert repo["worktree_setup"] is (not worktree_default and worktree) + assert isinstance(repo["remotes"], list) + assert len(repo["remotes"]) == 1 + assert repo["remotes"][0]["name"] == provider + if force_ssh or i == 1: + assert ( + repo["remotes"][0]["url"] + == f"ssh://git@example.com/myuser1/myproject{i}.git" + ) + assert repo["remotes"][0]["type"] == "ssh" + else: + assert ( + repo["remotes"][0]["url"] + == f"https://example.com/myuser1/myproject{i}.git" + ) + assert repo["remotes"][0]["type"] == "https" + else: + assert len(user_namespace_1["repos"]) == 2 + + for i in range(1, 3): + repo = [ + r for r in user_namespace_1["repos"] if r["name"] == f"myproject{i}" + ][0] + assert repo["worktree_setup"] is (not worktree_default and worktree) + assert isinstance(repo["remotes"], list) + assert len(repo["remotes"]) == 1 + assert repo["remotes"][0]["name"] == provider + if force_ssh or i == 1: + assert ( + repo["remotes"][0]["url"] + == f"ssh://git@example.com/myuser1/myproject{i}.git" + ) + assert repo["remotes"][0]["type"] == "ssh" + else: + assert ( + repo["remotes"][0]["url"] + == f"https://example.com/myuser1/myproject{i}.git" + ) + assert repo["remotes"][0]["type"] == "https" + + user_namespace_2 = [t for t in output["trees"] if t["root"] == "/myroot/myuser2"][0] + + assert set(user_namespace_2.keys()) == {"root", "repos"} + assert isinstance(user_namespace_2["repos"], list) + assert len(user_namespace_2["repos"]) == 1 + + repo = user_namespace_2["repos"][0] + assert repo["worktree_setup"] is (not worktree_default and worktree) + assert isinstance(repo["remotes"], list) + assert len(repo["remotes"]) == 1 + assert repo["remotes"][0]["name"] == provider + if force_ssh: + assert ( + repo["remotes"][0]["url"] == f"ssh://git@example.com/myuser2/myproject3.git" + ) + assert repo["remotes"][0]["type"] == "ssh" + else: + assert ( + repo["remotes"][0]["url"] == f"https://example.com/myuser2/myproject3.git" + ) + assert repo["remotes"][0]["type"] == "https" + + group_namespace_1 = [t for t in output["trees"] if t["root"] == "/myroot/mygroup1"][ + 0 + ] + + assert set(group_namespace_1.keys()) == {"root", "repos"} + assert isinstance(group_namespace_1["repos"], list) + + if with_group_filter: + assert len(group_namespace_1["repos"]) == 5 + + for i in range(1, 6): + repo = [ + r for r in group_namespace_1["repos"] if r["name"] == f"myproject{i}" + ][0] + assert repo["worktree_setup"] is (not worktree_default and worktree) + assert isinstance(repo["remotes"], list) + assert len(repo["remotes"]) == 1 + assert repo["remotes"][0]["name"] == provider + if force_ssh or i == 1: + assert ( + repo["remotes"][0]["url"] + == f"ssh://git@example.com/mygroup1/myproject{i}.git" + ) + assert repo["remotes"][0]["type"] == "ssh" + else: + assert ( + repo["remotes"][0]["url"] + == f"https://example.com/mygroup1/myproject{i}.git" + ) + assert repo["remotes"][0]["type"] == "https" + else: + assert len(group_namespace_1["repos"]) == 1 + + repo = group_namespace_1["repos"][0] + assert repo["worktree_setup"] is (not worktree_default and worktree) + assert isinstance(repo["remotes"], list) + assert len(repo["remotes"]) == 1 + assert repo["remotes"][0]["name"] == provider + if force_ssh: + assert ( + repo["remotes"][0]["url"] + == f"ssh://git@example.com/mygroup1/myproject4.git" + ) + assert repo["remotes"][0]["type"] == "ssh" + else: + assert ( + repo["remotes"][0]["url"] + == f"https://example.com/mygroup1/myproject4.git" + ) + assert repo["remotes"][0]["type"] == "https" + + group_namespace_2 = [t for t in output["trees"] if t["root"] == "/myroot/mygroup2"][ + 0 + ] + + assert set(group_namespace_2.keys()) == {"root", "repos"} + assert isinstance(group_namespace_2["repos"], list) + assert len(group_namespace_2["repos"]) == 1 + + repo = group_namespace_2["repos"][0] + assert repo["worktree_setup"] is (not worktree_default and worktree) + assert isinstance(repo["remotes"], list) + assert len(repo["remotes"]) == 1 + assert repo["remotes"][0]["name"] == provider + if force_ssh: + assert ( + repo["remotes"][0]["url"] + == f"ssh://git@example.com/mygroup2/myproject5.git" + ) + assert repo["remotes"][0]["type"] == "ssh" + else: + assert ( + repo["remotes"][0]["url"] == f"https://example.com/mygroup2/myproject5.git" + ) + assert repo["remotes"][0]["type"] == "https" diff --git a/e2e_tests/test_repos_sync.py b/e2e_tests/test_repos_sync.py index 2586943..67cdf3c 100644 --- a/e2e_tests/test_repos_sync.py +++ b/e2e_tests/test_repos_sync.py @@ -154,7 +154,7 @@ def test_repos_sync_config_is_valid_symlink(configtype): subprocess.run(["cat", config.name]) - cmd = grm(["repos", "sync", "--config", config_symlink]) + cmd = grm(["repos", "sync", "config", "--config", config_symlink]) assert cmd.returncode == 0 git_dir = os.path.join(target, "test") @@ -174,7 +174,7 @@ def test_repos_sync_config_is_invalid_symlink(): config_symlink = os.path.join(config_dir, "cfglink") os.symlink(nonexistent_dir, config_symlink) - cmd = grm(["repos", "sync", "--config", config_symlink]) + cmd = grm(["repos", "sync", "config", "--config", config_symlink]) assert cmd.returncode != 0 assert len(cmd.stdout) == 0 @@ -185,7 +185,7 @@ def test_repos_sync_config_is_invalid_symlink(): def test_repos_sync_config_is_directory(): with tempfile.TemporaryDirectory() as config: - cmd = grm(["repos", "sync", "--config", config]) + cmd = grm(["repos", "sync", "config", "--config", config]) assert cmd.returncode != 0 assert len(cmd.stdout) == 0 @@ -197,12 +197,11 @@ def test_repos_sync_config_is_unreadable(): config_path = os.path.join(config_dir, "cfg") open(config_path, "w") os.chmod(config_path, 0o0000) - cmd = grm(["repos", "sync", "--config", config_path]) + cmd = grm(["repos", "sync", "config", "--config", config_path]) assert os.path.exists(config_path) assert cmd.returncode != 0 assert len(cmd.stdout) == 0 - assert "permission denied" in cmd.stderr.lower() @pytest.mark.parametrize("configtype", ["toml", "yaml"]) @@ -213,7 +212,7 @@ def test_repos_sync_unmanaged_repos(configtype): with open(config.name, "w") as f: f.write(templates["repo_simple"][configtype].format(root=root)) - cmd = grm(["repos", "sync", "--config", config.name]) + cmd = grm(["repos", "sync", "config", "--config", config.name]) assert cmd.returncode == 0 git_dir = os.path.join(root, "test") @@ -232,7 +231,7 @@ def test_repos_sync_root_is_file(configtype): with open(config.name, "w") as f: f.write(templates["repo_simple"][configtype].format(root=target.name)) - cmd = grm(["repos", "sync", "--config", config.name]) + cmd = grm(["repos", "sync", "config", "--config", config.name]) assert cmd.returncode != 0 assert len(cmd.stdout) == 0 assert "not a directory" in cmd.stderr.lower() @@ -251,7 +250,7 @@ def test_repos_sync_normal_clone(configtype): ) ) - cmd = grm(["repos", "sync", "--config", config.name]) + cmd = grm(["repos", "sync", "config", "--config", config.name]) assert cmd.returncode == 0 git_dir = os.path.join(target, "test") @@ -283,7 +282,7 @@ def test_repos_sync_normal_init(configtype): with open(config.name, "w") as f: f.write(templates["repo_simple"][configtype].format(root=target)) - cmd = grm(["repos", "sync", "--config", config.name]) + cmd = grm(["repos", "sync", "config", "--config", config.name]) assert cmd.returncode == 0 git_dir = os.path.join(target, "test") @@ -309,7 +308,7 @@ def test_repos_sync_normal_add_remote(configtype): ) ) - cmd = grm(["repos", "sync", "--config", config.name]) + cmd = grm(["repos", "sync", "config", "--config", config.name]) assert cmd.returncode == 0 git_dir = os.path.join(target, "test") @@ -329,7 +328,7 @@ def test_repos_sync_normal_add_remote(configtype): ) ) - cmd = grm(["repos", "sync", "--config", config.name]) + cmd = grm(["repos", "sync", "config", "--config", config.name]) assert cmd.returncode == 0 with git.Repo(git_dir) as repo: assert set([str(r) for r in repo.remotes]) == { @@ -359,7 +358,7 @@ def test_repos_sync_normal_remove_remote(configtype): ) ) - cmd = grm(["repos", "sync", "--config", config.name]) + cmd = grm(["repos", "sync", "config", "--config", config.name]) assert cmd.returncode == 0 git_dir = os.path.join(target, "test") @@ -382,7 +381,7 @@ def test_repos_sync_normal_remove_remote(configtype): ) ) - cmd = grm(["repos", "sync", "--config", config.name]) + cmd = grm(["repos", "sync", "config", "--config", config.name]) assert cmd.returncode == 0 shell(f"cd {git_dir} && git remote -v") with git.Repo(git_dir) as repo: @@ -424,7 +423,7 @@ def test_repos_sync_normal_change_remote_url(configtype): ) ) - cmd = grm(["repos", "sync", "--config", config.name]) + cmd = grm(["repos", "sync", "config", "--config", config.name]) assert cmd.returncode == 0 git_dir = os.path.join(target, "test") @@ -444,7 +443,7 @@ def test_repos_sync_normal_change_remote_url(configtype): ) ) - cmd = grm(["repos", "sync", "--config", config.name]) + cmd = grm(["repos", "sync", "config", "--config", config.name]) assert cmd.returncode == 0 with git.Repo(git_dir) as repo: assert set([str(r) for r in repo.remotes]) == {"origin"} @@ -467,7 +466,7 @@ def test_repos_sync_normal_change_remote_name(configtype): ) ) - cmd = grm(["repos", "sync", "--config", config.name]) + cmd = grm(["repos", "sync", "config", "--config", config.name]) assert cmd.returncode == 0 git_dir = os.path.join(target, "test") @@ -487,7 +486,7 @@ def test_repos_sync_normal_change_remote_name(configtype): ) ) - cmd = grm(["repos", "sync", "--config", config.name]) + cmd = grm(["repos", "sync", "config", "--config", config.name]) assert cmd.returncode == 0 with git.Repo(git_dir) as repo: # See the note in `test_repos_sync_normal_remove_remote()` @@ -512,7 +511,7 @@ def test_repos_sync_worktree_clone(configtype): ) ) - cmd = grm(["repos", "sync", "--config", config.name]) + cmd = grm(["repos", "sync", "config", "--config", config.name]) assert cmd.returncode == 0 worktree_dir = f"{target}/test" @@ -538,7 +537,7 @@ def test_repos_sync_worktree_init(configtype): templates["worktree_repo_simple"][configtype].format(root=target) ) - cmd = grm(["repos", "sync", "--config", config.name]) + cmd = grm(["repos", "sync", "config", "--config", config.name]) assert cmd.returncode == 0 worktree_dir = f"{target}/test" @@ -573,7 +572,7 @@ def test_repos_sync_invalid_syntax(configtype): ) else: raise NotImplementedError() - cmd = grm(["repos", "sync", "--config", config.name]) + cmd = grm(["repos", "sync", "config", "--config", config.name]) assert cmd.returncode != 0 @@ -590,11 +589,11 @@ def test_repos_sync_unchanged(configtype): ) ) - cmd = grm(["repos", "sync", "--config", config.name]) + cmd = grm(["repos", "sync", "config", "--config", config.name]) assert cmd.returncode == 0 before = checksum_directory(target) - cmd = grm(["repos", "sync", "--config", config.name]) + cmd = grm(["repos", "sync", "config", "--config", config.name]) after = checksum_directory(target) assert cmd.returncode == 0 @@ -614,7 +613,7 @@ def test_repos_sync_normal_change_to_worktree(configtype): ) ) - cmd = grm(["repos", "sync", "--config", config.name]) + cmd = grm(["repos", "sync", "config", "--config", config.name]) assert cmd.returncode == 0 git_dir = os.path.join(target, "test") @@ -626,7 +625,7 @@ def test_repos_sync_normal_change_to_worktree(configtype): ) ) - cmd = grm(["repos", "sync", "--config", config.name]) + cmd = grm(["repos", "sync", "config", "--config", config.name]) assert cmd.returncode != 0 assert "already exists" in cmd.stderr assert "not using a worktree setup" in cmd.stderr @@ -645,7 +644,7 @@ def test_repos_sync_worktree_change_to_normal(configtype): ) ) - cmd = grm(["repos", "sync", "--config", config.name]) + cmd = grm(["repos", "sync", "config", "--config", config.name]) assert cmd.returncode == 0 git_dir = os.path.join(target, "test") @@ -657,7 +656,7 @@ def test_repos_sync_worktree_change_to_normal(configtype): ) ) - cmd = grm(["repos", "sync", "--config", config.name]) + cmd = grm(["repos", "sync", "config", "--config", config.name]) assert cmd.returncode != 0 assert "already exists" in cmd.stderr assert "using a worktree setup" in cmd.stderr diff --git a/e2e_tests/update_requirementstxt.sh b/e2e_tests/update_requirementstxt.sh deleted file mode 100755 index 2f3db3d..0000000 --- a/e2e_tests/update_requirementstxt.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -set -o nounset -set -o errexit - -# shellcheck disable=SC1091 -source ./venv/bin/activate - -pip --disable-pip-version-check install -r ./requirements.txt - -pip3 list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1 | while read -r package ; do - [[ "$package" == "pip" ]] && continue - [[ "$package" == "setuptools" ]] && continue - pip install --upgrade "${package}" - version="$(pip show "${package}" | grep '^Version' | cut -d ' ' -f 2)" - message="e2e_tests/pip: Update ${package} to ${version}" - pip freeze | grep -v '^pkg_resources' > requirements.txt - git add ./requirements.txt - git commit --message "${message}" -done diff --git a/src/config.rs b/src/config.rs index e81c6a0..36873fe 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,19 +1,61 @@ use serde::{Deserialize, Serialize}; +use std::process; + +use crate::output::*; use super::repo::RepoConfig; +use std::path::Path; + +use crate::get_token_from_command; +use crate::provider; +use crate::provider::Filter; +use crate::provider::Provider; + +pub type RemoteProvider = crate::provider::RemoteProvider; + +#[derive(Debug, Serialize, Deserialize)] +#[serde(untagged)] +pub enum Config { + ConfigTree(ConfigTree), + ConfigProvider(ConfigProvider), +} + #[derive(Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -pub struct Config { +pub struct ConfigTree { pub trees: Trees, } +#[derive(Debug, Serialize, Deserialize)] +pub struct ConfigProviderFilter { + pub access: Option, + pub owner: Option, + pub users: Option>, + pub groups: Option>, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct ConfigProvider { + pub provider: RemoteProvider, + pub token_command: String, + pub root: String, + pub filters: Option, + + pub force_ssh: Option, + + pub api_url: Option, + + pub worktree: Option, + pub init_worktree: Option, +} + #[derive(Debug, Serialize, Deserialize)] pub struct Trees(Vec); impl Trees { pub fn to_config(self) -> Config { - Config { trees: self } + Config::ConfigTree(ConfigTree { trees: self }) } pub fn from_vec(vec: Vec) -> Self { @@ -30,6 +72,81 @@ impl Trees { } impl Config { + pub fn trees(self) -> Result { + match self { + Config::ConfigTree(config) => Ok(config.trees), + Config::ConfigProvider(config) => { + let token = match get_token_from_command(&config.token_command) { + Ok(token) => token, + Err(error) => { + print_error(&format!("Getting token from command failed: {}", error)); + process::exit(1); + } + }; + + let filters = config.filters.unwrap_or(ConfigProviderFilter { + access: Some(false), + owner: Some(false), + users: Some(vec![]), + groups: Some(vec![]), + }); + + let filter = Filter::new( + filters.users.unwrap_or_default(), + filters.groups.unwrap_or_default(), + filters.owner.unwrap_or(false), + filters.access.unwrap_or(false), + ); + + let repos = match config.provider { + RemoteProvider::Github => { + match provider::Github::new(filter, token, config.api_url) { + Ok(provider) => provider, + Err(error) => { + print_error(&format!("Error: {}", error)); + process::exit(1); + } + } + .get_repos( + config.worktree.unwrap_or(false), + config.force_ssh.unwrap_or(false), + )? + } + RemoteProvider::Gitlab => { + match provider::Gitlab::new(filter, token, config.api_url) { + Ok(provider) => provider, + Err(error) => { + print_error(&format!("Error: {}", error)); + process::exit(1); + } + } + .get_repos( + config.worktree.unwrap_or(false), + config.force_ssh.unwrap_or(false), + )? + } + }; + + let mut trees = vec![]; + + for (namespace, namespace_repos) in repos { + let tree = Tree { + root: crate::path_as_string(&Path::new(&config.root).join(namespace)), + repos: Some(namespace_repos), + }; + trees.push(tree); + } + Ok(Trees(trees)) + } + } + } + + pub fn from_trees(trees: Vec) -> Self { + Config::ConfigTree(ConfigTree { + trees: Trees::from_vec(trees), + }) + } + pub fn as_toml(&self) -> Result { match toml::to_string(self) { Ok(toml) => Ok(toml), @@ -49,7 +166,10 @@ pub struct Tree { pub repos: Option>, } -pub fn read_config(path: &str) -> Result { +pub fn read_config<'a, T>(path: &str) -> Result +where + T: for<'de> serde::Deserialize<'de>, +{ let content = match std::fs::read_to_string(&path) { Ok(s) => s, Err(e) => { @@ -64,7 +184,7 @@ pub fn read_config(path: &str) -> Result { } }; - let config: Config = match toml::from_str(&content) { + let config: T = match toml::from_str(&content) { Ok(c) => c, Err(_) => match serde_yaml::from_str(&content) { Ok(c) => c, diff --git a/src/grm/cmd.rs b/src/grm/cmd.rs index 7b88379..27a1c82 100644 --- a/src/grm/cmd.rs +++ b/src/grm/cmd.rs @@ -31,20 +31,53 @@ pub struct Repos { #[derive(Parser)] pub enum ReposAction { - #[clap( - visible_alias = "run", - about = "Synchronize the repositories to the configured values" - )] - Sync(Sync), - #[clap(about = "Generate a repository configuration from an existing file tree")] - Find(Find), + #[clap(subcommand)] + Sync(SyncAction), + #[clap(subcommand)] + Find(FindAction), #[clap(about = "Show status of configured repositories")] Status(OptionalConfig), } #[derive(Parser)] -#[clap()] -pub struct Sync { +#[clap(about = "Sync local repositories with a configured list")] +pub enum SyncAction { + #[clap( + about = "Synchronize the repositories to the configured values" + )] + Config(Config), + #[clap(about = "Synchronize the repositories from a remote provider")] + Remote(SyncRemoteArgs), +} + +#[derive(Parser)] +#[clap(about = "Generate a repository configuration from existing repositories")] +pub enum FindAction { + #[clap(about = "Find local repositories")] + Local(FindLocalArgs), + #[clap(about = "Find repositories on remote provider")] + Remote(FindRemoteArgs), + #[clap(about = "Find repositories as defined in the configuration file")] + Config(FindConfigArgs), +} + +#[derive(Parser)] +pub struct FindLocalArgs { + #[clap(help = "The path to search through")] + pub path: String, + + #[clap( + arg_enum, + short, + long, + help = "Format to produce", + default_value_t = ConfigFormat::Toml, + )] + pub format: ConfigFormat, +} + +#[derive(Parser)] +pub struct FindConfigArgs { #[clap( short, long, @@ -52,6 +85,145 @@ pub struct Sync { help = "Path to the configuration file" )] pub config: String, + + #[clap( + arg_enum, + short, + long, + help = "Format to produce", + default_value_t = ConfigFormat::Toml, + )] + pub format: ConfigFormat, +} + +#[derive(Parser)] +#[clap()] +pub struct FindRemoteArgs { + #[clap(short, long, help = "Path to the configuration file")] + pub config: Option, + + #[clap(arg_enum, short, long, help = "Remote provider to use")] + pub provider: RemoteProvider, + + #[clap( + multiple_occurrences = true, + name = "user", + long, + help = "Users to get repositories from" + )] + pub users: Vec, + + #[clap( + multiple_occurrences = true, + name = "group", + long, + help = "Groups to get repositories from" + )] + pub groups: Vec, + + #[clap(long, help = "Get repositories that belong to the requesting user")] + pub owner: bool, + + #[clap(long, help = "Get repositories that the requesting user has access to")] + pub access: bool, + + #[clap(long, help = "Always use SSH, even for public repositories")] + pub force_ssh: bool, + + #[clap(long, help = "Command to get API token")] + pub token_command: String, + + #[clap(long, help = "Root of the repo tree to produce")] + pub root: String, + + #[clap( + arg_enum, + short, + long, + help = "Format to produce", + default_value_t = ConfigFormat::Toml, + )] + pub format: ConfigFormat, + + #[clap( + long, + help = "Use worktree setup for repositories", + possible_values = &["true", "false"], + default_value = "false", + default_missing_value = "true", + min_values = 0, + max_values = 1, + )] + pub worktree: String, + + #[clap(long, help = "Base URL for the API")] + pub api_url: Option, +} + +#[derive(Parser)] +#[clap()] +pub struct Config { + #[clap( + short, + long, + default_value = "./config.toml", + help = "Path to the configuration file" + )] + pub config: String, +} + +pub type RemoteProvider = grm::provider::RemoteProvider; + +#[derive(Parser)] +#[clap()] +pub struct SyncRemoteArgs { + #[clap(arg_enum, short, long, help = "Remote provider to use")] + pub provider: RemoteProvider, + + #[clap( + multiple_occurrences = true, + name = "user", + long, + help = "Users to get repositories from" + )] + pub users: Vec, + + #[clap( + multiple_occurrences = true, + name = "group", + long, + help = "Groups to get repositories from" + )] + pub groups: Vec, + + #[clap(long, help = "Get repositories that belong to the requesting user")] + pub owner: bool, + + #[clap(long, help = "Get repositories that the requesting user has access to")] + pub access: bool, + + #[clap(long, help = "Always use SSH, even for public repositories")] + pub force_ssh: bool, + + #[clap(long, help = "Command to get API token")] + pub token_command: String, + + #[clap(long, help = "Root of the repo tree to produce")] + pub root: String, + + #[clap( + long, + help = "Use worktree setup for repositories", + possible_values = &["true", "false"], + default_value = "false", + default_missing_value = "true", + min_values = 0, + max_values = 1, + )] + pub worktree: String, + + #[clap(long, help = "Base URL for the API")] + pub api_url: Option, } #[derive(Parser)] @@ -67,21 +239,6 @@ pub enum ConfigFormat { Toml, } -#[derive(Parser)] -pub struct Find { - #[clap(help = "The path to search through")] - pub path: String, - - #[clap( - arg_enum, - short, - long, - help = "Format to produce", - default_value_t = ConfigFormat::Toml, - )] - pub format: ConfigFormat, -} - #[derive(Parser)] pub struct Worktree { #[clap(subcommand, name = "action")] diff --git a/src/grm/main.rs b/src/grm/main.rs index 50547da..40a8652 100644 --- a/src/grm/main.rs +++ b/src/grm/main.rs @@ -5,6 +5,8 @@ mod cmd; use grm::config; use grm::output::*; +use grm::provider; +use grm::provider::Provider; use grm::repo; fn main() { @@ -12,26 +14,104 @@ fn main() { match opts.subcmd { cmd::SubCommand::Repos(repos) => match repos.action { - cmd::ReposAction::Sync(sync) => { - let config = match config::read_config(&sync.config) { - Ok(config) => config, - Err(error) => { - print_error(&error); - process::exit(1); - } - }; - match grm::sync_trees(config) { - Ok(success) => { - if !success { - process::exit(1) + cmd::ReposAction::Sync(sync) => match sync { + cmd::SyncAction::Config(args) => { + let config = match config::read_config(&args.config) { + Ok(config) => config, + Err(error) => { + print_error(&error); + process::exit(1); + } + }; + match grm::sync_trees(config) { + Ok(success) => { + if !success { + process::exit(1) + } + } + Err(error) => { + print_error(&format!("Error syncing trees: {}", error)); + process::exit(1); } } - Err(error) => { - print_error(&format!("Error syncing trees: {}", error)); - process::exit(1); + } + cmd::SyncAction::Remote(args) => { + let token = match grm::get_token_from_command(&args.token_command) { + Ok(token) => token, + Err(error) => { + print_error(&format!("Getting token from command failed: {}", error)); + process::exit(1); + } + }; + + let filter = grm::provider::Filter::new( + args.users, + args.groups, + args.owner, + args.access, + ); + + let worktree = args.worktree == "true"; + + let repos = match args.provider { + cmd::RemoteProvider::Github => { + match grm::provider::Github::new(filter, token, args.api_url) { + Ok(provider) => provider, + Err(error) => { + print_error(&format!("Error: {}", error)); + process::exit(1); + } + } + .get_repos(worktree, args.force_ssh) + } + cmd::RemoteProvider::Gitlab => { + match grm::provider::Gitlab::new(filter, token, args.api_url) { + Ok(provider) => provider, + Err(error) => { + print_error(&format!("Error: {}", error)); + process::exit(1); + } + } + .get_repos(worktree, args.force_ssh) + } + }; + + match repos { + Ok(repos) => { + let mut trees: Vec = vec![]; + + for (namespace, repolist) in repos { + let tree = config::Tree { + root: Path::new(&args.root) + .join(namespace) + .display() + .to_string(), + repos: Some(repolist), + }; + trees.push(tree); + } + + let config = config::Config::from_trees(trees); + + match grm::sync_trees(config) { + Ok(success) => { + if !success { + process::exit(1) + } + } + Err(error) => { + print_error(&format!("Error syncing trees: {}", error)); + process::exit(1); + } + } + } + Err(error) => { + print_error(&format!("Error: {}", error)); + process::exit(1); + } } } - } + }, cmd::ReposAction::Status(args) => match &args.config { Some(config_path) => { let config = match config::read_config(config_path) { @@ -79,47 +159,166 @@ fn main() { } } }, - cmd::ReposAction::Find(find) => { - let path = Path::new(&find.path); - if !path.exists() { - print_error(&format!("Path \"{}\" does not exist", path.display())); - process::exit(1); - } - if !path.is_dir() { - print_error(&format!("Path \"{}\" is not a directory", path.display())); - process::exit(1); - } - - let path = match path.canonicalize() { - Ok(path) => path, - Err(error) => { - print_error(&format!( - "Failed to canonicalize path \"{}\". This is a bug. Error message: {}", - &path.display(), - error - )); + cmd::ReposAction::Find(find) => match find { + cmd::FindAction::Local(args) => { + let path = Path::new(&args.path); + if !path.exists() { + print_error(&format!("Path \"{}\" does not exist", path.display())); process::exit(1); } - }; - - let (found_repos, warnings) = match grm::find_in_tree(&path) { - Ok((repos, warnings)) => (repos, warnings), - Err(error) => { - print_error(&error); + if !path.is_dir() { + print_error(&format!("Path \"{}\" is not a directory", path.display())); process::exit(1); } - }; - let trees = grm::config::Trees::from_vec(vec![found_repos]); - if trees.as_vec_ref().iter().all(|t| match &t.repos { - None => false, - Some(r) => r.is_empty(), - }) { - print_warning("No repositories found"); - } else { - let config = trees.to_config(); + let path = match path.canonicalize() { + Ok(path) => path, + Err(error) => { + print_error(&format!( + "Failed to canonicalize path \"{}\". This is a bug. Error message: {}", + &path.display(), + error + )); + process::exit(1); + } + }; - match find.format { + let (found_repos, warnings) = match grm::find_in_tree(&path) { + Ok((repos, warnings)) => (repos, warnings), + Err(error) => { + print_error(&error); + process::exit(1); + } + }; + + let trees = grm::config::Trees::from_vec(vec![found_repos]); + if trees.as_vec_ref().iter().all(|t| match &t.repos { + None => false, + Some(r) => r.is_empty(), + }) { + print_warning("No repositories found"); + } else { + let config = trees.to_config(); + + match args.format { + cmd::ConfigFormat::Toml => { + let toml = match config.as_toml() { + Ok(toml) => toml, + Err(error) => { + print_error(&format!( + "Failed converting config to TOML: {}", + &error + )); + process::exit(1); + } + }; + print!("{}", toml); + } + cmd::ConfigFormat::Yaml => { + let yaml = match config.as_yaml() { + Ok(yaml) => yaml, + Err(error) => { + print_error(&format!( + "Failed converting config to YAML: {}", + &error + )); + process::exit(1); + } + }; + print!("{}", yaml); + } + } + } + for warning in warnings { + print_warning(&warning); + } + } + cmd::FindAction::Config(args) => { + let config: crate::config::ConfigProvider = + match config::read_config(&args.config) { + Ok(config) => config, + Err(error) => { + print_error(&error); + process::exit(1); + } + }; + + let token = match grm::get_token_from_command(&config.token_command) { + Ok(token) => token, + Err(error) => { + print_error(&format!("Getting token from command failed: {}", error)); + process::exit(1); + } + }; + + let filters = config.filters.unwrap_or(grm::config::ConfigProviderFilter { + access: Some(false), + owner: Some(false), + users: Some(vec![]), + groups: Some(vec![]), + }); + + let filter = provider::Filter::new( + filters.users.unwrap_or_default(), + filters.groups.unwrap_or_default(), + filters.owner.unwrap_or(false), + filters.access.unwrap_or(false), + ); + + let repos = match config.provider { + provider::RemoteProvider::Github => { + match match provider::Github::new(filter, token, config.api_url) { + Ok(provider) => provider, + Err(error) => { + print_error(&format!("Error: {}", error)); + process::exit(1); + } + } + .get_repos( + config.worktree.unwrap_or(false), + config.force_ssh.unwrap_or(false), + ) { + Ok(provider) => provider, + Err(error) => { + print_error(&format!("Error: {}", error)); + process::exit(1); + } + } + } + provider::RemoteProvider::Gitlab => { + match match provider::Gitlab::new(filter, token, config.api_url) { + Ok(provider) => provider, + Err(error) => { + print_error(&format!("Error: {}", error)); + process::exit(1); + } + } + .get_repos( + config.worktree.unwrap_or(false), + config.force_ssh.unwrap_or(false), + ) { + Ok(provider) => provider, + Err(error) => { + print_error(&format!("Error: {}", error)); + process::exit(1); + } + } + } + }; + + let mut trees = vec![]; + + for (namespace, namespace_repos) in repos { + let tree = config::Tree { + root: grm::path_as_string(&Path::new(&config.root).join(namespace)), + repos: Some(namespace_repos), + }; + trees.push(tree); + } + + let config = config::Config::from_trees(trees); + + match args.format { cmd::ConfigFormat::Toml => { let toml = match config.as_toml() { Ok(toml) => toml, @@ -148,10 +347,94 @@ fn main() { } } } - for warning in warnings { - print_warning(&warning); + cmd::FindAction::Remote(args) => { + let token = match grm::get_token_from_command(&args.token_command) { + Ok(token) => token, + Err(error) => { + print_error(&format!("Getting token from command failed: {}", error)); + process::exit(1); + } + }; + + let filter = grm::provider::Filter::new( + args.users, + args.groups, + args.owner, + args.access, + ); + + let worktree = args.worktree == "true"; + + let repos = match args.provider { + cmd::RemoteProvider::Github => { + match grm::provider::Github::new(filter, token, args.api_url) { + Ok(provider) => provider, + Err(error) => { + print_error(&format!("Error: {}", error)); + process::exit(1); + } + } + .get_repos(worktree, args.force_ssh) + } + cmd::RemoteProvider::Gitlab => { + match grm::provider::Gitlab::new(filter, token, args.api_url) { + Ok(provider) => provider, + Err(error) => { + print_error(&format!("Error: {}", error)); + process::exit(1); + } + } + .get_repos(worktree, args.force_ssh) + } + }; + + let repos = repos.unwrap_or_else(|error| { + print_error(&format!("Error: {}", error)); + process::exit(1); + }); + + let mut trees: Vec = vec![]; + + for (namespace, repolist) in repos { + let tree = config::Tree { + root: Path::new(&args.root).join(namespace).display().to_string(), + repos: Some(repolist), + }; + trees.push(tree); + } + + let config = config::Config::from_trees(trees); + + match args.format { + cmd::ConfigFormat::Toml => { + let toml = match config.as_toml() { + Ok(toml) => toml, + Err(error) => { + print_error(&format!( + "Failed converting config to TOML: {}", + &error + )); + process::exit(1); + } + }; + print!("{}", toml); + } + cmd::ConfigFormat::Yaml => { + let yaml = match config.as_yaml() { + Ok(yaml) => yaml, + Err(error) => { + print_error(&format!( + "Failed converting config to YAML: {}", + &error + )); + process::exit(1); + } + }; + print!("{}", yaml); + } + } } - } + }, }, cmd::SubCommand::Worktree(args) => { let cwd = std::env::current_dir().unwrap_or_else(|error| { diff --git a/src/lib.rs b/src/lib.rs index d598aac..95b0b77 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,5 @@ #![feature(io_error_more)] +#![feature(const_option_ext)] use std::fs; use std::path::{Path, PathBuf}; @@ -6,13 +7,14 @@ use std::process; pub mod config; pub mod output; +pub mod provider; pub mod repo; pub mod table; use config::{Config, Tree}; use output::*; -use repo::{clone_repo, detect_remote_type, Remote, RepoConfig}; +use repo::{clone_repo, detect_remote_type, Remote, RemoteType, RepoConfig}; pub use repo::{RemoteTrackingStatus, Repo, RepoErrorKind, WorktreeRemoveFailureReason}; @@ -102,36 +104,57 @@ fn expand_path(path: &Path) -> PathBuf { Path::new(&expanded_path).to_path_buf() } +pub fn get_token_from_command(command: &str) -> Result { + let output = std::process::Command::new("/usr/bin/env") + .arg("sh") + .arg("-c") + .arg(command) + .output() + .map_err(|error| format!("Failed to run token-command: {}", error))?; + + let stderr = String::from_utf8(output.stderr).map_err(|error| error.to_string())?; + let stdout = String::from_utf8(output.stdout).map_err(|error| error.to_string())?; + + if !output.status.success() { + if !stderr.is_empty() { + return Err(format!("Token command failed: {}", stderr)); + } else { + return Err(String::from("Token command failed.")); + } + } + + if !stderr.is_empty() { + return Err(format!("Token command produced stderr: {}", stderr)); + } + + if stdout.is_empty() { + return Err(String::from("Token command did not produce output")); + } + + let token = stdout + .split('\n') + .next() + .ok_or_else(|| String::from("Output did not contain any newline"))?; + + Ok(token.to_string()) +} + fn sync_repo(root_path: &Path, repo: &RepoConfig) -> Result<(), String> { let repo_path = root_path.join(&repo.name); let actual_git_directory = get_actual_git_directory(&repo_path, repo.worktree_setup); - let mut repo_handle = None; - if repo_path.exists() { if repo.worktree_setup && !actual_git_directory.exists() { return Err(String::from( "Repo already exists, but is not using a worktree setup", )); - } - repo_handle = match Repo::open(&repo_path, repo.worktree_setup) { - Ok(repo) => Some(repo), - Err(error) => { - if !repo.worktree_setup && Repo::open(&repo_path, true).is_ok() { - return Err(String::from( - "Repo already exists, but is using a worktree setup", - )); - } else { - return Err(format!("Opening repository failed: {}", error)); - } - } }; } else if matches!(&repo.remotes, None) || repo.remotes.as_ref().unwrap().is_empty() { print_repo_action( &repo.name, "Repository does not have remotes configured, initializing new", ); - repo_handle = match Repo::init(&repo_path, repo.worktree_setup) { + match Repo::init(&repo_path, repo.worktree_setup) { Ok(r) => { print_repo_success(&repo.name, "Repository created"); Some(r) @@ -139,7 +162,7 @@ fn sync_repo(root_path: &Path, repo: &RepoConfig) -> Result<(), String> { Err(e) => { return Err(format!("Repository failed during init: {}", e)); } - } + }; } else { let first = repo.remotes.as_ref().unwrap().first().unwrap(); @@ -152,11 +175,32 @@ fn sync_repo(root_path: &Path, repo: &RepoConfig) -> Result<(), String> { } }; } - if let Some(remotes) = &repo.remotes { - let repo_handle = repo_handle.unwrap_or_else(|| { - Repo::open(&repo_path, repo.worktree_setup).unwrap_or_else(|_| process::exit(1)) - }); + let repo_handle = match Repo::open(&repo_path, repo.worktree_setup) { + Ok(repo) => repo, + Err(error) => { + if !repo.worktree_setup && Repo::open(&repo_path, true).is_ok() { + return Err(String::from( + "Repo already exists, but is using a worktree setup", + )); + } else { + return Err(format!("Opening repository failed: {}", error)); + } + } + }; + + if repo.worktree_setup { + match repo_handle.default_branch() { + Ok(branch) => { + add_worktree(&repo_path, &branch.name()?, None, None, false)?; + } + Err(_error) => print_repo_error( + &repo.name, + "Could not determine default branch, skipping worktree initializtion", + ), + } + } + if let Some(remotes) = &repo.remotes { let current_remotes: Vec = repo_handle .remotes() .map_err(|error| format!("Repository failed during getting the remotes: {}", error))?; @@ -231,7 +275,7 @@ pub fn find_unmanaged_repos( pub fn sync_trees(config: Config) -> Result { let mut failures = false; - for tree in config.trees.as_vec() { + for tree in config.trees()?.as_vec() { let repos = tree.repos.unwrap_or_default(); let root_path = expand_path(Path::new(&tree.root)); diff --git a/src/provider/github.rs b/src/provider/github.rs new file mode 100644 index 0000000..536f3c5 --- /dev/null +++ b/src/provider/github.rs @@ -0,0 +1,144 @@ +use serde::Deserialize; + +use super::ApiErrorResponse; +use super::Filter; +use super::JsonError; +use super::Project; +use super::Provider; +use super::SecretToken; + +const PROVIDER_NAME: &str = "github"; +const ACCEPT_HEADER_JSON: &str = "application/vnd.github.v3+json"; +const GITHUB_API_BASEURL: &str = + option_env!("GITHUB_API_BASEURL").unwrap_or("https://api.github.com"); + +#[derive(Deserialize)] +pub struct GithubProject { + pub name: String, + pub full_name: String, + pub clone_url: String, + pub ssh_url: String, + pub private: bool, +} + +#[derive(Deserialize)] +struct GithubUser { + #[serde(rename = "login")] + pub username: String, +} + +impl Project for GithubProject { + fn name(&self) -> String { + self.name.clone() + } + + fn namespace(&self) -> String { + self.full_name + .rsplit_once('/') + .expect("Github project name did not include a namespace") + .0 + .to_string() + } + + fn ssh_url(&self) -> String { + self.ssh_url.clone() + } + + fn http_url(&self) -> String { + self.clone_url.clone() + } + + fn private(&self) -> bool { + self.private + } +} + +#[derive(Deserialize)] +pub struct GithubApiErrorResponse { + pub message: String, +} + +impl JsonError for GithubApiErrorResponse { + fn to_string(self) -> String { + self.message + } +} + +pub struct Github { + filter: Filter, + secret_token: SecretToken, +} + +impl Provider for Github { + type Project = GithubProject; + type Error = GithubApiErrorResponse; + + fn new( + filter: Filter, + secret_token: SecretToken, + api_url_override: Option, + ) -> Result { + if api_url_override.is_some() { + return Err("API URL overriding is not supported for Github".to_string()); + } + Ok(Self { + filter, + secret_token, + }) + } + + fn name(&self) -> String { + String::from(PROVIDER_NAME) + } + + fn filter(&self) -> Filter { + self.filter.clone() + } + + fn secret_token(&self) -> SecretToken { + self.secret_token.clone() + } + + fn auth_header_key() -> String { + "token".to_string() + } + + fn get_user_projects( + &self, + user: &str, + ) -> Result, ApiErrorResponse> { + self.call_list( + &format!("{GITHUB_API_BASEURL}/users/{user}/repos"), + Some(ACCEPT_HEADER_JSON), + ) + } + + fn get_group_projects( + &self, + group: &str, + ) -> Result, ApiErrorResponse> { + self.call_list( + &format!("{GITHUB_API_BASEURL}/orgs/{group}/repos?type=all"), + Some(ACCEPT_HEADER_JSON), + ) + } + + fn get_accessible_projects( + &self, + ) -> Result, ApiErrorResponse> { + self.call_list( + &format!("{GITHUB_API_BASEURL}/user/repos"), + Some(ACCEPT_HEADER_JSON), + ) + } + + fn get_current_user(&self) -> Result> { + Ok(super::call::( + &format!("{GITHUB_API_BASEURL}/user"), + &Self::auth_header_key(), + &self.secret_token(), + Some(ACCEPT_HEADER_JSON), + )? + .username) + } +} diff --git a/src/provider/gitlab.rs b/src/provider/gitlab.rs new file mode 100644 index 0000000..170141b --- /dev/null +++ b/src/provider/gitlab.rs @@ -0,0 +1,165 @@ +use serde::Deserialize; + +use super::ApiErrorResponse; +use super::Filter; +use super::JsonError; +use super::Project; +use super::Provider; +use super::SecretToken; + +const PROVIDER_NAME: &str = "gitlab"; +const ACCEPT_HEADER_JSON: &str = "application/json"; +const GITLAB_API_BASEURL: &str = option_env!("GITLAB_API_BASEURL").unwrap_or("https://gitlab.com"); + +#[derive(Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum GitlabVisibility { + Private, + Internal, + Public, +} + +#[derive(Deserialize)] +pub struct GitlabProject { + #[serde(rename = "path")] + pub name: String, + pub path_with_namespace: String, + pub http_url_to_repo: String, + pub ssh_url_to_repo: String, + pub visibility: GitlabVisibility, +} + +#[derive(Deserialize)] +struct GitlabUser { + pub username: String, +} + +impl Project for GitlabProject { + fn name(&self) -> String { + self.name.clone() + } + + fn namespace(&self) -> String { + self.path_with_namespace + .rsplit_once('/') + .expect("Gitlab project name did not include a namespace") + .0 + .to_string() + } + + fn ssh_url(&self) -> String { + self.ssh_url_to_repo.clone() + } + + fn http_url(&self) -> String { + self.http_url_to_repo.clone() + } + + fn private(&self) -> bool { + matches!(self.visibility, GitlabVisibility::Private) + } +} + +#[derive(Deserialize)] +pub struct GitlabApiErrorResponse { + #[serde(alias = "error_description")] + pub message: String, +} + +impl JsonError for GitlabApiErrorResponse { + fn to_string(self) -> String { + self.message + } +} + +pub struct Gitlab { + filter: Filter, + secret_token: SecretToken, + api_url_override: Option, +} + +impl Gitlab { + fn api_url(&self) -> String { + self.api_url_override + .as_ref() + .unwrap_or(&GITLAB_API_BASEURL.to_string()) + .trim_end_matches('/') + .to_string() + } +} + +impl Provider for Gitlab { + type Project = GitlabProject; + type Error = GitlabApiErrorResponse; + + fn new( + filter: Filter, + secret_token: SecretToken, + api_url_override: Option, + ) -> Result { + Ok(Self { + filter, + secret_token, + api_url_override, + }) + } + + fn name(&self) -> String { + String::from(PROVIDER_NAME) + } + + fn filter(&self) -> Filter { + self.filter.clone() + } + + fn secret_token(&self) -> SecretToken { + self.secret_token.clone() + } + + fn auth_header_key() -> String { + "bearer".to_string() + } + + fn get_user_projects( + &self, + user: &str, + ) -> Result, ApiErrorResponse> { + self.call_list( + &format!("{}/api/v4/users/{}/projects", self.api_url(), user), + Some(ACCEPT_HEADER_JSON), + ) + } + + fn get_group_projects( + &self, + group: &str, + ) -> Result, ApiErrorResponse> { + self.call_list( + &format!( + "{}/api/v4/groups/{}/projects?include_subgroups=true&archived=false", + self.api_url(), + group + ), + Some(ACCEPT_HEADER_JSON), + ) + } + + fn get_accessible_projects( + &self, + ) -> Result, ApiErrorResponse> { + self.call_list( + &format!("{}/api/v4/projects", self.api_url(),), + Some(ACCEPT_HEADER_JSON), + ) + } + + fn get_current_user(&self) -> Result> { + Ok(super::call::( + &format!("{}/api/v4/user", self.api_url()), + &Self::auth_header_key(), + &self.secret_token(), + Some(ACCEPT_HEADER_JSON), + )? + .username) + } +} diff --git a/src/provider/mod.rs b/src/provider/mod.rs new file mode 100644 index 0000000..1811f8c --- /dev/null +++ b/src/provider/mod.rs @@ -0,0 +1,340 @@ +use serde::{Deserialize, Serialize}; + +// Required to use the `json()` method from the trait +use isahc::ReadResponseExt; + +pub mod github; +pub mod gitlab; + +pub use github::Github; +pub use gitlab::Gitlab; + +use crate::{Remote, RemoteType, RepoConfig}; + +use std::collections::HashMap; + +#[derive(Debug, Deserialize, Serialize, clap::ArgEnum, Clone)] +pub enum RemoteProvider { + #[serde(alias = "github", alias = "GitHub")] + Github, + #[serde(alias = "gitlab", alias = "GitLab")] + Gitlab, +} + +#[derive(Deserialize)] +#[serde(untagged)] +enum ProjectResponse { + Success(Vec), + Failure(U), +} + +pub trait Project { + fn into_repo_config( + self, + provider_name: &str, + worktree_setup: bool, + force_ssh: bool, + ) -> RepoConfig + where + Self: Sized, + { + RepoConfig { + name: self.name(), + worktree_setup, + remotes: Some(vec![Remote { + name: String::from(provider_name), + url: if force_ssh || self.private() { + self.ssh_url() + } else { + self.http_url() + }, + remote_type: if force_ssh || self.private() { + RemoteType::Ssh + } else { + RemoteType::Https + }, + }]), + } + } + + fn name(&self) -> String; + fn namespace(&self) -> String; + fn ssh_url(&self) -> String; + fn http_url(&self) -> String; + fn private(&self) -> bool; +} + +type SecretToken = String; + +#[derive(Clone)] +pub struct Filter { + users: Vec, + groups: Vec, + owner: bool, + access: bool, +} + +impl Filter { + pub fn new(users: Vec, groups: Vec, owner: bool, access: bool) -> Self { + Filter { + users, + groups, + owner, + access, + } + } +} + +pub enum ApiErrorResponse +where + T: JsonError, +{ + Json(T), + String(String), +} + +impl From for ApiErrorResponse +where + T: JsonError, +{ + fn from(s: String) -> ApiErrorResponse { + ApiErrorResponse::String(s) + } +} + +pub trait JsonError { + fn to_string(self) -> String; +} + +pub trait Provider { + type Project: serde::de::DeserializeOwned + Project; + type Error: serde::de::DeserializeOwned + JsonError; + + fn new( + filter: Filter, + secret_token: SecretToken, + api_url_override: Option, + ) -> Result + where + Self: Sized; + + fn name(&self) -> String; + fn filter(&self) -> Filter; + fn secret_token(&self) -> SecretToken; + fn auth_header_key() -> String; + + fn get_user_projects( + &self, + user: &str, + ) -> Result, ApiErrorResponse>; + + fn get_group_projects( + &self, + group: &str, + ) -> Result, ApiErrorResponse>; + + fn get_own_projects(&self) -> Result, ApiErrorResponse> { + self.get_user_projects(&self.get_current_user()?) + } + + fn get_accessible_projects(&self) -> Result, ApiErrorResponse>; + + fn get_current_user(&self) -> Result>; + + /// + /// Calls the API at specific uri and expects a successful response of Vec back, or an error + /// response U + /// + /// Handles paging with "link" HTTP headers properly and reads all pages to + /// the end. + fn call_list( + &self, + uri: &str, + accept_header: Option<&str>, + ) -> Result, ApiErrorResponse> { + let mut results = vec![]; + + let client = isahc::HttpClient::new().map_err(|error| error.to_string())?; + + let request = isahc::Request::builder() + .uri(uri) + .method("GET") + .header("accept", accept_header.unwrap_or("application/json")) + .header( + "authorization", + format!("{} {}", Self::auth_header_key(), &self.secret_token()), + ) + .body(()) + .map_err(|error| error.to_string())?; + + let mut response = client + .send(request) + .map_err(|error| ApiErrorResponse::String(error.to_string()))?; + + if !response.status().is_success() { + let r: Self::Error = response + .json() + .map_err(|error| format!("Failed deserializing error response: {}", error))?; + return Err(ApiErrorResponse::Json(r)); + } + + let result: Vec = response + .json() + .map_err(|error| format!("Failed deserializing response: {}", error))?; + + results.extend(result); + + if let Some(link_header) = response.headers().get("link") { + let link_header = link_header.to_str().map_err(|error| error.to_string())?; + + let link_header = + parse_link_header::parse(link_header).map_err(|error| error.to_string())?; + + let next_page = link_header.get(&Some(String::from("next"))); + + if let Some(page) = next_page { + let following_repos = self.call_list(&page.raw_uri, accept_header)?; + results.extend(following_repos); + } + } + + Ok(results) + } + + fn get_repos( + &self, + worktree_setup: bool, + force_ssh: bool, + ) -> Result>, String> { + let mut repos = vec![]; + + if self.filter().owner { + repos.extend(self.get_own_projects().map_err(|error| match error { + ApiErrorResponse::Json(x) => x.to_string(), + ApiErrorResponse::String(s) => s, + })?); + } + + if self.filter().access { + let accessible_projects = + self.get_accessible_projects() + .map_err(|error| match error { + ApiErrorResponse::Json(x) => x.to_string(), + ApiErrorResponse::String(s) => s, + })?; + + for accessible_project in accessible_projects { + let mut already_present = false; + for repo in &repos { + if repo.name() == accessible_project.name() + && repo.namespace() == accessible_project.namespace() + { + already_present = true; + } + } + if !already_present { + repos.push(accessible_project); + } + } + } + + for user in &self.filter().users { + let user_projects = self.get_user_projects(user).map_err(|error| match error { + ApiErrorResponse::Json(x) => x.to_string(), + ApiErrorResponse::String(s) => s, + })?; + + for user_project in user_projects { + let mut already_present = false; + for repo in &repos { + if repo.name() == user_project.name() + && repo.namespace() == user_project.namespace() + { + already_present = true; + } + } + if !already_present { + repos.push(user_project); + } + } + } + + for group in &self.filter().groups { + let group_projects = self + .get_group_projects(group) + .map_err(|error| match error { + ApiErrorResponse::Json(x) => x.to_string(), + ApiErrorResponse::String(s) => s, + })?; + for group_project in group_projects { + let mut already_present = false; + for repo in &repos { + if repo.name() == group_project.name() + && repo.namespace() == group_project.namespace() + { + already_present = true; + } + } + + if !already_present { + repos.push(group_project); + } + } + } + + let mut ret: HashMap> = HashMap::new(); + + for repo in repos { + let namespace = repo.namespace().clone(); + + let repo = repo.into_repo_config(&self.name(), worktree_setup, force_ssh); + + ret.entry(namespace).or_insert(vec![]).push(repo); + } + + Ok(ret) + } +} + +fn call( + uri: &str, + auth_header_key: &str, + secret_token: &str, + accept_header: Option<&str>, +) -> Result> +where + T: serde::de::DeserializeOwned, + U: serde::de::DeserializeOwned + JsonError, +{ + let client = isahc::HttpClient::new().map_err(|error| error.to_string())?; + + let request = isahc::Request::builder() + .uri(uri) + .header("accept", accept_header.unwrap_or("application/json")) + .header( + "authorization", + format!("{} {}", &auth_header_key, &secret_token), + ) + .body(()) + .map_err(|error| ApiErrorResponse::String(error.to_string()))?; + + let mut response = client + .send(request) + .map_err(|error| ApiErrorResponse::String(error.to_string()))?; + + let success = response.status().is_success(); + + if !success { + let response: U = response + .json() + .map_err(|error| format!("Failed deserializing error response: {}", error))?; + + return Err(ApiErrorResponse::Json(response)); + } + + let response: T = response + .json() + .map_err(|error| format!("Failed deserializing response: {}", error))?; + + Ok(response) +} diff --git a/src/table.rs b/src/table.rs index fcea35f..b395e20 100644 --- a/src/table.rs +++ b/src/table.rs @@ -144,7 +144,7 @@ pub fn get_worktree_status_table( pub fn get_status_table(config: crate::Config) -> Result<(Vec, Vec), String> { let mut errors = Vec::new(); let mut tables = Vec::new(); - for tree in config.trees.as_vec() { + for tree in config.trees()?.as_vec() { let repos = tree.repos.unwrap_or_default(); let root_path = crate::expand_path(Path::new(&tree.root)); From 881a33dc96a00a3cdb0fa8952dc2c5171f372648 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Thu, 26 May 2022 15:20:48 +0200 Subject: [PATCH 12/36] e2e: Add tests for worktree initialization --- e2e_tests/test_repos_sync.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/e2e_tests/test_repos_sync.py b/e2e_tests/test_repos_sync.py index 67cdf3c..91230c8 100644 --- a/e2e_tests/test_repos_sync.py +++ b/e2e_tests/test_repos_sync.py @@ -500,7 +500,8 @@ def test_repos_sync_normal_change_remote_name(configtype): @pytest.mark.parametrize("configtype", ["toml", "yaml"]) -def test_repos_sync_worktree_clone(configtype): +@pytest.mark.parametrize("init_worktree", [True, False, "default"]) +def test_repos_sync_worktree_clone(configtype, init_worktree): with tempfile.TemporaryDirectory() as target: with TempGitFileRemote() as (remote, head_commit_sha): with tempfile.NamedTemporaryFile() as config: @@ -511,13 +512,22 @@ def test_repos_sync_worktree_clone(configtype): ) ) - cmd = grm(["repos", "sync", "config", "--config", config.name]) + args = ["repos", "sync", "config", "--config", config.name] + if init_worktree is True: + args.append("--init-worktree=true") + if init_worktree is False: + args.append("--init-worktree=false") + + cmd = grm(args) assert cmd.returncode == 0 worktree_dir = f"{target}/test" assert os.path.exists(worktree_dir) - assert set(os.listdir(worktree_dir)) == {".git-main-working-tree"} + if init_worktree is True or init_worktree == "default": + assert set(os.listdir(worktree_dir)) == {".git-main-working-tree", "master"} + else: + assert set(os.listdir(worktree_dir)) == {".git-main-working-tree"} with git.Repo( os.path.join(worktree_dir, ".git-main-working-tree") From f2f1d5bcaf9d0f378c2fe73a42730a67f4d2a018 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Mon, 24 Jan 2022 17:24:24 +0100 Subject: [PATCH 13/36] Fix worktree initialization --- src/grm/cmd.rs | 22 ++++++++++++++++++++++ src/grm/main.rs | 4 ++-- src/lib.rs | 8 ++++---- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/grm/cmd.rs b/src/grm/cmd.rs index 27a1c82..3145b7d 100644 --- a/src/grm/cmd.rs +++ b/src/grm/cmd.rs @@ -170,6 +170,17 @@ pub struct Config { help = "Path to the configuration file" )] pub config: String, + + #[clap( + long, + help = "Check out the default worktree after clone", + possible_values = &["true", "false"], + default_value = "true", + default_missing_value = "true", + min_values = 0, + max_values = 1, + )] + pub init_worktree: String, } pub type RemoteProvider = grm::provider::RemoteProvider; @@ -224,6 +235,17 @@ pub struct SyncRemoteArgs { #[clap(long, help = "Base URL for the API")] pub api_url: Option, + + #[clap( + long, + help = "Check out the default worktree after clone", + possible_values = &["true", "false"], + default_value = "true", + default_missing_value = "true", + min_values = 0, + max_values = 1, + )] + pub init_worktree: String, } #[derive(Parser)] diff --git a/src/grm/main.rs b/src/grm/main.rs index 40a8652..3f202f0 100644 --- a/src/grm/main.rs +++ b/src/grm/main.rs @@ -23,7 +23,7 @@ fn main() { process::exit(1); } }; - match grm::sync_trees(config) { + match grm::sync_trees(config, args.init_worktree == "true") { Ok(success) => { if !success { process::exit(1) @@ -93,7 +93,7 @@ fn main() { let config = config::Config::from_trees(trees); - match grm::sync_trees(config) { + match grm::sync_trees(config, args.init_worktree == "true") { Ok(success) => { if !success { process::exit(1) diff --git a/src/lib.rs b/src/lib.rs index 95b0b77..fcda502 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -139,7 +139,7 @@ pub fn get_token_from_command(command: &str) -> Result { Ok(token.to_string()) } -fn sync_repo(root_path: &Path, repo: &RepoConfig) -> Result<(), String> { +fn sync_repo(root_path: &Path, repo: &RepoConfig, init_worktree: bool) -> Result<(), String> { let repo_path = root_path.join(&repo.name); let actual_git_directory = get_actual_git_directory(&repo_path, repo.worktree_setup); @@ -189,7 +189,7 @@ fn sync_repo(root_path: &Path, repo: &RepoConfig) -> Result<(), String> { } }; - if repo.worktree_setup { + if repo.worktree_setup && init_worktree { match repo_handle.default_branch() { Ok(branch) => { add_worktree(&repo_path, &branch.name()?, None, None, false)?; @@ -273,7 +273,7 @@ pub fn find_unmanaged_repos( Ok(unmanaged_repos) } -pub fn sync_trees(config: Config) -> Result { +pub fn sync_trees(config: Config, init_worktree: bool) -> Result { let mut failures = false; for tree in config.trees()?.as_vec() { let repos = tree.repos.unwrap_or_default(); @@ -281,7 +281,7 @@ pub fn sync_trees(config: Config) -> Result { let root_path = expand_path(Path::new(&tree.root)); for repo in &repos { - match sync_repo(&root_path, repo) { + match sync_repo(&root_path, repo, init_worktree) { Ok(_) => print_repo_success(&repo.name, "OK"), Err(error) => { print_repo_error(&repo.name, &error); From ad206297d8762e90a2f8efe23a963573a4c31f23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Thu, 26 May 2022 15:23:50 +0200 Subject: [PATCH 14/36] e2e: Test sync twice to verify no changes --- e2e_tests/test_repos_sync.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/e2e_tests/test_repos_sync.py b/e2e_tests/test_repos_sync.py index 91230c8..1f01fdd 100644 --- a/e2e_tests/test_repos_sync.py +++ b/e2e_tests/test_repos_sync.py @@ -518,24 +518,30 @@ def test_repos_sync_worktree_clone(configtype, init_worktree): if init_worktree is False: args.append("--init-worktree=false") - cmd = grm(args) - assert cmd.returncode == 0 + for i in [1, 2]: + cmd = grm(args) + assert cmd.returncode == 0 - worktree_dir = f"{target}/test" - assert os.path.exists(worktree_dir) + worktree_dir = f"{target}/test" + assert os.path.exists(worktree_dir) - if init_worktree is True or init_worktree == "default": - assert set(os.listdir(worktree_dir)) == {".git-main-working-tree", "master"} - else: - assert set(os.listdir(worktree_dir)) == {".git-main-working-tree"} + if init_worktree is True or init_worktree == "default": + assert set(os.listdir(worktree_dir)) == { + ".git-main-working-tree", + "master", + } + else: + assert set(os.listdir(worktree_dir)) == { + ".git-main-working-tree" + } - with git.Repo( - os.path.join(worktree_dir, ".git-main-working-tree") - ) as repo: - assert repo.bare - assert set([str(r) for r in repo.remotes]) == {"origin"} - assert str(repo.active_branch) == "master" - assert str(repo.head.commit) == head_commit_sha + with git.Repo( + os.path.join(worktree_dir, ".git-main-working-tree") + ) as repo: + assert repo.bare + assert set([str(r) for r in repo.remotes]) == {"origin"} + assert str(repo.active_branch) == "master" + assert str(repo.head.commit) == head_commit_sha @pytest.mark.parametrize("configtype", ["toml", "yaml"]) From 664d44eddc8a94d91b26066989036869b3e23408 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Thu, 26 May 2022 15:24:45 +0200 Subject: [PATCH 15/36] Only initialize worktrees for actually cloned repos --- src/lib.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index fcda502..ef5bf1a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -143,6 +143,8 @@ fn sync_repo(root_path: &Path, repo: &RepoConfig, init_worktree: bool) -> Result let repo_path = root_path.join(&repo.name); let actual_git_directory = get_actual_git_directory(&repo_path, repo.worktree_setup); + let mut newly_created = false; + if repo_path.exists() { if repo.worktree_setup && !actual_git_directory.exists() { return Err(String::from( @@ -174,6 +176,8 @@ fn sync_repo(root_path: &Path, repo: &RepoConfig, init_worktree: bool) -> Result return Err(format!("Repository failed during clone: {}", e)); } }; + + newly_created = true; } let repo_handle = match Repo::open(&repo_path, repo.worktree_setup) { @@ -181,15 +185,15 @@ fn sync_repo(root_path: &Path, repo: &RepoConfig, init_worktree: bool) -> Result Err(error) => { if !repo.worktree_setup && Repo::open(&repo_path, true).is_ok() { return Err(String::from( - "Repo already exists, but is using a worktree setup", - )); + "Repo already exists, but is using a worktree setup", + )); } else { return Err(format!("Opening repository failed: {}", error)); } } }; - if repo.worktree_setup && init_worktree { + if newly_created && repo.worktree_setup && init_worktree { match repo_handle.default_branch() { Ok(branch) => { add_worktree(&repo_path, &branch.name()?, None, None, false)?; From 127dd0535ede8e00cc0a51618fb60cfefe3456ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Sat, 14 May 2022 18:42:31 +0200 Subject: [PATCH 16/36] Normalize paths when printing configuration --- src/config.rs | 22 +++++++++++++++++++++- src/grm/main.rs | 8 ++++++-- src/lib.rs | 9 --------- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/config.rs b/src/config.rs index 36873fe..923a906 100644 --- a/src/config.rs +++ b/src/config.rs @@ -4,7 +4,6 @@ use std::process; use crate::output::*; use super::repo::RepoConfig; - use std::path::Path; use crate::get_token_from_command; @@ -147,6 +146,27 @@ impl Config { }) } + pub fn normalize(&mut self) { + if let Config::ConfigTree(config) = self { + let home = super::env_home().display().to_string(); + for tree in &mut config.trees.0 { + if tree.root.starts_with(&home) { + // The tilde is not handled differently, it's just a normal path component for `Path`. + // Therefore we can treat it like that during **output**. + // + // The `unwrap()` is safe here as we are testing via `starts_with()` + // beforehand + let mut path = tree.root.strip_prefix(&home).unwrap(); + if path.starts_with('/') { + path = path.strip_prefix('/').unwrap(); + } + + tree.root = Path::new("~").join(path).display().to_string(); + } + } + } + } + pub fn as_toml(&self) -> Result { match toml::to_string(self) { Ok(toml) => Ok(toml), diff --git a/src/grm/main.rs b/src/grm/main.rs index 3f202f0..7b5f54e 100644 --- a/src/grm/main.rs +++ b/src/grm/main.rs @@ -198,7 +198,9 @@ fn main() { }) { print_warning("No repositories found"); } else { - let config = trees.to_config(); + let mut config = trees.to_config(); + + config.normalize(); match args.format { cmd::ConfigFormat::Toml => { @@ -403,7 +405,9 @@ fn main() { trees.push(tree); } - let config = config::Config::from_trees(trees); + let mut config = config::Config::from_trees(trees); + + config.normalize(); match args.format { cmd::ConfigFormat::Toml => { diff --git a/src/lib.rs b/src/lib.rs index ef5bf1a..809778c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -487,15 +487,6 @@ pub fn find_in_tree(path: &Path) -> Result<(Tree, Vec), String> { } } } - let home = env_home(); - if root.starts_with(&home) { - // The tilde is not handled differently, it's just a normal path component for `Path`. - // Therefore we can treat it like that during **output**. - // - // The `unwrap()` is safe here as we are testing via `starts_with()` - // beforehand - root = Path::new("~").join(root.strip_prefix(&home).unwrap()); - } Ok(( Tree { From 3e8aad2221be2ed3d641a9ed6c21d4ecee84be93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Fri, 20 May 2022 00:39:42 +0200 Subject: [PATCH 17/36] Format cargo update script with black --- depcheck/update-cargo-dependencies.py | 40 ++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/depcheck/update-cargo-dependencies.py b/depcheck/update-cargo-dependencies.py index 4ebadbc..e69e74d 100755 --- a/depcheck/update-cargo-dependencies.py +++ b/depcheck/update-cargo-dependencies.py @@ -62,7 +62,10 @@ for tier in ["dependencies", "dev-dependencies"]: for version_entry in open(info_file, "r").readlines(): version = semver.VersionInfo.parse(json.loads(version_entry)["vers"]) if latest_version is None or version > latest_version: - if current_version.prerelease is None and version.prerelease is not None: + if ( + current_version.prerelease is None + and version.prerelease is not None + ): # skip prereleases, except when we are on a prerelease already print(f"{name}: Skipping prerelease version {version}") continue @@ -91,7 +94,15 @@ for tier in ["dependencies", "dev-dependencies"]: try: cmd = subprocess.run( - ["cargo", "update", "-Z", "no-index-update", "--aggressive", "--package", name], + [ + "cargo", + "update", + "-Z", + "no-index-update", + "--aggressive", + "--package", + name, + ], check=True, capture_output=True, text=True, @@ -103,9 +114,16 @@ for tier in ["dependencies", "dev-dependencies"]: message = f"dependencies: Update {name} to {latest_version}" subprocess.run( - ["git", "commit", "--message", message, "../Cargo.toml", "../Cargo.lock"], + [ + "git", + "commit", + "--message", + message, + "../Cargo.toml", + "../Cargo.lock", + ], check=True, - capture_output=True + capture_output=True, ) @@ -114,11 +132,19 @@ for tier in ["dependencies", "dev-dependencies"]: while True: with open("../Cargo.lock", "r") as f: cargo_lock = tomlkit.parse(f.read()) - for package in cargo_lock['package']: + for package in cargo_lock["package"]: spec = f"{package['name']}:{package['version']}" try: cmd = subprocess.run( - ["cargo", "update", "-Z", "no-index-update", "--aggressive", "--package", spec], + [ + "cargo", + "update", + "-Z", + "no-index-update", + "--aggressive", + "--package", + spec, + ], check=True, capture_output=True, text=True, @@ -134,7 +160,7 @@ while True: cmd = subprocess.run( ["git", "commit", "--message", message, "../Cargo.lock"], check=True, - capture_output=True + capture_output=True, ) break else: From c994c9024752e02c104cc5cf988735ba0083645a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Fri, 20 May 2022 17:45:44 +0200 Subject: [PATCH 18/36] Justfile: Remove Cargo.lock check --- Justfile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Justfile b/Justfile index 84e7886..9503515 100644 --- a/Justfile +++ b/Justfile @@ -1,13 +1,10 @@ set positional-arguments -check: check-cargo-lock test +check: test cargo check cargo fmt --check cargo clippy --no-deps -- -Dwarnings -check-cargo-lock: - cargo update --locked - lint-fix: cargo clippy --no-deps --fix From af45b136122392813e1ae6dd4088dcfd2ef79ba5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Thu, 26 May 2022 16:06:05 +0200 Subject: [PATCH 19/36] Justfile: Add target for formatting --- Justfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Justfile b/Justfile index 9503515..c67d7c3 100644 --- a/Justfile +++ b/Justfile @@ -5,6 +5,10 @@ check: test cargo fmt --check cargo clippy --no-deps -- -Dwarnings +fmt: + cargo fmt + git ls-files | grep '\.py$' | xargs black + lint-fix: cargo clippy --no-deps --fix From 1db3eadd4c3b0fde3a340f918d564d2401c1fbfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Thu, 26 May 2022 16:18:17 +0200 Subject: [PATCH 20/36] Fix formatting --- src/grm/cmd.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/grm/cmd.rs b/src/grm/cmd.rs index 3145b7d..89340e9 100644 --- a/src/grm/cmd.rs +++ b/src/grm/cmd.rs @@ -42,9 +42,7 @@ pub enum ReposAction { #[derive(Parser)] #[clap(about = "Sync local repositories with a configured list")] pub enum SyncAction { - #[clap( - about = "Synchronize the repositories to the configured values" - )] + #[clap(about = "Synchronize the repositories to the configured values")] Config(Config), #[clap(about = "Synchronize the repositories from a remote provider")] Remote(SyncRemoteArgs), From 50a0f4d76630369fde8898ad288a58bd3051a842 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Thu, 26 May 2022 16:18:56 +0200 Subject: [PATCH 21/36] Fail properly when default branch cannot be detected --- src/repo.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/repo.rs b/src/repo.rs index a632cb4..d369708 100644 --- a/src/repo.rs +++ b/src/repo.rs @@ -1271,14 +1271,14 @@ impl Repo { }; let default_branch_name = match &config { - None => guess_default_branch()?, + None => guess_default_branch().ok(), Some(config) => match &config.persistent_branches { - None => guess_default_branch()?, + None => guess_default_branch().ok(), Some(persistent_branches) => { if persistent_branches.is_empty() { - guess_default_branch()? + guess_default_branch().ok() } else { - persistent_branches[0].clone() + Some(persistent_branches[0].clone()) } } }, @@ -1290,8 +1290,10 @@ impl Repo { if dirname == WORKTREE_CONFIG_FILE_NAME { continue; } - if dirname == default_branch_name { - continue; + if let Some(default_branch_name) = default_branch_name { + if dirname == default_branch_name { + continue; + } } if !&worktrees.iter().any(|worktree| worktree.name() == dirname) { unmanaged_worktrees.push(dirname); From 35e7c34d11fe8d445b29b7585c23d5a4f42ec54e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Thu, 26 May 2022 16:19:10 +0200 Subject: [PATCH 22/36] Do not panic when finding unmanaged worktrees fails --- src/table.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/table.rs b/src/table.rs index b395e20..fa7cac1 100644 --- a/src/table.rs +++ b/src/table.rs @@ -132,7 +132,7 @@ pub fn get_worktree_status_table( )); } } - for worktree in Repo::find_unmanaged_worktrees(repo, directory).unwrap() { + for worktree in Repo::find_unmanaged_worktrees(repo, directory)? { errors.push(format!( "Found {}, which is not a valid worktree directory!", &worktree From 433dc090e0c35e56bb22c7a0b234c7b73ea5ae46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Thu, 26 May 2022 16:20:42 +0200 Subject: [PATCH 23/36] Prefix shell commands with dollar sign --- docs/src/worktrees.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/src/worktrees.md b/docs/src/worktrees.md index 72258f8..81cae45 100644 --- a/docs/src/worktrees.md +++ b/docs/src/worktrees.md @@ -269,7 +269,7 @@ wt convert`. This command has to be run in the root of the repository you want to convert: ``` -grm wt convert +$ grm wt convert [✔] Conversion successful ``` @@ -285,7 +285,7 @@ To fetch all remote references from all remotes in a worktree setup, you can use the following command: ``` -grm wt fetch +$ grm wt fetch [✔] Fetched from all remotes ``` @@ -295,7 +295,7 @@ Often, you may want to pull all remote changes into your worktrees. For this, use the `git pull` equivalent: ``` -grm wt pull +$ grm wt pull [✔] master: Done [✔] my-cool-branch: Done ``` @@ -304,7 +304,7 @@ This will refuse when there are local changes, or if the branch cannot be fast forwarded. If you want to rebase your local branches, use the `--rebase` switch: ``` -grm wt pull --rebase +$ grm wt pull --rebase [✔] master: Done [✔] my-cool-branch: Done ``` @@ -319,7 +319,7 @@ for persistent branches that change on the remote side. There is a similar rebase feature that rebases onto the **default** branch instead: ``` -grm wt rebase +$ grm wt rebase [✔] master: Done [✔] my-cool-branch: Done ``` @@ -331,7 +331,7 @@ use the `--pull` flag, and `--rebase` if you want to rebase instead of aborting on non-fast-forwards: ``` -grm wt rebase --pull --rebase +$ grm wt rebase --pull --rebase [✔] master: Done [✔] my-cool-branch: Done ``` From 10e02c20a1b844aacc36d55e29b2d785f3416af2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Thu, 26 May 2022 16:22:39 +0200 Subject: [PATCH 24/36] e2e: Add tests for nested repository checkouts --- e2e_tests/test_repos_sync.py | 89 ++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/e2e_tests/test_repos_sync.py b/e2e_tests/test_repos_sync.py index 1f01fdd..998d927 100644 --- a/e2e_tests/test_repos_sync.py +++ b/e2e_tests/test_repos_sync.py @@ -133,6 +133,50 @@ templates = { """ ), }, + "nested_trees": { + "toml": """ + [[trees]] + root = "{root}" + + [[trees.repos]] + name = "outer" + + [[trees.repos.remotes]] + name = "origin" + url = "file://{remote1}" + type = "file" + + [[trees]] + root = "{root}/subdir" + + [[trees.repos]] + name = "inner" + + [[trees.repos.remotes]] + name = "origin" + url = "file://{remote2}" + type = "file" + """, + "yaml": textwrap.dedent( + """ + trees: + - root: "{root}" + repos: + - name: outer + remotes: + - name: origin + url: "file://{remote1}" + type: "file" + - root: "{root}/subdir" + repos: + - name: inner + remotes: + - name: origin + url: "file://{remote2}" + type: "file" + """ + ), + }, } @@ -275,6 +319,51 @@ def test_repos_sync_normal_clone(configtype): assert urls[0] == f"file://{remote2}" +@pytest.mark.parametrize("configtype", ["toml", "yaml"]) +def test_repos_sync_nested_clone(configtype): + with tempfile.TemporaryDirectory() as target: + with TempGitFileRemote() as (remote1, remote1_head_commit_sha): + with TempGitFileRemote() as (remote2, remote2_head_commit_sha): + with tempfile.NamedTemporaryFile() as config: + with open(config.name, "w") as f: + f.write( + templates["nested_trees"][configtype].format( + root=target, remote1=remote1, remote2=remote2 + ) + ) + + cmd = grm(["repos", "sync", "config", "--config", config.name]) + assert cmd.returncode == 0 + + def validate(git_dir, sha, remote): + assert os.path.exists(git_dir) + with git.Repo(git_dir) as repo: + assert not repo.bare + assert not repo.is_dirty() + assert set([str(r) for r in repo.remotes]) == {"origin"} + assert str(repo.active_branch) == "master" + assert str(repo.head.commit) == sha + + assert len(repo.remotes) == 1 + urls = list(repo.remote("origin").urls) + assert len(urls) == 1 + assert urls[0] == f"file://{remote}" + + validate( + os.path.join(target, "outer"), remote1_head_commit_sha, remote1 + ) + validate( + os.path.join(target, "subdir", "inner"), + remote2_head_commit_sha, + remote2, + ) + + cmd = grm(["repos", "sync", "config", "--config", config.name]) + print(cmd.stdout) + print(cmd.stderr) + assert not "found unmanaged repository" in cmd.stderr.lower() + + @pytest.mark.parametrize("configtype", ["toml", "yaml"]) def test_repos_sync_normal_init(configtype): with tempfile.TemporaryDirectory() as target: From 6ef759a14eff56853766971274a928a8bcc84bcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Thu, 26 May 2022 16:30:11 +0200 Subject: [PATCH 25/36] Separate config structs from internal structs --- src/config.rs | 145 ++++++++++++++++++++++++++++++++++++-------- src/grm/main.rs | 54 ++++++++++------- src/lib.rs | 53 ++++++++++------ src/provider/mod.rs | 15 ++--- src/repo.rs | 37 +++++------ src/table.rs | 20 +++--- tests/repo.rs | 8 +-- 7 files changed, 217 insertions(+), 115 deletions(-) diff --git a/src/config.rs b/src/config.rs index 923a906..2a5648d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -3,27 +3,32 @@ use std::process; use crate::output::*; -use super::repo::RepoConfig; use std::path::Path; -use crate::get_token_from_command; +use crate::{get_token_from_command, Remote, Repo, Tree}; + use crate::provider; use crate::provider::Filter; use crate::provider::Provider; pub type RemoteProvider = crate::provider::RemoteProvider; +pub type RemoteType = crate::repo::RemoteType; + +fn worktree_setup_default() -> bool { + false +} #[derive(Debug, Serialize, Deserialize)] #[serde(untagged)] pub enum Config { - ConfigTree(ConfigTree), + ConfigTrees(ConfigTrees), ConfigProvider(ConfigProvider), } #[derive(Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -pub struct ConfigTree { - pub trees: Trees, +pub struct ConfigTrees { + pub trees: Vec, } #[derive(Debug, Serialize, Deserialize)] @@ -50,30 +55,100 @@ pub struct ConfigProvider { } #[derive(Debug, Serialize, Deserialize)] -pub struct Trees(Vec); +#[serde(deny_unknown_fields)] +pub struct RemoteConfig { + pub name: String, + pub url: String, + #[serde(rename = "type")] + pub remote_type: RemoteType, +} -impl Trees { +impl RemoteConfig { + pub fn from_remote(remote: Remote) -> Self { + Self { + name: remote.name, + url: remote.url, + remote_type: remote.remote_type, + } + } + + pub fn into_remote(self) -> Remote { + Remote { + name: self.name, + url: self.url, + remote_type: self.remote_type, + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RepoConfig { + pub name: String, + + #[serde(default = "worktree_setup_default")] + pub worktree_setup: bool, + + pub remotes: Option>, +} + +impl RepoConfig { + pub fn from_repo(repo: Repo) -> Self { + Self { + name: repo.name, + worktree_setup: repo.worktree_setup, + remotes: repo + .remotes + .map(|remotes| remotes.into_iter().map(RemoteConfig::from_remote).collect()), + } + } + + pub fn into_repo(self) -> Repo { + Repo { + name: self.name, + worktree_setup: self.worktree_setup, + remotes: self.remotes.map(|remotes| { + remotes + .into_iter() + .map(|remote| remote.into_remote()) + .collect() + }), + } + } +} + +impl ConfigTrees { pub fn to_config(self) -> Config { - Config::ConfigTree(ConfigTree { trees: self }) + Config::ConfigTrees(self) } - pub fn from_vec(vec: Vec) -> Self { - Trees(vec) + pub fn from_vec(vec: Vec) -> Self { + ConfigTrees { trees: vec } } - pub fn as_vec(self) -> Vec { - self.0 + pub fn from_trees(vec: Vec) -> Self { + ConfigTrees { + trees: vec.into_iter().map(ConfigTree::from_tree).collect(), + } } - pub fn as_vec_ref(&self) -> &Vec { - self.0.as_ref() + pub fn trees(self) -> Vec { + self.trees + } + + pub fn trees_mut(&mut self) -> &mut Vec { + &mut self.trees + } + + pub fn trees_ref(&self) -> &Vec { + self.trees.as_ref() } } impl Config { - pub fn trees(self) -> Result { + pub fn trees(self) -> Result, String> { match self { - Config::ConfigTree(config) => Ok(config.trees), + Config::ConfigTrees(config) => Ok(config.trees), Config::ConfigProvider(config) => { let token = match get_token_from_command(&config.token_command) { Ok(token) => token, @@ -129,27 +204,29 @@ impl Config { let mut trees = vec![]; for (namespace, namespace_repos) in repos { - let tree = Tree { + let repos = namespace_repos + .into_iter() + .map(RepoConfig::from_repo) + .collect(); + let tree = ConfigTree { root: crate::path_as_string(&Path::new(&config.root).join(namespace)), - repos: Some(namespace_repos), + repos: Some(repos), }; trees.push(tree); } - Ok(Trees(trees)) + Ok(trees) } } } - pub fn from_trees(trees: Vec) -> Self { - Config::ConfigTree(ConfigTree { - trees: Trees::from_vec(trees), - }) + pub fn from_trees(trees: Vec) -> Self { + Config::ConfigTrees(ConfigTrees { trees }) } pub fn normalize(&mut self) { - if let Config::ConfigTree(config) = self { + if let Config::ConfigTrees(config) = self { let home = super::env_home().display().to_string(); - for tree in &mut config.trees.0 { + for tree in &mut config.trees_mut().iter_mut() { if tree.root.starts_with(&home) { // The tilde is not handled differently, it's just a normal path component for `Path`. // Therefore we can treat it like that during **output**. @@ -181,11 +258,27 @@ impl Config { #[derive(Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -pub struct Tree { +pub struct ConfigTree { pub root: String, pub repos: Option>, } +impl ConfigTree { + pub fn from_repos(root: String, repos: Vec) -> Self { + Self { + root, + repos: Some(repos.into_iter().map(RepoConfig::from_repo).collect()), + } + } + + pub fn from_tree(tree: Tree) -> Self { + Self { + root: tree.root, + repos: Some(tree.repos.into_iter().map(RepoConfig::from_repo).collect()), + } + } +} + pub fn read_config<'a, T>(path: &str) -> Result where T: for<'de> serde::Deserialize<'de>, diff --git a/src/grm/main.rs b/src/grm/main.rs index 7b5f54e..0f7c0c1 100644 --- a/src/grm/main.rs +++ b/src/grm/main.rs @@ -5,6 +5,7 @@ mod cmd; use grm::config; use grm::output::*; +use grm::path_as_string; use grm::provider; use grm::provider::Provider; use grm::repo; @@ -78,16 +79,13 @@ fn main() { match repos { Ok(repos) => { - let mut trees: Vec = vec![]; + let mut trees: Vec = vec![]; for (namespace, repolist) in repos { - let tree = config::Tree { - root: Path::new(&args.root) - .join(namespace) - .display() - .to_string(), - repos: Some(repolist), - }; + let tree = config::ConfigTree::from_repos( + Path::new(&args.root).join(namespace).display().to_string(), + repolist, + ); trees.push(tree); } @@ -191,8 +189,8 @@ fn main() { } }; - let trees = grm::config::Trees::from_vec(vec![found_repos]); - if trees.as_vec_ref().iter().all(|t| match &t.repos { + let trees = grm::config::ConfigTrees::from_trees(vec![found_repos]); + if trees.trees_ref().iter().all(|t| match &t.repos { None => false, Some(r) => r.is_empty(), }) { @@ -311,9 +309,14 @@ fn main() { let mut trees = vec![]; for (namespace, namespace_repos) in repos { - let tree = config::Tree { - root: grm::path_as_string(&Path::new(&config.root).join(namespace)), - repos: Some(namespace_repos), + let tree = config::ConfigTree { + root: path_as_string(&Path::new(&config.root).join(namespace)), + repos: Some( + namespace_repos + .into_iter() + .map(grm::config::RepoConfig::from_repo) + .collect(), + ), }; trees.push(tree); } @@ -395,12 +398,17 @@ fn main() { process::exit(1); }); - let mut trees: Vec = vec![]; + let mut trees: Vec = vec![]; for (namespace, repolist) in repos { - let tree = config::Tree { + let tree = config::ConfigTree { root: Path::new(&args.root).join(namespace).display().to_string(), - repos: Some(repolist), + repos: Some( + repolist + .into_iter() + .map(grm::config::RepoConfig::from_repo) + .collect(), + ), }; trees.push(tree); } @@ -506,7 +514,7 @@ fn main() { } }; - let repo = grm::Repo::open(&cwd, true).unwrap_or_else(|error| { + let repo = grm::RepoHandle::open(&cwd, true).unwrap_or_else(|error| { print_error(&format!("Error opening repository: {}", error)); process::exit(1); }); @@ -539,7 +547,7 @@ fn main() { } } cmd::WorktreeAction::Status(_args) => { - let repo = grm::Repo::open(&cwd, true).unwrap_or_else(|error| { + let repo = grm::RepoHandle::open(&cwd, true).unwrap_or_else(|error| { print_error(&format!("Error opening repository: {}", error)); process::exit(1); }); @@ -564,7 +572,7 @@ fn main() { // * Remove all files // * Set `core.bare` to `true` - let repo = grm::Repo::open(&cwd, false).unwrap_or_else(|error| { + let repo = grm::RepoHandle::open(&cwd, false).unwrap_or_else(|error| { if error.kind == grm::RepoErrorKind::NotFound { print_error("Directory does not contain a git repository"); } else { @@ -592,7 +600,7 @@ fn main() { } } cmd::WorktreeAction::Clean(_args) => { - let repo = grm::Repo::open(&cwd, true).unwrap_or_else(|error| { + let repo = grm::RepoHandle::open(&cwd, true).unwrap_or_else(|error| { if error.kind == grm::RepoErrorKind::NotFound { print_error("Directory does not contain a git repository"); } else { @@ -626,7 +634,7 @@ fn main() { } } cmd::WorktreeAction::Fetch(_args) => { - let repo = grm::Repo::open(&cwd, true).unwrap_or_else(|error| { + let repo = grm::RepoHandle::open(&cwd, true).unwrap_or_else(|error| { if error.kind == grm::RepoErrorKind::NotFound { print_error("Directory does not contain a git repository"); } else { @@ -642,7 +650,7 @@ fn main() { print_success("Fetched from all remotes"); } cmd::WorktreeAction::Pull(args) => { - let repo = grm::Repo::open(&cwd, true).unwrap_or_else(|error| { + let repo = grm::RepoHandle::open(&cwd, true).unwrap_or_else(|error| { if error.kind == grm::RepoErrorKind::NotFound { print_error("Directory does not contain a git repository"); } else { @@ -683,7 +691,7 @@ fn main() { print_error("There is no point in using --rebase without --pull"); process::exit(1); } - let repo = grm::Repo::open(&cwd, true).unwrap_or_else(|error| { + let repo = grm::RepoHandle::open(&cwd, true).unwrap_or_else(|error| { if error.kind == grm::RepoErrorKind::NotFound { print_error("Directory does not contain a git repository"); } else { diff --git a/src/lib.rs b/src/lib.rs index 809778c..44957c2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,12 +11,14 @@ pub mod provider; pub mod repo; pub mod table; -use config::{Config, Tree}; +use config::Config; use output::*; -use repo::{clone_repo, detect_remote_type, Remote, RemoteType, RepoConfig}; +use repo::{clone_repo, detect_remote_type, Remote, RemoteType}; -pub use repo::{RemoteTrackingStatus, Repo, RepoErrorKind, WorktreeRemoveFailureReason}; +pub use repo::{ + RemoteTrackingStatus, Repo, RepoErrorKind, RepoHandle, WorktreeRemoveFailureReason, +}; const GIT_MAIN_WORKTREE_DIRECTORY: &str = ".git-main-working-tree"; const BRANCH_NAMESPACE_SEPARATOR: &str = "/"; @@ -24,6 +26,11 @@ const BRANCH_NAMESPACE_SEPARATOR: &str = "/"; const GIT_CONFIG_BARE_KEY: &str = "core.bare"; const GIT_CONFIG_PUSH_DEFAULT: &str = "push.default"; +pub struct Tree { + root: String, + repos: Vec, +} + #[cfg(test)] mod tests { use super::*; @@ -139,7 +146,7 @@ pub fn get_token_from_command(command: &str) -> Result { Ok(token.to_string()) } -fn sync_repo(root_path: &Path, repo: &RepoConfig, init_worktree: bool) -> Result<(), String> { +fn sync_repo(root_path: &Path, repo: &Repo, init_worktree: bool) -> Result<(), String> { let repo_path = root_path.join(&repo.name); let actual_git_directory = get_actual_git_directory(&repo_path, repo.worktree_setup); @@ -156,7 +163,7 @@ fn sync_repo(root_path: &Path, repo: &RepoConfig, init_worktree: bool) -> Result &repo.name, "Repository does not have remotes configured, initializing new", ); - match Repo::init(&repo_path, repo.worktree_setup) { + match RepoHandle::init(&repo_path, repo.worktree_setup) { Ok(r) => { print_repo_success(&repo.name, "Repository created"); Some(r) @@ -180,10 +187,10 @@ fn sync_repo(root_path: &Path, repo: &RepoConfig, init_worktree: bool) -> Result newly_created = true; } - let repo_handle = match Repo::open(&repo_path, repo.worktree_setup) { + let repo_handle = match RepoHandle::open(&repo_path, repo.worktree_setup) { Ok(repo) => repo, Err(error) => { - if !repo.worktree_setup && Repo::open(&repo_path, true).is_ok() { + if !repo.worktree_setup && RepoHandle::open(&repo_path, true).is_ok() { return Err(String::from( "Repo already exists, but is using a worktree setup", )); @@ -264,7 +271,7 @@ fn sync_repo(root_path: &Path, repo: &RepoConfig, init_worktree: bool) -> Result pub fn find_unmanaged_repos( root_path: &Path, - managed_repos: &[RepoConfig], + managed_repos: &[Repo], ) -> Result, String> { let mut unmanaged_repos = Vec::new(); @@ -279,8 +286,16 @@ pub fn find_unmanaged_repos( pub fn sync_trees(config: Config, init_worktree: bool) -> Result { let mut failures = false; - for tree in config.trees()?.as_vec() { - let repos = tree.repos.unwrap_or_default(); + + let trees = config.trees()?; + + for tree in trees { + let repos: Vec = tree + .repos + .unwrap_or_default() + .into_iter() + .map(|repo| repo.into_repo()) + .collect(); let root_path = expand_path(Path::new(&tree.root)); @@ -372,18 +387,18 @@ fn get_actual_git_directory(path: &Path, is_worktree: bool) -> PathBuf { /// The bool in the return value specifies whether there is a repository /// in root itself. #[allow(clippy::type_complexity)] -fn find_repos(root: &Path) -> Result, Vec, bool)>, String> { - let mut repos: Vec = Vec::new(); +fn find_repos(root: &Path) -> Result, Vec, bool)>, String> { + let mut repos: Vec = Vec::new(); let mut repo_in_root = false; let mut warnings = Vec::new(); for path in find_repo_paths(root)? { - let is_worktree = Repo::detect_worktree(&path); + let is_worktree = RepoHandle::detect_worktree(&path); if path == root { repo_in_root = true; } - match Repo::open(&path, is_worktree) { + match RepoHandle::open(&path, is_worktree) { Err(error) => { warnings.push(format!( "Error opening repo {}{}: {}", @@ -445,7 +460,7 @@ fn find_repos(root: &Path) -> Result, Vec, bool) } let remotes = results; - repos.push(RepoConfig { + repos.push(Repo{ name: match path == root { true => match &root.parent() { Some(parent) => path_as_string(path.strip_prefix(parent).unwrap()), @@ -468,7 +483,7 @@ fn find_repos(root: &Path) -> Result, Vec, bool) pub fn find_in_tree(path: &Path) -> Result<(Tree, Vec), String> { let mut warnings = Vec::new(); - let (repos, repo_in_root): (Vec, bool) = match find_repos(path)? { + let (repos, repo_in_root): (Vec, bool) = match find_repos(path)? { Some((vec, mut repo_warnings, repo_in_root)) => { warnings.append(&mut repo_warnings); (vec, repo_in_root) @@ -491,7 +506,7 @@ pub fn find_in_tree(path: &Path) -> Result<(Tree, Vec), String> { Ok(( Tree { root: root.into_os_string().into_string().unwrap(), - repos: Some(repos), + repos, }, warnings, )) @@ -504,7 +519,7 @@ pub fn add_worktree( track: Option<(&str, &str)>, no_track: bool, ) -> Result<(), String> { - let repo = Repo::open(directory, true).map_err(|error| match error.kind { + let repo = RepoHandle::open(directory, true).map_err(|error| match error.kind { RepoErrorKind::NotFound => { String::from("Current directory does not contain a worktree setup") } @@ -579,7 +594,7 @@ pub fn add_worktree( remote: &mut repo::RemoteHandle, branch_name: &str, remote_branch_name: &str, - repo: &repo::Repo, + repo: &repo::RepoHandle, ) -> Result<(), String> { if !remote.is_pushable()? { return Err(format!( diff --git a/src/provider/mod.rs b/src/provider/mod.rs index 1811f8c..04d8605 100644 --- a/src/provider/mod.rs +++ b/src/provider/mod.rs @@ -9,7 +9,7 @@ pub mod gitlab; pub use github::Github; pub use gitlab::Gitlab; -use crate::{Remote, RemoteType, RepoConfig}; +use crate::{Remote, RemoteType, Repo}; use std::collections::HashMap; @@ -29,16 +29,11 @@ enum ProjectResponse { } pub trait Project { - fn into_repo_config( - self, - provider_name: &str, - worktree_setup: bool, - force_ssh: bool, - ) -> RepoConfig + fn into_repo_config(self, provider_name: &str, worktree_setup: bool, force_ssh: bool) -> Repo where Self: Sized, { - RepoConfig { + Repo { name: self.name(), worktree_setup, remotes: Some(vec![Remote { @@ -205,7 +200,7 @@ pub trait Provider { &self, worktree_setup: bool, force_ssh: bool, - ) -> Result>, String> { + ) -> Result>, String> { let mut repos = vec![]; if self.filter().owner { @@ -282,7 +277,7 @@ pub trait Provider { } } - let mut ret: HashMap> = HashMap::new(); + let mut ret: HashMap> = HashMap::new(); for repo in repos { let namespace = repo.namespace().clone(); diff --git a/src/repo.rs b/src/repo.rs index d369708..5ea257f 100644 --- a/src/repo.rs +++ b/src/repo.rs @@ -104,27 +104,18 @@ impl std::fmt::Display for RepoError { } } -#[derive(Debug, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] +#[derive(Debug)] pub struct Remote { pub name: String, pub url: String, - #[serde(rename = "type")] pub remote_type: RemoteType, } -fn worktree_setup_default() -> bool { - false -} - -#[derive(Debug, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct RepoConfig { +#[derive(Debug)] +pub struct Repo { pub name: String, - #[serde(default = "worktree_setup_default")] pub worktree_setup: bool, - pub remotes: Option>, } @@ -182,7 +173,7 @@ impl Worktree { } pub fn forward_branch(&self, rebase: bool, stash: bool) -> Result, String> { - let repo = Repo::open(Path::new(&self.name), false) + let repo = RepoHandle::open(Path::new(&self.name), false) .map_err(|error| format!("Error opening worktree: {}", error))?; if let Ok(remote_branch) = repo.find_local_branch(&self.name)?.upstream() { @@ -286,7 +277,7 @@ impl Worktree { config: &Option, stash: bool, ) -> Result, String> { - let repo = Repo::open(Path::new(&self.name), false) + let repo = RepoHandle::open(Path::new(&self.name), false) .map_err(|error| format!("Error opening worktree: {}", error))?; let guess_default_branch = || { @@ -468,14 +459,14 @@ pub fn detect_remote_type(remote_url: &str) -> Option { None } -pub struct Repo(git2::Repository); +pub struct RepoHandle(git2::Repository); pub struct Branch<'a>(git2::Branch<'a>); fn convert_libgit2_error(error: git2::Error) -> String { error.message().to_string() } -impl Repo { +impl RepoHandle { pub fn open(path: &Path, is_worktree: bool) -> Result { let open_func = match is_worktree { true => Repository::open_bare, @@ -507,7 +498,7 @@ impl Repo { // Right now, we just open the repo AGAIN. It is safe, as we are only accessing the stash // with the second reference, so there are no cross effects. But it just smells. Also, // using `unwrap()` here as we are already sure that the repo is openable(?). - let mut repo = Repo::open(self.0.path(), false).unwrap(); + let mut repo = RepoHandle::open(self.0.path(), false).unwrap(); repo.0 .stash_save2(&author, None, Some(git2::StashFlags::INCLUDE_UNTRACKED)) .map_err(convert_libgit2_error)?; @@ -515,7 +506,7 @@ impl Repo { } pub fn stash_pop(&self) -> Result<(), String> { - let mut repo = Repo::open(self.0.path(), false).unwrap(); + let mut repo = RepoHandle::open(self.0.path(), false).unwrap(); repo.0 .stash_pop( 0, @@ -659,7 +650,7 @@ impl Repo { .map_err(convert_libgit2_error)?, }; - let repo = Repo(repo); + let repo = RepoHandle(repo); if is_worktree { repo.set_config_push(GitPushDefaultSetting::Upstream)?; @@ -788,7 +779,7 @@ impl Repo { } } - let worktree_repo = Repo::open(root_dir, true).map_err(|error| { + let worktree_repo = RepoHandle::open(root_dir, true).map_err(|error| { WorktreeConversionFailureReason::Error(format!( "Opening newly converted repository failed: {}", error @@ -1068,7 +1059,7 @@ impl Repo { name ))); } - let worktree_repo = Repo::open(worktree_dir, false).map_err(|error| { + let worktree_repo = RepoHandle::open(worktree_dir, false).map_err(|error| { WorktreeRemoveFailureReason::Error(format!("Error opening repo: {}", error)) })?; @@ -1427,7 +1418,7 @@ impl RemoteHandle<'_> { &mut self, local_branch_name: &str, remote_branch_name: &str, - _repo: &Repo, + _repo: &RepoHandle, ) -> Result<(), String> { if !self.is_pushable()? { return Err(String::from("Trying to push to a non-pushable remote")); @@ -1493,7 +1484,7 @@ pub fn clone_repo( } } - let repo = Repo::open(&clone_target, false)?; + let repo = RepoHandle::open(&clone_target, false)?; if is_worktree { repo.set_config_push(GitPushDefaultSetting::Upstream)?; diff --git a/src/table.rs b/src/table.rs index fa7cac1..72b0f78 100644 --- a/src/table.rs +++ b/src/table.rs @@ -1,4 +1,4 @@ -use crate::Repo; +use crate::RepoHandle; use comfy_table::{Cell, Table}; @@ -21,7 +21,7 @@ fn add_table_header(table: &mut Table) { fn add_repo_status( table: &mut Table, repo_name: &str, - repo_handle: &crate::Repo, + repo_handle: &crate::RepoHandle, is_worktree: bool, ) -> Result<(), String> { let repo_status = repo_handle.status(is_worktree)?; @@ -99,7 +99,7 @@ fn add_repo_status( // Don't return table, return a type that implements Display(?) pub fn get_worktree_status_table( - repo: &crate::Repo, + repo: &crate::RepoHandle, directory: &Path, ) -> Result<(impl std::fmt::Display, Vec), String> { let worktrees = repo.get_worktrees()?; @@ -111,7 +111,7 @@ pub fn get_worktree_status_table( for worktree in &worktrees { let worktree_dir = &directory.join(&worktree.name()); if worktree_dir.exists() { - let repo = match crate::Repo::open(worktree_dir, false) { + let repo = match crate::RepoHandle::open(worktree_dir, false) { Ok(repo) => repo, Err(error) => { errors.push(format!( @@ -132,7 +132,7 @@ pub fn get_worktree_status_table( )); } } - for worktree in Repo::find_unmanaged_worktrees(repo, directory)? { + for worktree in RepoHandle::find_unmanaged_worktrees(repo, directory)? { errors.push(format!( "Found {}, which is not a valid worktree directory!", &worktree @@ -144,7 +144,7 @@ pub fn get_worktree_status_table( pub fn get_status_table(config: crate::Config) -> Result<(Vec
, Vec), String> { let mut errors = Vec::new(); let mut tables = Vec::new(); - for tree in config.trees()?.as_vec() { + for tree in config.trees()? { let repos = tree.repos.unwrap_or_default(); let root_path = crate::expand_path(Path::new(&tree.root)); @@ -163,7 +163,7 @@ pub fn get_status_table(config: crate::Config) -> Result<(Vec
, Vec repo, @@ -207,7 +207,7 @@ fn add_worktree_table_header(table: &mut Table) { fn add_worktree_status( table: &mut Table, worktree: &crate::repo::Worktree, - repo: &crate::Repo, + repo: &crate::RepoHandle, ) -> Result<(), String> { let repo_status = repo.status(false)?; @@ -272,10 +272,10 @@ pub fn show_single_repo_status( let mut table = Table::new(); let mut warnings = Vec::new(); - let is_worktree = crate::Repo::detect_worktree(path); + let is_worktree = crate::RepoHandle::detect_worktree(path); add_table_header(&mut table); - let repo_handle = crate::Repo::open(path, is_worktree); + let repo_handle = crate::RepoHandle::open(path, is_worktree); if let Err(error) = repo_handle { if error.kind == crate::RepoErrorKind::NotFound { diff --git a/tests/repo.rs b/tests/repo.rs index efcb10e..5027b4f 100644 --- a/tests/repo.rs +++ b/tests/repo.rs @@ -8,13 +8,13 @@ use helpers::*; fn open_empty_repo() { let tmpdir = init_tmpdir(); assert!(matches!( - Repo::open(tmpdir.path(), true), + RepoHandle::open(tmpdir.path(), true), Err(RepoError { kind: RepoErrorKind::NotFound }) )); assert!(matches!( - Repo::open(tmpdir.path(), false), + RepoHandle::open(tmpdir.path(), false), Err(RepoError { kind: RepoErrorKind::NotFound }) @@ -25,7 +25,7 @@ fn open_empty_repo() { #[test] fn create_repo() -> Result<(), Box> { let tmpdir = init_tmpdir(); - let repo = Repo::init(tmpdir.path(), false)?; + let repo = RepoHandle::init(tmpdir.path(), false)?; assert!(!repo.is_bare()); assert!(repo.is_empty()?); cleanup_tmpdir(tmpdir); @@ -35,7 +35,7 @@ fn create_repo() -> Result<(), Box> { #[test] fn create_repo_with_worktree() -> Result<(), Box> { let tmpdir = init_tmpdir(); - let repo = Repo::init(tmpdir.path(), true)?; + let repo = RepoHandle::init(tmpdir.path(), true)?; assert!(repo.is_bare()); assert!(repo.is_empty()?); cleanup_tmpdir(tmpdir); From f2d2482476df486b75ed813bc4fc2afc493ecbd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Thu, 26 May 2022 01:33:15 +0200 Subject: [PATCH 26/36] e2e: Add tests for subdirectory checkouts --- e2e_tests/test_repos_sync.py | 59 ++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/e2e_tests/test_repos_sync.py b/e2e_tests/test_repos_sync.py index 998d927..8302532 100644 --- a/e2e_tests/test_repos_sync.py +++ b/e2e_tests/test_repos_sync.py @@ -133,6 +133,32 @@ templates = { """ ), }, + "repo_in_subdirectory": { + "toml": """ + [[trees]] + root = "{root}" + + [[trees.repos]] + name = "outer/inner" + + [[trees.repos.remotes]] + name = "origin" + url = "file://{remote}" + type = "file" + """, + "yaml": textwrap.dedent( + """ + trees: + - root: "{root}" + repos: + - name: outer/inner + remotes: + - name: origin + url: "file://{remote}" + type: "file" + """ + ), + }, "nested_trees": { "toml": """ [[trees]] @@ -319,6 +345,39 @@ def test_repos_sync_normal_clone(configtype): assert urls[0] == f"file://{remote2}" +@pytest.mark.parametrize("configtype", ["toml", "yaml"]) +def test_repos_sync_repo_in_subdirectory(configtype): + with tempfile.TemporaryDirectory() as target: + with TempGitFileRemote() as (remote, remote_head_commit_sha): + with tempfile.NamedTemporaryFile() as config: + with open(config.name, "w") as f: + f.write( + templates["repo_in_subdirectory"][configtype].format( + root=target, remote=remote + ) + ) + + cmd = grm(["repos", "sync", "config", "--config", config.name]) + assert cmd.returncode == 0 + + git_dir = os.path.join(target, "outer", "inner") + assert os.path.exists(git_dir) + with git.Repo(git_dir) as repo: + assert not repo.bare + assert not repo.is_dirty() + assert set([str(r) for r in repo.remotes]) == {"origin"} + assert str(repo.active_branch) == "master" + assert str(repo.head.commit) == remote_head_commit_sha + + assert len(repo.remotes) == 1 + urls = list(repo.remote("origin").urls) + assert len(urls) == 1 + assert urls[0] == f"file://{remote}" + + cmd = grm(["repos", "sync", "config", "--config", config.name]) + assert not "found unmanaged repository" in cmd.stderr.lower() + + @pytest.mark.parametrize("configtype", ["toml", "yaml"]) def test_repos_sync_nested_clone(configtype): with tempfile.TemporaryDirectory() as target: From b8c552fb6202315f6d3f1fbeb5edf2fd1fc7b79f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Thu, 26 May 2022 16:30:32 +0200 Subject: [PATCH 27/36] Give repos a namespace to allow subdirectories --- src/config.rs | 17 ++++++++++++++--- src/grm/main.rs | 23 +++++++++++++++++------ src/lib.rs | 34 +++++++++++++++++++++++++--------- src/provider/github.rs | 12 ++++++------ src/provider/gitlab.rs | 12 ++++++------ src/provider/mod.rs | 15 ++++++++++----- src/repo.rs | 15 ++++++++++++++- 7 files changed, 92 insertions(+), 36 deletions(-) diff --git a/src/config.rs b/src/config.rs index 2a5648d..bbead78 100644 --- a/src/config.rs +++ b/src/config.rs @@ -5,7 +5,7 @@ use crate::output::*; use std::path::Path; -use crate::{get_token_from_command, Remote, Repo, Tree}; +use crate::{get_token_from_command, path_as_string, Remote, Repo, Tree}; use crate::provider; use crate::provider::Filter; @@ -104,8 +104,15 @@ impl RepoConfig { } pub fn into_repo(self) -> Repo { + let (namespace, name) = if let Some((namespace, name)) = self.name.rsplit_once('/') { + (Some(namespace.to_string()), name.to_string()) + } else { + (None, self.name) + }; + Repo { - name: self.name, + name, + namespace, worktree_setup: self.worktree_setup, remotes: self.remotes.map(|remotes| { remotes @@ -209,7 +216,11 @@ impl Config { .map(RepoConfig::from_repo) .collect(); let tree = ConfigTree { - root: crate::path_as_string(&Path::new(&config.root).join(namespace)), + root: if let Some(namespace) = namespace { + path_as_string(&Path::new(&config.root).join(namespace)) + } else { + path_as_string(Path::new(&config.root)) + }, repos: Some(repos), }; trees.push(tree); diff --git a/src/grm/main.rs b/src/grm/main.rs index 0f7c0c1..630ee67 100644 --- a/src/grm/main.rs +++ b/src/grm/main.rs @@ -82,10 +82,13 @@ fn main() { let mut trees: Vec = vec![]; for (namespace, repolist) in repos { - let tree = config::ConfigTree::from_repos( - Path::new(&args.root).join(namespace).display().to_string(), - repolist, - ); + let root = if let Some(namespace) = namespace { + path_as_string(&Path::new(&args.root).join(namespace)) + } else { + path_as_string(Path::new(&args.root)) + }; + + let tree = config::ConfigTree::from_repos(root, repolist); trees.push(tree); } @@ -310,7 +313,11 @@ fn main() { for (namespace, namespace_repos) in repos { let tree = config::ConfigTree { - root: path_as_string(&Path::new(&config.root).join(namespace)), + root: if let Some(namespace) = namespace { + path_as_string(&Path::new(&config.root).join(namespace)) + } else { + path_as_string(Path::new(&config.root)) + }, repos: Some( namespace_repos .into_iter() @@ -402,7 +409,11 @@ fn main() { for (namespace, repolist) in repos { let tree = config::ConfigTree { - root: Path::new(&args.root).join(namespace).display().to_string(), + root: if let Some(namespace) = namespace { + path_as_string(&Path::new(&args.root).join(namespace)) + } else { + path_as_string(Path::new(&args.root)) + }, repos: Some( repolist .into_iter() diff --git a/src/lib.rs b/src/lib.rs index 44957c2..c6322f8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -147,7 +147,7 @@ pub fn get_token_from_command(command: &str) -> Result { } fn sync_repo(root_path: &Path, repo: &Repo, init_worktree: bool) -> Result<(), String> { - let repo_path = root_path.join(&repo.name); + let repo_path = root_path.join(&repo.fullname()); let actual_git_directory = get_actual_git_directory(&repo_path, repo.worktree_setup); let mut newly_created = false; @@ -460,17 +460,33 @@ fn find_repos(root: &Path) -> Result, Vec, bool)>, Str } let remotes = results; - repos.push(Repo{ - name: match path == root { - true => match &root.parent() { + let (namespace, name) = if path == root { + ( + None, + match &root.parent() { Some(parent) => path_as_string(path.strip_prefix(parent).unwrap()), None => { warnings.push(String::from("Getting name of the search root failed. Do you have a git repository in \"/\"?")); - continue - }, - } - false => path_as_string(path.strip_prefix(&root).unwrap()), - }, + continue; + } + }, + ) + } else { + let name = path.strip_prefix(&root).unwrap(); + let namespace = name.parent().unwrap(); + ( + if namespace != Path::new("") { + Some(path_as_string(namespace).to_string()) + } else { + None + }, + path_as_string(name), + ) + }; + + repos.push(Repo { + name, + namespace, remotes: Some(remotes), worktree_setup: is_worktree, }); diff --git a/src/provider/github.rs b/src/provider/github.rs index 536f3c5..3a843c3 100644 --- a/src/provider/github.rs +++ b/src/provider/github.rs @@ -32,12 +32,12 @@ impl Project for GithubProject { self.name.clone() } - fn namespace(&self) -> String { - self.full_name - .rsplit_once('/') - .expect("Github project name did not include a namespace") - .0 - .to_string() + fn namespace(&self) -> Option { + if let Some((namespace, _name)) = self.full_name.rsplit_once('/') { + Some(namespace.to_string()) + } else { + None + } } fn ssh_url(&self) -> String { diff --git a/src/provider/gitlab.rs b/src/provider/gitlab.rs index 170141b..ccadd7f 100644 --- a/src/provider/gitlab.rs +++ b/src/provider/gitlab.rs @@ -39,12 +39,12 @@ impl Project for GitlabProject { self.name.clone() } - fn namespace(&self) -> String { - self.path_with_namespace - .rsplit_once('/') - .expect("Gitlab project name did not include a namespace") - .0 - .to_string() + fn namespace(&self) -> Option { + if let Some((namespace, _name)) = self.path_with_namespace.rsplit_once('/') { + Some(namespace.to_string()) + } else { + None + } } fn ssh_url(&self) -> String { diff --git a/src/provider/mod.rs b/src/provider/mod.rs index 04d8605..20253b6 100644 --- a/src/provider/mod.rs +++ b/src/provider/mod.rs @@ -35,6 +35,7 @@ pub trait Project { { Repo { name: self.name(), + namespace: self.namespace(), worktree_setup, remotes: Some(vec![Remote { name: String::from(provider_name), @@ -53,7 +54,7 @@ pub trait Project { } fn name(&self) -> String; - fn namespace(&self) -> String; + fn namespace(&self) -> Option; fn ssh_url(&self) -> String; fn http_url(&self) -> String; fn private(&self) -> bool; @@ -200,7 +201,7 @@ pub trait Provider { &self, worktree_setup: bool, force_ssh: bool, - ) -> Result>, String> { + ) -> Result, Vec>, String> { let mut repos = vec![]; if self.filter().owner { @@ -277,12 +278,16 @@ pub trait Provider { } } - let mut ret: HashMap> = HashMap::new(); + let mut ret: HashMap, Vec> = HashMap::new(); for repo in repos { - let namespace = repo.namespace().clone(); + let namespace = repo.namespace(); - let repo = repo.into_repo_config(&self.name(), worktree_setup, force_ssh); + let mut repo = repo.into_repo_config(&self.name(), worktree_setup, force_ssh); + + // Namespace is already part of the hashmap key. I'm not too happy + // about the data exchange format here. + repo.remove_namespace(); ret.entry(namespace).or_insert(vec![]).push(repo); } diff --git a/src/repo.rs b/src/repo.rs index 5ea257f..02d040f 100644 --- a/src/repo.rs +++ b/src/repo.rs @@ -114,11 +114,24 @@ pub struct Remote { #[derive(Debug)] pub struct Repo { pub name: String, - + pub namespace: Option, pub worktree_setup: bool, pub remotes: Option>, } +impl Repo { + pub fn fullname(&self) -> String { + match &self.namespace { + Some(namespace) => format!("{}/{}", namespace, self.name), + None => self.name.clone(), + } + } + + pub fn remove_namespace(&mut self) { + self.namespace = None + } +} + pub struct RepoChanges { pub files_new: usize, pub files_modified: usize, From b17f4d68ef62fd5fa43df048c2368f901af767eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Thu, 26 May 2022 16:30:37 +0200 Subject: [PATCH 28/36] Fix handling of unmanaged repositories Before, there were warnings in case of nested trees. --- src/lib.rs | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index c6322f8..cd05ce5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -272,13 +272,15 @@ fn sync_repo(root_path: &Path, repo: &Repo, init_worktree: bool) -> Result<(), S pub fn find_unmanaged_repos( root_path: &Path, managed_repos: &[Repo], -) -> Result, String> { +) -> Result, String> { let mut unmanaged_repos = Vec::new(); - for repo in find_repo_paths(root_path)? { - let name = path_as_string(repo.strip_prefix(&root_path).unwrap()); - if !managed_repos.iter().any(|r| r.name == name) { - unmanaged_repos.push(name); + for repo_path in find_repo_paths(root_path)? { + if !managed_repos + .iter() + .any(|r| Path::new(root_path).join(r.fullname()) == repo_path) + { + unmanaged_repos.push(repo_path); } } Ok(unmanaged_repos) @@ -287,6 +289,9 @@ pub fn find_unmanaged_repos( pub fn sync_trees(config: Config, init_worktree: bool) -> Result { let mut failures = false; + let mut unmanaged_repos_absolute_paths = vec![]; + let mut managed_repos_absolute_paths = vec![]; + let trees = config.trees()?; for tree in trees { @@ -300,6 +305,7 @@ pub fn sync_trees(config: Config, init_worktree: bool) -> Result { let root_path = expand_path(Path::new(&tree.root)); for repo in &repos { + managed_repos_absolute_paths.push(root_path.join(repo.fullname())); match sync_repo(&root_path, repo, init_worktree) { Ok(_) => print_repo_success(&repo.name, "OK"), Err(error) => { @@ -310,10 +316,8 @@ pub fn sync_trees(config: Config, init_worktree: bool) -> Result { } match find_unmanaged_repos(&root_path, &repos) { - Ok(unmanaged_repos) => { - for name in unmanaged_repos { - print_warning(&format!("Found unmanaged repository: {}", name)); - } + Ok(repos) => { + unmanaged_repos_absolute_paths.extend(repos); } Err(error) => { print_error(&format!("Error getting unmanaged repos: {}", error)); @@ -322,6 +326,21 @@ pub fn sync_trees(config: Config, init_worktree: bool) -> Result { } } + for unmanaged_repo_absolute_path in &unmanaged_repos_absolute_paths { + if managed_repos_absolute_paths + .iter() + .any(|managed_repo_absolute_path| { + managed_repo_absolute_path == unmanaged_repo_absolute_path + }) + { + continue; + } + print_warning(&format!( + "Found unmanaged repository: \"{}\"", + path_as_string(unmanaged_repo_absolute_path) + )); + } + Ok(!failures) } From f41b9b1684bb34377f4d677d1a13678af855a219 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Thu, 26 May 2022 03:19:57 +0200 Subject: [PATCH 29/36] Add pycache to gitignore --- e2e_tests/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 e2e_tests/.gitignore diff --git a/e2e_tests/.gitignore b/e2e_tests/.gitignore new file mode 100644 index 0000000..a348e50 --- /dev/null +++ b/e2e_tests/.gitignore @@ -0,0 +1 @@ +/__pycache__/ From 1212917fae923bb59801b17a92a0e443e536d579 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Thu, 26 May 2022 14:02:32 +0200 Subject: [PATCH 30/36] Add unit tests for `Repo::fullname()` --- src/repo.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/repo.rs b/src/repo.rs index 02d040f..d020330 100644 --- a/src/repo.rs +++ b/src/repo.rs @@ -447,6 +447,26 @@ mod tests { fn check_unsupported_protocol_git() { detect_remote_type("git://example.com"); } + + #[test] + fn repo_check_fullname() { + let with_namespace = Repo { + name: "name".to_string(), + namespace: Some("namespace".to_string()), + worktree_setup: false, + remotes: None, + }; + + let without_namespace = Repo { + name: "name".to_string(), + namespace: None, + worktree_setup: false, + remotes: None, + }; + + assert_eq!(with_namespace.fullname(), "namespace/name"); + assert_eq!(without_namespace.fullname(), "name"); + } } pub fn detect_remote_type(remote_url: &str) -> Option { From 62c1e430b26a009c6c035cba4caccfeb19878927 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Thu, 26 May 2022 14:03:30 +0200 Subject: [PATCH 31/36] Derive Eq when deriving PartialEq There is a clippy lint for this. --- src/repo.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/repo.rs b/src/repo.rs index d020330..e8782f6 100644 --- a/src/repo.rs +++ b/src/repo.rs @@ -7,7 +7,7 @@ use crate::output::*; const WORKTREE_CONFIG_FILE_NAME: &str = "grm.toml"; -#[derive(Debug, Serialize, Deserialize, PartialEq)] +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum RemoteType { Ssh, @@ -31,7 +31,7 @@ pub enum GitPushDefaultSetting { Upstream, } -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Eq)] pub enum RepoErrorKind { NotFound, Unknown(String), From c439595d92cc755c370541b5bd61eb94970f2e66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Thu, 26 May 2022 18:53:21 +0200 Subject: [PATCH 32/36] Justfile: Add target to lint --- Justfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Justfile b/Justfile index c67d7c3..4c6e46a 100644 --- a/Justfile +++ b/Justfile @@ -9,6 +9,9 @@ fmt: cargo fmt git ls-files | grep '\.py$' | xargs black +lint: + cargo clippy --no-deps + lint-fix: cargo clippy --no-deps --fix From 4841920c64aac10d62e44ce36108992a02a7e01f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Thu, 26 May 2022 19:07:16 +0200 Subject: [PATCH 33/36] dependencies: Update serde_json to 1.0.81 --- Cargo.lock | 14 ++++---------- Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c356d4e..b60c2b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -402,7 +402,7 @@ checksum = "ff8670570af52249509a86f5e3e18a08c60b177071826898fde8997cf5f6bfbb" dependencies = [ "bytes", "fnv", - "itoa 1.0.2", + "itoa", ] [[package]] @@ -464,12 +464,6 @@ dependencies = [ "waker-fn", ] -[[package]] -name = "itoa" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" - [[package]] name = "itoa" version = "1.0.2" @@ -899,11 +893,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.59" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcac07dbffa1c65e7f816ab9eba78eb142c6d44410f4eeba1e26e4f5dfa56b95" +checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" dependencies = [ - "itoa 0.4.8", + "itoa", "ryu", "serde", ] diff --git a/Cargo.toml b/Cargo.toml index 25fed11..aa0411c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,7 +69,7 @@ version = "=5.0.1" version = "=0.8.24" [dependencies.serde_json] -version = "=1.0.59" +version = "=1.0.81" [dependencies.isahc] version = "=1.7.1" From 95cffc5f0eb4b5ae33e192a8fc1ca0adf8923d0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Thu, 26 May 2022 19:07:17 +0200 Subject: [PATCH 34/36] dependencies: Update isahc to 1.7.2 --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b60c2b8..5b98d66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -437,9 +437,9 @@ dependencies = [ [[package]] name = "isahc" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "480d9158c9977bff0bc024a11dcad04efcd3955c1e55301092b13fc439d41720" +checksum = "334e04b4d781f436dc315cb1e7515bd96826426345d498149e4bde36b67f8ee9" dependencies = [ "async-channel", "castaway", diff --git a/Cargo.toml b/Cargo.toml index aa0411c..cd8059e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,7 +72,7 @@ version = "=0.8.24" version = "=1.0.81" [dependencies.isahc] -version = "=1.7.1" +version = "=1.7.2" features = ["json"] [dependencies.parse_link_header] From 5b78c3ba9e9063a45d70c88f3e7be7c71e4e4fdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Thu, 26 May 2022 19:09:27 +0200 Subject: [PATCH 35/36] Release v0.7.0 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5b98d66..086f464 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -332,7 +332,7 @@ dependencies = [ [[package]] name = "git-repo-manager" -version = "0.6.2" +version = "0.7.0" dependencies = [ "clap", "comfy-table", diff --git a/Cargo.toml b/Cargo.toml index cd8059e..7d068e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "git-repo-manager" -version = "0.6.2" +version = "0.7.0" edition = "2021" authors = [ "Hannes Körber ", From 2d34ba1bd70245d72f6babb67e387bbcc939a1fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20K=C3=B6rber?= Date: Thu, 26 May 2022 19:11:19 +0200 Subject: [PATCH 36/36] Fix forge documentation --- docs/src/forge_integration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/forge_integration.md b/docs/src/forge_integration.md index f9382a7..22b105b 100644 --- a/docs/src/forge_integration.md +++ b/docs/src/forge_integration.md @@ -10,7 +10,7 @@ Imagine you are just starting out with `grm` and want to clone all your reposito from GitHub. This is as simple as: ```bash -$ grm repos sync remote --provider github --owner --token-command "pass show github_grm_access_token --path ~/projects" +$ grm repos sync remote --provider github --owner --token-command "pass show github_grm_access_token" --path ~/projects ``` You will end up with your projects cloned into `~/projects/{your_github_username}/`