hannes/git-repo-manager
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #51 from BapRx/feat/exclude-paths-based-on-regex

Browse Source 
chore(repo/find): Exlude paths based on regex
This commit is contained in:
Hannes Körber
2023-02-07 22:50:02 +01:00
committed by GitHub
parent 9b4ed2837e 1262ec5a33
commit 60a777276f
26 changed files with 203 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Justfile
View File

@@ -13,6 +13,7 @@ clean:
fmt:
cargo fmt
git ls-files | grep '\.py$' | xargs isort
git ls-files | grep '\.py$' | xargs black
git ls-files | grep '\.sh$' | xargs -L 1 shfmt --indent 4 --write
@@ -23,6 +24,7 @@ fmt-check:
lint:
cargo clippy --no-deps -- -Dwarnings
git ls-files | grep '\.py$' | xargs ruff --ignore E501
git ls-files | grep '\.sh$' | xargs -L 1 shellcheck --norc
lint-fix:

5
depcheck/update-cargo-dependencies.py
View File

@@ -1,9 +1,8 @@
#!/usr/bin/env python3
import subprocess
import os
import json
import sys
import os
import subprocess
import semver
import tomlkit

4
docs/src/developing.md
View File

@@ -34,8 +34,8 @@ You will need the following tools:
[here](https://github.com/casey/just#installation) for installation
instructions (it's most likely just a simple `cargo install just`).
* Docker & docker-compose for the e2e tests
* `black` and `shfmt` for formatting.
* `shellcheck` for shell script linting
* `isort`, `black` and `shfmt` for formatting.
* `ruff` and `shellcheck` for linting.
* `mdbook` for the documentation
Here are the tools:

15
docs/src/local_configuration.md
View File

@@ -11,7 +11,7 @@ Let's try it out:
## Get the example configuration
```bash
$ curl --proto '=https' --tlsv1.2 -sSfO https://raw.githubusercontent.com/hakoerber/git-repo-manager/master/example.config.toml
curl --proto '=https' --tlsv1.2 -sSfO https://raw.githubusercontent.com/hakoerber/git-repo-manager/master/example.config.toml
```
Then, you're ready to run the first sync. This will clone all configured
@@ -30,7 +30,7 @@ $ grm repos sync config --config example.config.toml
If you run it again, it will report no changes:
```
```bash
$ grm repos sync config -c example.config.toml
[✔] git-repo-manager: OK
[✔] dotfiles: OK
@@ -43,11 +43,18 @@ write a configuration from scratch. Luckily, GRM has a way to generate a
configuration from an existing file tree:
```bash
$ grm repos find local ~/your/project/root > config.toml
grm repos find local ~/your/project/root > config.toml
```
This will detect all repositories and remotes and write them to `config.toml`.
You can exclude repositories from the generated configuration by providing
a regex that will be test against the path of each discovered repository:
```bash
grm repos find local ~/your/project/root --exclude "^.*/subdir/match-(foo|bar)/.*$" > config.toml
```
### Show the state of your projects
```bash
@@ -65,7 +72,7 @@ $ grm repos status --config example.config.toml
You can also use `status` without `--config` to check the repository you're
currently in:
```
```bash
$ cd ~/example-projects/dotfiles
$ grm repos status
╭──────────┬──────────┬────────┬──────────┬───────┬─────────╮

2
docs/src/testing.md
View File

@@ -49,7 +49,7 @@ Note: You will most likely not need to read this.
Each test parameter will exponentially increase the number of tests that will be
run. As a general rule, comprehensiveness is more important than test suite
runtime (so if in doubt, better to add another parameter to catch every edge
case). But try to keep the total runtime sane. Currently, the whole `just e2e`
case). But try to keep the total runtime sane. Currently, the whole `just test-e2e`
target runs ~8'000 tests and takes around 5 minutes on my machine, exlucding
binary and docker build time. I'd say that keeping it under 10 minutes is a good
idea.

2
e2e_tests/conftest.py
View File

@@ -1,7 +1,5 @@
import os
from helpers import *
def pytest_configure(config):
os.environ["GIT_AUTHOR_NAME"] = "Example user"

4
e2e_tests/docker-rest/flask/app.py
View File

@@ -3,5 +3,5 @@ from flask import Flask
app = Flask(__name__)
app.url_map.strict_slashes = False
import github
import gitlab
import github # noqa: E402,F401
import gitlab # noqa: E402,F401

8
e2e_tests/docker-rest/flask/github.py
View File

@@ -1,10 +1,8 @@
import os.path
from app import app
from flask import Flask, request, abort, jsonify, make_response
import jinja2
from app import app
from flask import abort, jsonify, make_response, request
def check_headers():
@@ -48,7 +46,7 @@ def add_pagination(response, page, last_page):
def read_project_files(namespaces=[]):
last_page = 4
page = username = int(request.args.get("page", "1"))
page = int(request.args.get("page", "1"))
response_file = f"./github_api_page_{page}.json.j2"
if not os.path.exists(response_file):
return jsonify([])

8
e2e_tests/docker-rest/flask/gitlab.py
View File

@@ -1,10 +1,8 @@
import os.path
from app import app
from flask import Flask, request, abort, jsonify, make_response
import jinja2
from app import app
from flask import abort, jsonify, make_response, request
def check_headers():
@@ -48,7 +46,7 @@ def add_pagination(response, page, last_page):
def read_project_files(namespaces=[]):
last_page = 4
page = username = int(request.args.get("page", "1"))
page = int(request.args.get("page", "1"))
response_file = f"./gitlab_api_page_{page}.json"
if not os.path.exists(response_file):
return jsonify([])

7
e2e_tests/helpers.py
View File

@@ -1,12 +1,12 @@
#!/usr/bin/env python3
import hashlib
import inspect
import os
import os.path
import shutil
import subprocess
import tempfile
import hashlib
import shutil
import inspect
import git
@@ -176,7 +176,6 @@ class TempGitRemote:
newobj.remoteid = remoteid
return newobj, remoteid
else:
refresh = False
if cachekey not in cls.obj:
tmpdir = get_temporary_directory()
shell(

2
e2e_tests/test_basic.py
View File

@@ -1,6 +1,6 @@
#!/usr/bin/env python3
from helpers import *
from helpers import grm
def test_invalid_command():

43
e2e_tests/test_repos_find.py
View File

@@ -1,12 +1,13 @@
#!/usr/bin/env python3
import os
import re
import tempfile
import toml
import pytest
import toml
import yaml
from helpers import *
from helpers import NonExistentPath, TempGitRepository, grm, shell
def test_repos_find_nonexistent():
@@ -63,9 +64,10 @@ def test_repos_find_non_git_repos():
assert len(cmd.stderr) != 0
@pytest.mark.parametrize("default", [True, False])
@pytest.mark.parametrize("default_format", [True, False])
@pytest.mark.parametrize("configtype", ["toml", "yaml"])
def test_repos_find(configtype, default):
@pytest.mark.parametrize("exclude", [None, "^.*/repo2$", "^not_matching$"])
def test_repos_find(configtype, exclude, default_format):
with tempfile.TemporaryDirectory() as tmpdir:
shell(
f"""
@@ -99,13 +101,19 @@ def test_repos_find(configtype, default):
)
args = ["repos", "find", "local", tmpdir]
if not default:
if not default_format:
args += ["--format", configtype]
if exclude:
args += ["--exclude", exclude]
cmd = grm(args)
assert cmd.returncode == 0
if exclude == "^.*/repo2$":
assert re.match(r"^.*\[skipped\] .*\/repo2$", cmd.stderr.lower())
assert "repo2" in cmd.stderr.lower()
else:
assert len(cmd.stderr) == 0
if default or configtype == "toml":
if default_format or configtype == "toml":
output = toml.loads(cmd.stdout)
elif configtype == "yaml":
output = yaml.safe_load(cmd.stdout)
@@ -120,7 +128,7 @@ def test_repos_find(configtype, default):
assert set(tree.keys()) == {"root", "repos"}
assert tree["root"] == tmpdir
assert isinstance(tree["repos"], list)
assert len(tree["repos"]) == 2
assert len(tree["repos"]) == (1 if exclude == "^.*/repo2$" else 2)
repo1 = [r for r in tree["repos"] if r["name"] == "repo1"][0]
assert repo1["worktree_setup"] is False
@@ -137,6 +145,9 @@ def test_repos_find(configtype, default):
assert someremote["type"] == "ssh"
assert someremote["url"] == "ssh://example.com/repo2.git"
if exclude == "^.*/repo2$":
assert [r for r in tree["repos"] if r["name"] == "repo2"] == []
else:
repo2 = [r for r in tree["repos"] if r["name"] == "repo2"][0]
assert repo2["worktree_setup"] is False
assert isinstance(repo1["remotes"], list)
@@ -148,19 +159,19 @@ def test_repos_find(configtype, default):
assert origin["url"] == "https://example.com/repo2.git"
@pytest.mark.parametrize("default", [True, False])
@pytest.mark.parametrize("default_format", [True, False])
@pytest.mark.parametrize("configtype", ["toml", "yaml"])
def test_repos_find_in_root(configtype, default):
def test_repos_find_in_root(configtype, default_format):
with TempGitRepository() as repo_dir:
args = ["repos", "find", "local", repo_dir]
if not default:
if not default_format:
args += ["--format", configtype]
cmd = grm(args)
assert cmd.returncode == 0
assert len(cmd.stderr) == 0
if default or configtype == "toml":
if default_format or configtype == "toml":
output = toml.loads(cmd.stdout)
elif configtype == "yaml":
output = yaml.safe_load(cmd.stdout)
@@ -194,8 +205,8 @@ def test_repos_find_in_root(configtype, default):
@pytest.mark.parametrize("configtype", ["toml", "yaml"])
@pytest.mark.parametrize("default", [True, False])
def test_repos_find_with_invalid_repo(configtype, default):
@pytest.mark.parametrize("default_format", [True, False])
def test_repos_find_with_invalid_repo(configtype, default_format):
with tempfile.TemporaryDirectory() as tmpdir:
shell(
f"""
@@ -229,13 +240,13 @@ def test_repos_find_with_invalid_repo(configtype, default):
)
args = ["repos", "find", "local", tmpdir]
if not default:
if not default_format:
args += ["--format", configtype]
cmd = grm(args)
assert cmd.returncode == 0
assert "broken" in cmd.stderr
if default or configtype == "toml":
if default_format or configtype == "toml":
output = toml.loads(cmd.stdout)
elif configtype == "yaml":
output = yaml.safe_load(cmd.stdout)

34
e2e_tests/test_repos_find_remote.py
View File

@@ -1,14 +1,13 @@
#!/usr/bin/env python3
import re
import os
import re
import tempfile
import toml
import pytest
import toml
import yaml
from helpers import *
from helpers import grm
ALTERNATE_DOMAIN = os.environ["ALTERNATE_DOMAIN"]
PROVIDERS = ["github", "gitlab"]
@@ -275,9 +274,9 @@ def test_repos_find_remote_user(
if not worktree_default:
cfg += f"worktree = {str(worktree).lower()}\n"
if force_ssh:
cfg += f"force_ssh = true\n"
cfg += "force_ssh = true\n"
if override_remote_name:
cfg += f'remote_name = "otherremote"\n'
cfg += 'remote_name = "otherremote"\n'
if use_owner:
cfg += """
[filters]
@@ -475,7 +474,7 @@ def test_repos_find_remote_group(
if not worktree_default:
cfg += f"worktree = {str(worktree).lower()}\n"
if force_ssh:
cfg += f"force_ssh = true\n"
cfg += "force_ssh = true\n"
if use_alternate_endpoint:
cfg += f'api_url = "http://{ALTERNATE_DOMAIN}:5000/{provider}"\n'
cfg += """
@@ -591,7 +590,7 @@ def test_repos_find_remote_user_and_group(
if not worktree_default:
cfg += f"worktree = {str(worktree).lower()}\n"
if force_ssh:
cfg += f"force_ssh = true\n"
cfg += "force_ssh = true\n"
if use_alternate_endpoint:
cfg += f'api_url = "http://{ALTERNATE_DOMAIN}:5000/{provider}"\n'
cfg += """
@@ -742,7 +741,7 @@ def test_repos_find_remote_owner(
if not worktree_default:
cfg += f"worktree = {str(worktree).lower()}\n"
if force_ssh:
cfg += f"force_ssh = true\n"
cfg += "force_ssh = true\n"
if use_alternate_endpoint:
cfg += f'api_url = "http://{ALTERNATE_DOMAIN}:5000/{provider}"\n'
cfg += """
@@ -873,13 +872,11 @@ def test_repos_find_remote_owner(
assert repo["remotes"][0]["name"] == "origin"
if force_ssh:
assert (
repo["remotes"][0]["url"] == f"ssh://git@example.com/myuser2/myproject3.git"
repo["remotes"][0]["url"] == "ssh://git@example.com/myuser2/myproject3.git"
)
assert repo["remotes"][0]["type"] == "ssh"
else:
assert (
repo["remotes"][0]["url"] == f"https://example.com/myuser2/myproject3.git"
)
assert repo["remotes"][0]["url"] == "https://example.com/myuser2/myproject3.git"
assert repo["remotes"][0]["type"] == "https"
group_namespace_1 = [t for t in output["trees"] if t["root"] == "/myroot/mygroup1"][
@@ -923,13 +920,13 @@ def test_repos_find_remote_owner(
if force_ssh:
assert (
repo["remotes"][0]["url"]
== f"ssh://git@example.com/mygroup1/myproject4.git"
== "ssh://git@example.com/mygroup1/myproject4.git"
)
assert repo["remotes"][0]["type"] == "ssh"
else:
assert (
repo["remotes"][0]["url"]
== f"https://example.com/mygroup1/myproject4.git"
== "https://example.com/mygroup1/myproject4.git"
)
assert repo["remotes"][0]["type"] == "https"
@@ -948,12 +945,11 @@ def test_repos_find_remote_owner(
assert repo["remotes"][0]["name"] == "origin"
if force_ssh:
assert (
repo["remotes"][0]["url"]
== f"ssh://git@example.com/mygroup2/myproject5.git"
repo["remotes"][0]["url"] == "ssh://git@example.com/mygroup2/myproject5.git"
)
assert repo["remotes"][0]["type"] == "ssh"
else:
assert (
repo["remotes"][0]["url"] == f"https://example.com/mygroup2/myproject5.git"
repo["remotes"][0]["url"] == "https://example.com/mygroup2/myproject5.git"
)
assert repo["remotes"][0]["type"] == "https"

4
e2e_tests/test_repos_status.py
View File

@@ -1,8 +1,6 @@
#!/usr/bin/env python3
import tempfile
from helpers import *
from helpers import RepoTree, grm
def test_repos_sync_worktree_clone():

33
e2e_tests/test_repos_sync.py
View File

@@ -1,14 +1,21 @@
#!/usr/bin/env python3
import tempfile
import os
import re
import subprocess
import tempfile
import textwrap
import pytest
import toml
import git
from helpers import *
import pytest
from helpers import (
NonExistentPath,
TempGitFileRemote,
TempGitRepository,
checksum_directory,
grm,
shell,
)
templates = {
"repo_simple": {
@@ -291,7 +298,9 @@ def test_repos_sync_unmanaged_repos(configtype):
# this removes the prefix (root) from the path (unmanaged_repo)
unmanaged_repo_name = os.path.relpath(unmanaged_repo, root)
regex = f".*unmanaged.*{unmanaged_repo_name}"
assert any([re.match(regex, l) for l in cmd.stderr.lower().split("\n")])
assert any(
[re.match(regex, line) for line in cmd.stderr.lower().split("\n")]
)
@pytest.mark.parametrize("configtype", ["toml", "yaml"])
@@ -374,7 +383,7 @@ def test_repos_sync_repo_in_subdirectory(configtype):
assert urls[0] == f"file://{remote}"
cmd = grm(["repos", "sync", "config", "--config", config.name])
assert not "found unmanaged repository" in cmd.stderr.lower()
assert "found unmanaged repository" not in cmd.stderr.lower()
@pytest.mark.parametrize("configtype", ["toml", "yaml"])
@@ -419,7 +428,7 @@ def test_repos_sync_nested_clone(configtype):
cmd = grm(["repos", "sync", "config", "--config", config.name])
print(cmd.stdout)
print(cmd.stderr)
assert not "found unmanaged repository" in cmd.stderr.lower()
assert "found unmanaged repository" not in cmd.stderr.lower()
@pytest.mark.parametrize("configtype", ["toml", "yaml"])
@@ -720,14 +729,14 @@ def test_repos_sync_invalid_syntax(configtype):
with open(config.name, "w") as f:
if configtype == "toml":
f.write(
f"""
"""
[[trees]]
root = invalid as there are no quotes ;)
"""
)
elif configtype == "yaml":
f.write(
f"""
"""
trees:
wrong:
indentation:
@@ -779,8 +788,6 @@ def test_repos_sync_normal_change_to_worktree(configtype):
cmd = grm(["repos", "sync", "config", "--config", config.name])
assert cmd.returncode == 0
git_dir = os.path.join(target, "test")
with open(config.name, "w") as f:
f.write(
templates["worktree_repo_with_remote"][configtype].format(
@@ -810,8 +817,6 @@ def test_repos_sync_worktree_change_to_normal(configtype):
cmd = grm(["repos", "sync", "config", "--config", config.name])
assert cmd.returncode == 0
git_dir = os.path.join(target, "test")
with open(config.name, "w") as f:
f.write(
templates["repo_with_remote"][configtype].format(

17
e2e_tests/test_worktree_clean.py
View File

@@ -1,8 +1,17 @@
#!/usr/bin/env python3
import pytest
import os
from helpers import *
import pytest
from helpers import (
NonGitDir,
TempGitRepository,
TempGitRepositoryWorktree,
checksum_directory,
funcname,
grm,
shell,
)
def test_worktree_clean():
@@ -153,13 +162,13 @@ def test_worktree_clean_configured_default_branch(
with open(os.path.join(base_dir, "grm.toml"), "w") as f:
if branch_list_empty:
f.write(
f"""
"""
persistent_branches = []
"""
)
else:
f.write(
f"""
"""
persistent_branches = [
"mybranch"
]

3
e2e_tests/test_worktree_config_presistent_branch.py
View File

@@ -2,7 +2,8 @@
import os.path
from helpers import *
import git
from helpers import TempGitRepositoryWorktree, checksum_directory, funcname, grm, shell
def test_worktree_never_clean_persistent_branches():

12
e2e_tests/test_worktree_conversion.py
View File

@@ -1,8 +1,16 @@
#!/usr/bin/env python3
import tempfile
import os
from helpers import *
from helpers import (
EmptyDir,
NonGitDir,
TempGitRepository,
TempGitRepositoryWorktree,
checksum_directory,
funcname,
grm,
)
def test_convert():

12
e2e_tests/test_worktree_fetch.py
View File

@@ -1,11 +1,17 @@
#!/usr/bin/env python3
from helpers import *
import re
import pytest
import git
import pytest
from helpers import (
EmptyDir,
TempGitFileRemote,
TempGitRepositoryWorktree,
funcname,
grm,
shell,
)
def test_worktree_fetch():

6
e2e_tests/test_worktree_rebase.py
View File

@@ -1,11 +1,11 @@
#!/usr/bin/env python3
from helpers import *
import os
import re
import pytest
import git
import pytest
from helpers import TempGitRepositoryWorktree, funcname, grm, shell
@pytest.mark.parametrize("pull", [True, False])

11
e2e_tests/test_worktree_status.py
View File

@@ -1,10 +1,17 @@
#!/usr/bin/env python3
import os
import re
from helpers import *
import pytest
from helpers import (
NonGitDir,
TempGitRepository,
TempGitRepositoryWorktree,
funcname,
grm,
shell,
)
@pytest.mark.parametrize("has_config", [True, False])

20
e2e_tests/test_worktrees.py
View File

@@ -1,12 +1,18 @@
#!/usr/bin/env python3
from helpers import *
import datetime
import os.path
import git
import pytest
import datetime
import os.path
from helpers import (
TempGitRepositoryWorktree,
checksum_directory,
funcname,
grm,
shell,
tempfile,
)
@pytest.mark.parametrize(
@@ -151,7 +157,8 @@ def test_worktree_add(
]
)
cachefn = lambda nr: "_".join(
def cachefn(nr):
return "_".join(
[
str(nr),
str(default_remote),
@@ -163,6 +170,7 @@ def test_worktree_add(
str(worktree_name),
]
)
remote1_cache_key = cachefn(1)
remote2_cache_key = cachefn(2)
@@ -281,7 +289,7 @@ def test_worktree_add(
and remotes_differ
):
assert (
f"branch exists on multiple remotes, but they deviate"
"branch exists on multiple remotes, but they deviate"
in cmd.stderr.lower()
)
assert len(cmd.stderr.strip().split("\n")) == base + 1

3
rust-toolchain.toml Normal file
View File

@@ -0,0 +1,3 @@
[toolchain]
channel = "nightly"
targets = ["x86_64-unknown-linux-musl"]

8
src/grm/cmd.rs
View File

@@ -63,6 +63,14 @@ pub struct FindLocalArgs {
#[clap(help = "The path to search through")]
pub path: String,
#[clap(
short,
long,
help = "Exclude repositories that match the given regex",
name = "REGEX"
)]
pub exclude: Option<String>,
#[clap(
value_enum,
short,

3
src/grm/main.rs
View File

@@ -199,7 +199,8 @@ fn main() {
}
};
let (found_repos, warnings) = match find_in_tree(&path) {
let (found_repos, warnings) = match find_in_tree(&path, args.exclude.as_deref())
{
Ok((repos, warnings)) => (repos, warnings),
Err(error) => {
print_error(&error);

20
src/lib.rs
View File

@@ -19,12 +19,22 @@ pub mod worktree;
/// The bool in the return value specifies whether there is a repository
/// in root itself.
#[allow(clippy::type_complexity)]
fn find_repos(root: &Path) -> Result<Option<(Vec<repo::Repo>, Vec<String>, bool)>, String> {
fn find_repos(
root: &Path,
exclusion_pattern: Option<&str>,
) -> Result<Option<(Vec<repo::Repo>, Vec<String>, bool)>, String> {
let mut repos: Vec<repo::Repo> = Vec::new();
let mut repo_in_root = false;
let mut warnings = Vec::new();
let exlusion_regex: regex::Regex = regex::Regex::new(exclusion_pattern.unwrap_or(r"^$"))
.map_err(|e| format!("invalid regex: {e}"))?;
for path in tree::find_repo_paths(root)? {
if exclusion_pattern.is_some() && exlusion_regex.is_match(&path::path_as_string(&path)) {
warnings.push(format!("[skipped] {}", &path::path_as_string(&path)));
continue;
}
let is_worktree = repo::RepoHandle::detect_worktree(&path);
if path == root {
repo_in_root = true;
@@ -130,10 +140,14 @@ fn find_repos(root: &Path) -> Result<Option<(Vec<repo::Repo>, Vec<String>, bool)
Ok(Some((repos, warnings, repo_in_root)))
}
pub fn find_in_tree(path: &Path) -> Result<(tree::Tree, Vec<String>), String> {
pub fn find_in_tree(
path: &Path,
exclusion_pattern: Option<&str>,
) -> Result<(tree::Tree, Vec<String>), String> {
let mut warnings = Vec::new();
let (repos, repo_in_root): (Vec<repo::Repo>, bool) = match find_repos(path)? {
let (repos, repo_in_root): (Vec<repo::Repo>, bool) = match find_repos(path, exclusion_pattern)?
{
Some((vec, mut repo_warnings, repo_in_root)) => {
warnings.append(&mut repo_warnings);
(vec, repo_in_root)