hannes/git-repo-manager
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

e2e: Add caching to git repositories

Browse Source 
It's very expensive to create new repositories from scratch. To avoid
this, a new repo & remotes are only created if necessary (depending on a
cache key given on request). If not created, they are simply copied from
a stored, clean repository / remote.
This commit is contained in:
Hannes Körber
2022-06-23 19:08:55 +02:00
parent 8c384741b3
commit 88961e1c6b
9 changed files with 236 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
e2e_tests/conftest.py
View File

@@ -1,8 +1,14 @@
import os
from helpers import *
def pytest_configure(config):
os.environ["GIT_AUTHOR_NAME"] = "Example user"
os.environ["GIT_AUTHOR_EMAIL"] = "user@example.com"
os.environ["GIT_COMMITTER_NAME"] = "Example user"
os.environ["GIT_COMMITTER_EMAIL"] = "user@example.com"
def pytest_unconfigure(config):
pass

258
e2e_tests/helpers.py
View File

@@ -5,12 +5,26 @@ import os.path
import subprocess
import tempfile
import hashlib
import shutil
import inspect
import git
binary = os.environ["GRM_BINARY"]
def funcname():
return inspect.stack()[1][3]
def copytree(src, dest):
shutil.copytree(src, dest, dirs_exist_ok=True)
def get_temporary_directory(dir=None):
return tempfile.TemporaryDirectory(dir=dir)
def grm(args, cwd=None, is_invalid=False):
cmd = subprocess.run([binary] + args, cwd=cwd, capture_output=True, text=True)
if not is_invalid:
@@ -116,78 +130,204 @@ def checksum_directory(path):
class TempGitRepository:
def __init__(self, dir=None):
self.dir = dir
pass
def __enter__(self):
self.tmpdir = tempfile.TemporaryDirectory(dir=self.dir)
self.remote_1_dir = tempfile.TemporaryDirectory()
self.remote_2_dir = tempfile.TemporaryDirectory()
shell(
f"""
self.tmpdir = get_temporary_directory(self.dir)
self.remote_1 = get_temporary_directory()
self.remote_2 = get_temporary_directory()
cmd = f"""
cd {self.tmpdir.name}
git init
git -c init.defaultBranch=master init
echo test > root-commit
git add root-commit
git commit -m "root-commit"
git remote add origin file://{self.remote_1_dir.name}
git remote add otherremote file://{self.remote_2_dir.name}
git remote add origin file://{self.remote_1.name}
git remote add otherremote file://{self.remote_2.name}
"""
)
shell(cmd)
return self.tmpdir.name
def __exit__(self, exc_type, exc_val, exc_tb):
del self.tmpdir
del self.remote_1_dir
del self.remote_2_dir
pass
class TempGitRemote:
obj = {}
def __init__(self, tmpdir, remoteid=None):
self.tmpdir = tmpdir
self.remoteid = remoteid
@classmethod
def get(cls, cachekey=None, initfunc=None):
if cachekey is None:
tmpdir = get_temporary_directory()
shell(
f"""
cd {tmpdir.name}
git -c init.defaultBranch=master init --bare
"""
)
newobj = cls(tmpdir)
remoteid = None
if initfunc is not None:
remoteid = newobj.init(initfunc)
newobj.remoteid = remoteid
return newobj, remoteid
else:
refresh = False
if cachekey not in cls.obj:
tmpdir = get_temporary_directory()
shell(
f"""
cd {tmpdir.name}
git -c init.defaultBranch=master init --bare
"""
)
newobj = cls(tmpdir)
remoteid = newobj.init(initfunc)
newobj.remoteid = remoteid
cls.obj[cachekey] = newobj
return cls.clone(cls.obj[cachekey])
@classmethod
def clone(cls, source):
new_remote = get_temporary_directory()
copytree(source.tmpdir.name, new_remote.name)
return cls(new_remote, source.remoteid), source.remoteid
def init(self, func):
return func(self.tmpdir.name)
def __enter__(self):
return self.tmpdir
def __exit__(self, exc_type, exc_val, exc_tb):
pass
class TempGitRepositoryWorktree:
def __init__(self):
pass
obj = {}
def __init__(self, remotes, tmpdir, commit, remote1, remote2, remote1id, remote2id):
self.remotes = remotes
self.tmpdir = tmpdir
self.commit = commit
self.remote1 = remote1
self.remote2 = remote2
self.remote1id = remote1id
self.remote2id = remote2id
@classmethod
def get(cls, cachekey, branch=None, remotes=2, basedir=None, remote_setup=None):
if cachekey not in cls.obj:
tmpdir = get_temporary_directory()
shell(
f"""
cd {tmpdir.name}
git -c init.defaultBranch=master init
echo test > root-commit-in-worktree-1
git add root-commit-in-worktree-1
git commit -m "root-commit-in-worktree-1"
echo test > root-commit-in-worktree-2
git add root-commit-in-worktree-2
git commit -m "root-commit-in-worktree-2"
git ls-files | xargs rm -rf
mv .git .git-main-working-tree
git --git-dir .git-main-working-tree config core.bare true
"""
)
repo = git.Repo(f"{tmpdir.name}/.git-main-working-tree")
commit = repo.head.commit.hexsha
if branch is not None:
repo.create_head(branch)
remote1 = None
remote2 = None
remote1id = None
remote2id = None
if remotes >= 1:
cachekeyremote, initfunc = (remote_setup or ((None, None),))[0]
remote1, remote1id = TempGitRemote.get(
cachekey=cachekeyremote, initfunc=initfunc
)
remote1 = remote1
remote1id = remote1id
shell(
f"""
cd {tmpdir.name}
git --git-dir .git-main-working-tree remote add origin file://{remote1.tmpdir.name}
"""
)
repo.remotes.origin.fetch()
repo.remotes.origin.push("master")
if remotes >= 2:
cachekeyremote, initfunc = (remote_setup or (None, (None, None)))[1]
remote2, remote2id = TempGitRemote.get(
cachekey=cachekeyremote, initfunc=initfunc
)
remote2 = remote2
remote2id = remote2id
shell(
f"""
cd {tmpdir.name}
git --git-dir .git-main-working-tree remote add otherremote file://{remote2.tmpdir.name}
"""
)
repo.remotes.otherremote.fetch()
repo.remotes.otherremote.push("master")
cls.obj[cachekey] = cls(
remotes, tmpdir, commit, remote1, remote2, remote1id, remote2id
)
return cls.clone(cls.obj[cachekey], remote_setup=remote_setup)
@classmethod
def clone(cls, source, remote_setup):
newdir = get_temporary_directory()
copytree(source.tmpdir.name, newdir.name)
remote1 = None
remote2 = None
remote1id = None
remote2id = None
repo = git.Repo(os.path.join(newdir.name, ".git-main-working-tree"))
if source.remotes >= 1:
cachekey, initfunc = (remote_setup or ((None, None),))[0]
remote1, remote1id = TempGitRemote.get(cachekey=cachekey, initfunc=initfunc)
if remote1id != source.remote1id:
repo.remotes.origin.fetch()
repo.remotes.origin.push("master")
if source.remotes >= 2:
cachekey, initfunc = (remote_setup or (None, (None, None)))[1]
remote2, remote2id = TempGitRemote.get(cachekey=cachekey, initfunc=initfunc)
if remote2id != source.remote2id:
repo.remotes.otherremote.fetch()
repo.remotes.otherremote.push("master")
return cls(
source.remotes,
newdir,
source.commit,
remote1,
remote2,
remote1id,
remote2id,
)
def __enter__(self):
self.tmpdir = tempfile.TemporaryDirectory()
self.remote_1_dir = tempfile.TemporaryDirectory()
self.remote_2_dir = tempfile.TemporaryDirectory()
shell(
f"""
cd {self.remote_1_dir.name}
git init --bare
"""
)
shell(
f"""
cd {self.remote_2_dir.name}
git init --bare
"""
)
shell(
f"""
cd {self.tmpdir.name}
git init
echo test > root-commit-in-worktree-1
git add root-commit-in-worktree-1
git commit -m "root-commit-in-worktree-1"
echo test > root-commit-in-worktree-2
git add root-commit-in-worktree-2
git commit -m "root-commit-in-worktree-2"
git remote add origin file://{self.remote_1_dir.name}
git remote add otherremote file://{self.remote_2_dir.name}
git push origin HEAD:master
git ls-files | xargs rm -rf
mv .git .git-main-working-tree
git --git-dir .git-main-working-tree config core.bare true
"""
)
commit = git.Repo(
f"{self.tmpdir.name}/.git-main-working-tree"
).head.commit.hexsha
return (self.tmpdir.name, commit)
return (self.tmpdir.name, self.commit)
def __exit__(self, exc_type, exc_val, exc_tb):
del self.tmpdir
del self.remote_1_dir
del self.remote_2_dir
pass
class RepoTree:
@@ -195,7 +335,7 @@ class RepoTree:
pass
def __enter__(self):
self.root = tempfile.TemporaryDirectory()
self.root = get_temporary_directory()
self.config = tempfile.NamedTemporaryFile()
with open(self.config.name, "w") as f:
f.write(
@@ -226,7 +366,7 @@ class EmptyDir:
pass
def __enter__(self):
self.tmpdir = tempfile.TemporaryDirectory()
self.tmpdir = get_temporary_directory()
return self.tmpdir.name
def __exit__(self, exc_type, exc_val, exc_tb):
@@ -238,7 +378,7 @@ class NonGitDir:
pass
def __enter__(self):
self.tmpdir = tempfile.TemporaryDirectory()
self.tmpdir = get_temporary_directory()
shell(
f"""
cd {self.tmpdir.name}
@@ -258,7 +398,7 @@ class TempGitFileRemote:
pass
def __enter__(self):
self.tmpdir = tempfile.TemporaryDirectory()
self.tmpdir = get_temporary_directory()
shell(
f"""
cd {self.tmpdir.name}

18
e2e_tests/test_worktree_clean.py
View File

@@ -6,7 +6,7 @@ from helpers import *
def test_worktree_clean():
with TempGitRepositoryWorktree() as (base_dir, _commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, _commit):
cmd = grm(["wt", "add", "test", "--track", "origin/test"], cwd=base_dir)
assert cmd.returncode == 0
assert "test" in os.listdir(base_dir)
@@ -17,7 +17,7 @@ def test_worktree_clean():
def test_worktree_clean_refusal_no_tracking_branch():
with TempGitRepositoryWorktree() as (base_dir, _commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, _commit):
cmd = grm(["wt", "add", "test"], cwd=base_dir)
assert cmd.returncode == 0
@@ -31,7 +31,7 @@ def test_worktree_clean_refusal_no_tracking_branch():
def test_worktree_clean_refusal_uncommited_changes_new_file():
with TempGitRepositoryWorktree() as (base_dir, _commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, _commit):
cmd = grm(["wt", "add", "test", "--track", "origin/test"], cwd=base_dir)
assert cmd.returncode == 0
@@ -47,7 +47,7 @@ def test_worktree_clean_refusal_uncommited_changes_new_file():
def test_worktree_clean_refusal_uncommited_changes_changed_file():
with TempGitRepositoryWorktree() as (base_dir, _commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, _commit):
cmd = grm(["wt", "add", "test", "--track", "origin/test"], cwd=base_dir)
assert cmd.returncode == 0
@@ -63,7 +63,7 @@ def test_worktree_clean_refusal_uncommited_changes_changed_file():
def test_worktree_clean_refusal_uncommited_changes_cleand_file():
with TempGitRepositoryWorktree() as (base_dir, _commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, _commit):
cmd = grm(["wt", "add", "test", "--track", "origin/test"], cwd=base_dir)
assert cmd.returncode == 0
@@ -81,7 +81,7 @@ def test_worktree_clean_refusal_uncommited_changes_cleand_file():
def test_worktree_clean_refusal_commited_changes():
with TempGitRepositoryWorktree() as (base_dir, _commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, _commit):
cmd = grm(["wt", "add", "test", "--track", "origin/test"], cwd=base_dir)
assert cmd.returncode == 0
@@ -99,7 +99,7 @@ def test_worktree_clean_refusal_commited_changes():
def test_worktree_clean_refusal_tracking_branch_mismatch():
with TempGitRepositoryWorktree() as (base_dir, _commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, _commit):
cmd = grm(["wt", "add", "test", "--track", "origin/test"], cwd=base_dir)
assert cmd.returncode == 0
@@ -117,7 +117,7 @@ def test_worktree_clean_refusal_tracking_branch_mismatch():
def test_worktree_clean_fail_from_subdir():
with TempGitRepositoryWorktree() as (base_dir, _commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, _commit):
cmd = grm(["wt", "add", "test"], cwd=base_dir)
assert cmd.returncode == 0
@@ -148,7 +148,7 @@ def test_worktree_clean_non_git():
def test_worktree_clean_configured_default_branch(
configure_default_branch, branch_list_empty
):
with TempGitRepositoryWorktree() as (base_dir, _commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, _commit):
if configure_default_branch:
with open(os.path.join(base_dir, "grm.toml"), "w") as f:
if branch_list_empty:

8
e2e_tests/test_worktree_config_presistent_branch.py
View File

@@ -6,7 +6,7 @@ from helpers import *
def test_worktree_never_clean_persistent_branches():
with TempGitRepositoryWorktree() as (base_dir, _commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, _commit):
with open(os.path.join(base_dir, "grm.toml"), "w") as f:
f.write(
"""
@@ -33,7 +33,7 @@ def test_worktree_never_clean_persistent_branches():
def test_worktree_clean_branch_merged_into_persistent():
with TempGitRepositoryWorktree() as (base_dir, _commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, _commit):
with open(os.path.join(base_dir, "grm.toml"), "w") as f:
f.write(
"""
@@ -72,7 +72,7 @@ def test_worktree_clean_branch_merged_into_persistent():
def test_worktree_no_clean_unmerged_branch():
with TempGitRepositoryWorktree() as (base_dir, _commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, _commit):
with open(os.path.join(base_dir, "grm.toml"), "w") as f:
f.write(
"""
@@ -105,7 +105,7 @@ def test_worktree_no_clean_unmerged_branch():
def test_worktree_delete_branch_merged_into_persistent():
with TempGitRepositoryWorktree() as (base_dir, _commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, _commit):
with open(os.path.join(base_dir, "grm.toml"), "w") as f:
f.write(
"""

2
e2e_tests/test_worktree_conversion.py
View File

@@ -23,7 +23,7 @@ def test_convert():
def test_convert_already_worktree():
with TempGitRepositoryWorktree() as (git_dir, _commit):
with TempGitRepositoryWorktree.get(funcname()) as (git_dir, _commit):
before = checksum_directory(git_dir)
cmd = grm(["wt", "convert"], cwd=git_dir)

4
e2e_tests/test_worktree_fetch.py
View File

@@ -9,7 +9,7 @@ import git
def test_worktree_fetch():
with TempGitRepositoryWorktree() as (base_dir, root_commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, root_commit):
with TempGitFileRemote() as (remote_path, _remote_sha):
shell(
f"""
@@ -56,7 +56,7 @@ def test_worktree_fetch():
@pytest.mark.parametrize("has_changes", [True, False])
@pytest.mark.parametrize("stash", [True, False])
def test_worktree_pull(rebase, ffable, has_changes, stash):
with TempGitRepositoryWorktree() as (base_dir, root_commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, root_commit):
with TempGitFileRemote() as (remote_path, _remote_sha):
shell(
f"""

2
e2e_tests/test_worktree_rebase.py
View File

@@ -14,7 +14,7 @@ import git
@pytest.mark.parametrize("has_changes", [True, False])
@pytest.mark.parametrize("stash", [True, False])
def test_worktree_rebase(pull, rebase, ffable, has_changes, stash):
with TempGitRepositoryWorktree() as (base_dir, _root_commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, _root_commit):
with open(os.path.join(base_dir, "grm.toml"), "w") as f:
f.write('persistent_branches = ["mybasebranch"]')

6
e2e_tests/test_worktree_status.py
View File

@@ -9,7 +9,7 @@ import pytest
@pytest.mark.parametrize("has_config", [True, False])
def test_worktree_status(has_config):
with TempGitRepositoryWorktree() as (base_dir, _commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, _commit):
if has_config:
with open(os.path.join(base_dir, "grm.toml"), "w") as f:
f.write("")
@@ -24,7 +24,7 @@ def test_worktree_status(has_config):
def test_worktree_status_fail_from_subdir():
with TempGitRepositoryWorktree() as (base_dir, _commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, _commit):
cmd = grm(["wt", "add", "test"], cwd=base_dir)
assert cmd.returncode == 0
@@ -51,7 +51,7 @@ def test_worktree_status_non_git():
def test_worktree_status_warn_with_non_worktree_dir():
with TempGitRepositoryWorktree() as (base_dir, _commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, _commit):
cmd = grm(["wt", "add", "test"], cwd=base_dir)
assert cmd.returncode == 0

22
e2e_tests/test_worktrees.py
View File

@@ -96,7 +96,7 @@ def test_worktree_add(
def test_worktree_add_invalid_name():
with TempGitRepositoryWorktree() as (base_dir, _commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, _commit):
for worktree_name in ["/absolute/path" "trailingslash/"]:
args = ["wt", "add", worktree_name]
cmd = grm(args, cwd=base_dir)
@@ -127,7 +127,7 @@ def test_worktree_add_into_invalid_subdirectory():
def test_worktree_add_with_tracking(
remote_branch_already_exists, has_config, has_default, has_prefix
):
with TempGitRepositoryWorktree() as (base_dir, _commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, _commit):
if has_config:
with open(os.path.join(base_dir, "grm.toml"), "w") as f:
f.write(
@@ -229,7 +229,7 @@ def test_worktree_add_with_explicit_no_tracking(
def test_worktree_delete():
with TempGitRepositoryWorktree() as (base_dir, _commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, _commit):
cmd = grm(["wt", "add", "test", "--track", "origin/test"], cwd=base_dir)
assert cmd.returncode == 0
assert "test" in os.listdir(base_dir)
@@ -246,7 +246,7 @@ def test_worktree_delete():
def test_worktree_delete_refusal_no_tracking_branch():
with TempGitRepositoryWorktree() as (base_dir, _commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, _commit):
cmd = grm(["wt", "add", "test"], cwd=base_dir)
assert cmd.returncode == 0
@@ -262,7 +262,7 @@ def test_worktree_delete_refusal_no_tracking_branch():
def test_worktree_delete_refusal_uncommited_changes_new_file():
with TempGitRepositoryWorktree() as (base_dir, _commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, _commit):
cmd = grm(["wt", "add", "test", "--track", "origin/test"], cwd=base_dir)
assert cmd.returncode == 0
@@ -280,7 +280,7 @@ def test_worktree_delete_refusal_uncommited_changes_new_file():
def test_worktree_delete_refusal_uncommited_changes_changed_file():
with TempGitRepositoryWorktree() as (base_dir, _commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, _commit):
cmd = grm(["wt", "add", "test", "--track", "origin/test"], cwd=base_dir)
assert cmd.returncode == 0
@@ -298,7 +298,7 @@ def test_worktree_delete_refusal_uncommited_changes_changed_file():
def test_worktree_delete_refusal_uncommited_changes_deleted_file():
with TempGitRepositoryWorktree() as (base_dir, _commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, _commit):
cmd = grm(["wt", "add", "test", "--track", "origin/test"], cwd=base_dir)
assert cmd.returncode == 0
@@ -318,7 +318,7 @@ def test_worktree_delete_refusal_uncommited_changes_deleted_file():
def test_worktree_delete_refusal_commited_changes():
with TempGitRepositoryWorktree() as (base_dir, _commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, _commit):
cmd = grm(["wt", "add", "test", "--track", "origin/test"], cwd=base_dir)
assert cmd.returncode == 0
@@ -338,7 +338,7 @@ def test_worktree_delete_refusal_commited_changes():
def test_worktree_delete_refusal_tracking_branch_mismatch():
with TempGitRepositoryWorktree() as (base_dir, _commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, _commit):
cmd = grm(["wt", "add", "test", "--track", "origin/test"], cwd=base_dir)
assert cmd.returncode == 0
@@ -358,7 +358,7 @@ def test_worktree_delete_refusal_tracking_branch_mismatch():
def test_worktree_delete_force_refusal():
with TempGitRepositoryWorktree() as (base_dir, _commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, _commit):
cmd = grm(["wt", "add", "test"], cwd=base_dir)
assert cmd.returncode == 0
@@ -368,7 +368,7 @@ def test_worktree_delete_force_refusal():
def test_worktree_add_delete_add():
with TempGitRepositoryWorktree() as (base_dir, _commit):
with TempGitRepositoryWorktree.get(funcname()) as (base_dir, _commit):
cmd = grm(["wt", "add", "test", "--track", "origin/test"], cwd=base_dir)
assert cmd.returncode == 0
assert "test" in os.listdir(base_dir)