From fb0c5fd06184f33a03d8d4fb29a3aef8b1dafe78 Mon Sep 17 00:00:00 2001 From: Michael Muré Date: Sun, 8 Nov 2020 17:54:28 +0100 Subject: repo: expose all lamport clocks, move clocks in their own folder --- repository/gogit.go | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) (limited to 'repository/gogit.go') diff --git a/repository/gogit.go b/repository/gogit.go index bdac259d..5abdef39 100644 --- a/repository/gogit.go +++ b/repository/gogit.go @@ -24,6 +24,8 @@ import ( "github.com/MichaelMure/git-bug/util/lamport" ) +const clockPath = "clocks" + var _ ClockedRepo = &GoGitRepo{} var _ TestedRepo = &GoGitRepo{} @@ -677,6 +679,37 @@ func (repo *GoGitRepo) ListCommits(ref string) ([]Hash, error) { return hashes, nil } +func (repo *GoGitRepo) AllClocks() (map[string]lamport.Clock, error) { + repo.clocksMutex.Lock() + defer repo.clocksMutex.Unlock() + + result := make(map[string]lamport.Clock) + + files, err := ioutil.ReadDir(filepath.Join(repo.path, "git-bug", clockPath)) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, err + } + + for _, file := range files { + name := file.Name() + if c, ok := repo.clocks[name]; ok { + result[name] = c + } else { + c, err := lamport.LoadPersistedClock(repo.LocalStorage(), filepath.Join(clockPath, name)) + if err != nil { + return nil, err + } + repo.clocks[name] = c + result[name] = c + } + } + + return result, nil +} + // GetOrCreateClock return a Lamport clock stored in the Repo. // If the clock doesn't exist, it's created. func (repo *GoGitRepo) GetOrCreateClock(name string) (lamport.Clock, error) { @@ -691,7 +724,7 @@ func (repo *GoGitRepo) GetOrCreateClock(name string) (lamport.Clock, error) { return nil, err } - c, err = lamport.NewPersistedClock(repo.localStorage, name+"-clock") + c, err = lamport.NewPersistedClock(repo.LocalStorage(), filepath.Join(clockPath, name)) if err != nil { return nil, err } @@ -705,7 +738,7 @@ func (repo *GoGitRepo) getClock(name string) (lamport.Clock, error) { return c, nil } - c, err := lamport.LoadPersistedClock(repo.localStorage, name+"-clock") + c, err := lamport.LoadPersistedClock(repo.LocalStorage(), filepath.Join(clockPath, name)) if err == nil { repo.clocks[name] = c return c, nil @@ -716,6 +749,24 @@ func (repo *GoGitRepo) getClock(name string) (lamport.Clock, error) { return nil, err } +// Increment is equivalent to c = GetOrCreateClock(name) + c.Increment() +func (repo *GoGitRepo) Increment(name string) (lamport.Time, error) { + c, err := repo.GetOrCreateClock(name) + if err != nil { + return lamport.Time(0), err + } + return c.Increment() +} + +// Witness is equivalent to c = GetOrCreateClock(name) + c.Witness(time) +func (repo *GoGitRepo) Witness(name string, time lamport.Time) error { + c, err := repo.GetOrCreateClock(name) + if err != nil { + return err + } + return c.Witness(time) +} + // AddRemote add a new remote to the repository // Not in the interface because it's only used for testing func (repo *GoGitRepo) AddRemote(name string, url string) error { -- cgit From 5c4e7de01281da51e32b4926dc0ef15b17a2d397 Mon Sep 17 00:00:00 2001 From: Michael Muré Date: Mon, 21 Dec 2020 11:05:47 +0100 Subject: repository: partially add two new functions to RepoData --- repository/gogit.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) (limited to 'repository/gogit.go') diff --git a/repository/gogit.go b/repository/gogit.go index 5abdef39..64ccb773 100644 --- a/repository/gogit.go +++ b/repository/gogit.go @@ -595,6 +595,14 @@ func (repo *GoGitRepo) FindCommonAncestor(commit1 Hash, commit2 Hash) (Hash, err return Hash(commits[0].Hash.String()), nil } +func (repo *GoGitRepo) ResolveRef(ref string) (Hash, error) { + r, err := repo.r.Reference(plumbing.ReferenceName(ref), false) + if err != nil { + return "", err + } + return Hash(r.Hash().String()), nil +} + // UpdateRef will create or update a Git reference func (repo *GoGitRepo) UpdateRef(ref string, hash Hash) error { return repo.r.Storer.SetReference(plumbing.NewHashReference(plumbing.ReferenceName(ref), plumbing.NewHash(hash.String()))) @@ -679,6 +687,25 @@ func (repo *GoGitRepo) ListCommits(ref string) ([]Hash, error) { return hashes, nil } +func (repo *GoGitRepo) ReadCommit(hash Hash) (Commit, error) { + commit, err := repo.r.CommitObject(plumbing.NewHash(hash.String())) + if err != nil { + return Commit{}, err + } + + parents := make([]Hash, len(commit.ParentHashes)) + for i, parentHash := range commit.ParentHashes { + parents[i] = Hash(parentHash.String()) + } + + return Commit{ + Hash: hash, + Parents: parents, + TreeHash: Hash(commit.TreeHash.String()), + }, nil + +} + func (repo *GoGitRepo) AllClocks() (map[string]lamport.Clock, error) { repo.clocksMutex.Lock() defer repo.clocksMutex.Unlock() -- cgit From 8d63c983c982f93cc48d3996d6bd097ddeeb327f Mon Sep 17 00:00:00 2001 From: Michael Muré Date: Sun, 3 Jan 2021 23:59:25 +0100 Subject: WIP --- repository/gogit.go | 92 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 56 insertions(+), 36 deletions(-) (limited to 'repository/gogit.go') diff --git a/repository/gogit.go b/repository/gogit.go index 64ccb773..d6eb8621 100644 --- a/repository/gogit.go +++ b/repository/gogit.go @@ -20,6 +20,7 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/filemode" "github.com/go-git/go-git/v5/plumbing/object" + "golang.org/x/crypto/openpgp" "github.com/MichaelMure/git-bug/util/lamport" ) @@ -521,12 +522,13 @@ func (repo *GoGitRepo) ReadTree(hash Hash) ([]TreeEntry, error) { } // StoreCommit will store a Git commit with the given Git tree -func (repo *GoGitRepo) StoreCommit(treeHash Hash) (Hash, error) { - return repo.StoreCommitWithParent(treeHash, "") +func (repo *GoGitRepo) StoreCommit(treeHash Hash, parents ...Hash) (Hash, error) { + return repo.StoreSignedCommit(treeHash, nil, parents...) } -// StoreCommit will store a Git commit with the given Git tree -func (repo *GoGitRepo) StoreCommitWithParent(treeHash Hash, parent Hash) (Hash, error) { +// StoreCommit will store a Git commit with the given Git tree. If signKey is not nil, the commit +// will be signed accordingly. +func (repo *GoGitRepo) StoreSignedCommit(treeHash Hash, signKey *openpgp.Entity, parents ...Hash) (Hash, error) { cfg, err := repo.r.Config() if err != nil { return "", err @@ -547,8 +549,28 @@ func (repo *GoGitRepo) StoreCommitWithParent(treeHash Hash, parent Hash) (Hash, TreeHash: plumbing.NewHash(treeHash.String()), } - if parent != "" { - commit.ParentHashes = []plumbing.Hash{plumbing.NewHash(parent.String())} + for _, parent := range parents { + commit.ParentHashes = append(commit.ParentHashes, plumbing.NewHash(parent.String())) + } + + // Compute the signature if needed + if signKey != nil { + // first get the serialized commit + encoded := &plumbing.MemoryObject{} + if err := commit.Encode(encoded); err != nil { + return "", err + } + r, err := encoded.Reader() + if err != nil { + return "", err + } + + // sign the data + var sig bytes.Buffer + if err := openpgp.ArmoredDetachSign(&sig, signKey, r, nil); err != nil { + return "", err + } + commit.PGPSignature = sig.String() } obj := repo.r.Storer.NewEncodedObject() @@ -608,6 +630,13 @@ func (repo *GoGitRepo) UpdateRef(ref string, hash Hash) error { return repo.r.Storer.SetReference(plumbing.NewHashReference(plumbing.ReferenceName(ref), plumbing.NewHash(hash.String()))) } +// MergeRef merge other into ref and update the reference +// If the update is not fast-forward, the callback treeHashFn will be called for the caller to generate +// the Tree to store in the merge commit. +func (repo *GoGitRepo) MergeRef(ref string, otherRef string, treeHashFn func() Hash) error { + return nonNativeMerge(repo, ref, otherRef, treeHashFn) +} + // RemoveRef will remove a Git reference func (repo *GoGitRepo) RemoveRef(ref string) error { return repo.r.Storer.RemoveReference(plumbing.ReferenceName(ref)) @@ -657,38 +686,16 @@ func (repo *GoGitRepo) CopyRef(source string, dest string) error { // ListCommits will return the list of tree hashes of a ref, in chronological order func (repo *GoGitRepo) ListCommits(ref string) ([]Hash, error) { - r, err := repo.r.Reference(plumbing.ReferenceName(ref), false) - if err != nil { - return nil, err - } + return nonNativeListCommits(repo, ref) +} - commit, err := repo.r.CommitObject(r.Hash()) +func (repo *GoGitRepo) ReadCommit(hash Hash) (Commit, error) { + encoded, err := repo.r.Storer.EncodedObject(plumbing.CommitObject, plumbing.NewHash(hash.String())) if err != nil { - return nil, err - } - hashes := []Hash{Hash(commit.Hash.String())} - - for { - commit, err = commit.Parent(0) - if err == object.ErrParentNotFound { - break - } - if err != nil { - return nil, err - } - - if commit.NumParents() > 1 { - return nil, fmt.Errorf("multiple parents") - } - - hashes = append([]Hash{Hash(commit.Hash.String())}, hashes...) + return Commit{}, err } - return hashes, nil -} - -func (repo *GoGitRepo) ReadCommit(hash Hash) (Commit, error) { - commit, err := repo.r.CommitObject(plumbing.NewHash(hash.String())) + commit, err := object.DecodeCommit(repo.r.Storer, encoded) if err != nil { return Commit{}, err } @@ -698,12 +705,25 @@ func (repo *GoGitRepo) ReadCommit(hash Hash) (Commit, error) { parents[i] = Hash(parentHash.String()) } - return Commit{ + result := Commit{ Hash: hash, Parents: parents, TreeHash: Hash(commit.TreeHash.String()), - }, nil + } + + if commit.PGPSignature != "" { + result.SignedData, err = encoded.Reader() + if err != nil { + return Commit{}, err + } + result.Signature, err = deArmorSignature(strings.NewReader(commit.PGPSignature)) + if err != nil { + return Commit{}, err + } + } + + return result, nil } func (repo *GoGitRepo) AllClocks() (map[string]lamport.Clock, error) { -- cgit From dc5059bc3372941e2908739831188768335ac50b Mon Sep 17 00:00:00 2001 From: Michael Muré Date: Sun, 24 Jan 2021 19:45:21 +0100 Subject: entity: more progress on merging and signing --- repository/gogit.go | 7 ------- 1 file changed, 7 deletions(-) (limited to 'repository/gogit.go') diff --git a/repository/gogit.go b/repository/gogit.go index d6eb8621..fe434d88 100644 --- a/repository/gogit.go +++ b/repository/gogit.go @@ -630,13 +630,6 @@ func (repo *GoGitRepo) UpdateRef(ref string, hash Hash) error { return repo.r.Storer.SetReference(plumbing.NewHashReference(plumbing.ReferenceName(ref), plumbing.NewHash(hash.String()))) } -// MergeRef merge other into ref and update the reference -// If the update is not fast-forward, the callback treeHashFn will be called for the caller to generate -// the Tree to store in the merge commit. -func (repo *GoGitRepo) MergeRef(ref string, otherRef string, treeHashFn func() Hash) error { - return nonNativeMerge(repo, ref, otherRef, treeHashFn) -} - // RemoveRef will remove a Git reference func (repo *GoGitRepo) RemoveRef(ref string) error { return repo.r.Storer.RemoveReference(plumbing.ReferenceName(ref)) -- cgit From e35c7c4d170d1b682992c95f1c14772158501015 Mon Sep 17 00:00:00 2001 From: Michael Muré Date: Fri, 5 Feb 2021 11:18:38 +0100 Subject: entity: more testing and bug fixing --- repository/gogit.go | 46 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 7 deletions(-) (limited to 'repository/gogit.go') diff --git a/repository/gogit.go b/repository/gogit.go index fe434d88..c26fde9f 100644 --- a/repository/gogit.go +++ b/repository/gogit.go @@ -353,13 +353,17 @@ func (repo *GoGitRepo) ClearBleveIndex(name string) error { return nil } -// FetchRefs fetch git refs from a remote -func (repo *GoGitRepo) FetchRefs(remote string, refSpec string) (string, error) { +// FetchRefs fetch git refs matching a directory prefix to a remote +// Ex: prefix="foo" will fetch any remote refs matching "refs/foo/*" locally. +// The equivalent git refspec would be "refs/foo/*:refs/remotes//foo/*" +func (repo *GoGitRepo) FetchRefs(remote string, prefix string) (string, error) { + refspec := fmt.Sprintf("refs/%s/*:refs/remotes/%s/%s/*", prefix, remote, prefix) + buf := bytes.NewBuffer(nil) err := repo.r.Fetch(&gogit.FetchOptions{ RemoteName: remote, - RefSpecs: []config.RefSpec{config.RefSpec(refSpec)}, + RefSpecs: []config.RefSpec{config.RefSpec(refspec)}, Progress: buf, }) if err == gogit.NoErrAlreadyUpToDate { @@ -372,13 +376,41 @@ func (repo *GoGitRepo) FetchRefs(remote string, refSpec string) (string, error) return buf.String(), nil } -// PushRefs push git refs to a remote -func (repo *GoGitRepo) PushRefs(remote string, refSpec string) (string, error) { +// PushRefs push git refs matching a directory prefix to a remote +// Ex: prefix="foo" will push any local refs matching "refs/foo/*" to the remote. +// The equivalent git refspec would be "refs/foo/*:refs/foo/*" +// +// Additionally, PushRefs will update the local references in refs/remotes//foo to match +// the remote state. +func (repo *GoGitRepo) PushRefs(remote string, prefix string) (string, error) { + refspec := fmt.Sprintf("refs/%s/*:refs/%s/*", prefix, prefix) + + remo, err := repo.r.Remote(remote) + if err != nil { + return "", err + } + + // to make sure that the push also create the corresponding refs/remotes//... references, + // we need to have a default fetch refspec configured on the remote, to make our refs "track" the remote ones. + // This does not change the config on disk, only on memory. + hasCustomFetch := false + fetchRefspec := fmt.Sprintf("refs/%s/*:refs/remotes/%s/%s/*", prefix, remote, prefix) + for _, r := range remo.Config().Fetch { + if string(r) == fetchRefspec { + hasCustomFetch = true + break + } + } + + if !hasCustomFetch { + remo.Config().Fetch = append(remo.Config().Fetch, config.RefSpec(fetchRefspec)) + } + buf := bytes.NewBuffer(nil) - err := repo.r.Push(&gogit.PushOptions{ + err = remo.Push(&gogit.PushOptions{ RemoteName: remote, - RefSpecs: []config.RefSpec{config.RefSpec(refSpec)}, + RefSpecs: []config.RefSpec{config.RefSpec(refspec)}, Progress: buf, }) if err == gogit.NoErrAlreadyUpToDate { -- cgit From 2bdb1b60ff83de157f1a0d9ed42555d96b945fa6 Mon Sep 17 00:00:00 2001 From: Michael Muré Date: Tue, 9 Feb 2021 10:46:33 +0100 Subject: entity: working commit signatures --- repository/gogit.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'repository/gogit.go') diff --git a/repository/gogit.go b/repository/gogit.go index c26fde9f..73672933 100644 --- a/repository/gogit.go +++ b/repository/gogit.go @@ -715,12 +715,7 @@ func (repo *GoGitRepo) ListCommits(ref string) ([]Hash, error) { } func (repo *GoGitRepo) ReadCommit(hash Hash) (Commit, error) { - encoded, err := repo.r.Storer.EncodedObject(plumbing.CommitObject, plumbing.NewHash(hash.String())) - if err != nil { - return Commit{}, err - } - - commit, err := object.DecodeCommit(repo.r.Storer, encoded) + commit, err := repo.r.CommitObject(plumbing.NewHash(hash.String())) if err != nil { return Commit{}, err } @@ -737,6 +732,15 @@ func (repo *GoGitRepo) ReadCommit(hash Hash) (Commit, error) { } if commit.PGPSignature != "" { + // I can't find a way to just remove the signature when reading the encoded commit so we need to + // re-encode the commit without signature. + + encoded := &plumbing.MemoryObject{} + err := commit.EncodeWithoutSignature(encoded) + if err != nil { + return Commit{}, err + } + result.SignedData, err = encoded.Reader() if err != nil { return Commit{}, err -- cgit From 9434d2ea5c6da5e856d0bbb02046a5886dfaa600 Mon Sep 17 00:00:00 2001 From: Michael Muré Date: Sun, 21 Mar 2021 22:37:19 +0100 Subject: repo: fix security issue that could lead to arbitrary code execution see https://blog.golang.org/path-security for details --- repository/gogit.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'repository/gogit.go') diff --git a/repository/gogit.go b/repository/gogit.go index bdac259d..f2d2b57e 100644 --- a/repository/gogit.go +++ b/repository/gogit.go @@ -5,7 +5,6 @@ import ( "fmt" "io/ioutil" "os" - "os/exec" "path/filepath" "sort" "strings" @@ -20,6 +19,7 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/filemode" "github.com/go-git/go-git/v5/plumbing/object" + "golang.org/x/sys/execabs" "github.com/MichaelMure/git-bug/util/lamport" ) @@ -261,7 +261,7 @@ func (repo *GoGitRepo) GetCoreEditor() (string, error) { } for _, cmd := range priorities { - if _, err = exec.LookPath(cmd); err == nil { + if _, err = execabs.LookPath(cmd); err == nil { return cmd, nil } -- cgit From 554992523574684ecce36d38bf5310bff52c8c03 Mon Sep 17 00:00:00 2001 From: Michael Muré Date: Sun, 4 Apr 2021 13:28:21 +0200 Subject: cache: many fixes following the dag entity migration --- repository/gogit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'repository/gogit.go') diff --git a/repository/gogit.go b/repository/gogit.go index 248c34d5..20454bd7 100644 --- a/repository/gogit.go +++ b/repository/gogit.go @@ -335,7 +335,7 @@ func (repo *GoGitRepo) ClearBleveIndex(name string) error { repo.indexesMutex.Lock() defer repo.indexesMutex.Unlock() - path := filepath.Join(repo.path, "indexes", name) + path := filepath.Join(repo.path, "git-bug", "indexes", name) err := os.RemoveAll(path) if err != nil { -- cgit