From 28adf41af6a5c0f4b19875912d056b03ea6710f8 Mon Sep 17 00:00:00 2001 From: Mark Date: Tue, 1 Dec 2020 16:34:02 +0000 Subject: Move new RepoStorage interface to GoGitRepo, revert repoTest --- repository/gogit.go | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'repository/gogit.go') diff --git a/repository/gogit.go b/repository/gogit.go index c59409d4..45906ea4 100644 --- a/repository/gogit.go +++ b/repository/gogit.go @@ -13,6 +13,7 @@ import ( "sync" "time" + "github.com/go-git/go-billy" gogit "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" @@ -32,6 +33,12 @@ type GoGitRepo struct { clocks map[string]lamport.Clock keyring Keyring + RepoStorage +} + +type RepoStorage interface { + // Storage returns a billy.Filesystem giving access to $RepoPath/.git/git-bug + Storage() billy.Filesystem } func NewGoGitRepo(path string, clockLoaders []ClockLoader) (*GoGitRepo, error) { -- cgit From bca9ae82745ffd619fd321f4200016c184849f94 Mon Sep 17 00:00:00 2001 From: Michael Muré Date: Tue, 1 Dec 2020 21:19:23 +0100 Subject: repo: more work towards RepoStorage --- repository/gogit.go | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) (limited to 'repository/gogit.go') diff --git a/repository/gogit.go b/repository/gogit.go index 45906ea4..874885db 100644 --- a/repository/gogit.go +++ b/repository/gogit.go @@ -13,7 +13,8 @@ import ( "sync" "time" - "github.com/go-git/go-billy" + "github.com/go-git/go-billy/v5" + "github.com/go-git/go-billy/v5/osfs" gogit "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" @@ -24,6 +25,7 @@ import ( ) var _ ClockedRepo = &GoGitRepo{} +var _ TestedRepo = &GoGitRepo{} type GoGitRepo struct { r *gogit.Repository @@ -33,12 +35,6 @@ type GoGitRepo struct { clocks map[string]lamport.Clock keyring Keyring - RepoStorage -} - -type RepoStorage interface { - // Storage returns a billy.Filesystem giving access to $RepoPath/.git/git-bug - Storage() billy.Filesystem } func NewGoGitRepo(path string, clockLoaders []ClockLoader) (*GoGitRepo, error) { @@ -202,11 +198,6 @@ func (repo *GoGitRepo) Keyring() Keyring { return repo.keyring } -// GetPath returns the path to the repo. -func (repo *GoGitRepo) GetPath() string { - return repo.path -} - // GetUserName returns the name the the user has used to configure git func (repo *GoGitRepo) GetUserName() (string, error) { return repo.AnyConfig().ReadString("user.name") @@ -277,6 +268,11 @@ func (repo *GoGitRepo) GetRemotes() (map[string]string, error) { return result, nil } +// LocalStorage return a billy.Filesystem giving access to $RepoPath/.git/git-bug +func (repo *GoGitRepo) LocalStorage() billy.Filesystem { + return osfs.New(repo.path) +} + // FetchRefs fetch git refs from a remote func (repo *GoGitRepo) FetchRefs(remote string, refSpec string) (string, error) { buf := bytes.NewBuffer(nil) @@ -660,3 +656,16 @@ func (repo *GoGitRepo) AddRemote(name string, url string) error { return err } + +// GetLocalRemote return the URL to use to add this repo as a local remote +func (repo *GoGitRepo) GetLocalRemote() string { + return repo.path +} + +// EraseFromDisk delete this repository entirely from the disk +func (repo *GoGitRepo) EraseFromDisk() error { + path := filepath.Clean(strings.TrimSuffix(repo.path, string(filepath.Separator)+".git")) + + // fmt.Println("Cleaning repo:", path) + return os.RemoveAll(path) +} -- cgit From 4ef2c1104079032336da9f2a7879f2432c2609ce Mon Sep 17 00:00:00 2001 From: Michael Muré Date: Sat, 5 Dec 2020 03:08:54 +0100 Subject: repo: finish RepoStorage move --- repository/gogit.go | 152 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 89 insertions(+), 63 deletions(-) (limited to 'repository/gogit.go') diff --git a/repository/gogit.go b/repository/gogit.go index 874885db..65c2ab54 100644 --- a/repository/gogit.go +++ b/repository/gogit.go @@ -6,20 +6,22 @@ import ( "io/ioutil" "os" "os/exec" - stdpath "path" "path/filepath" "sort" "strings" "sync" "time" + "github.com/99designs/keyring" "github.com/go-git/go-billy/v5" + "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-billy/v5/osfs" gogit "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/filemode" "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/storage/memory" "github.com/MichaelMure/git-bug/util/lamport" ) @@ -28,16 +30,18 @@ var _ ClockedRepo = &GoGitRepo{} var _ TestedRepo = &GoGitRepo{} type GoGitRepo struct { - r *gogit.Repository - path string + r *gogit.Repository + path string + isMemory bool clocksMutex sync.Mutex clocks map[string]lamport.Clock - keyring Keyring + keyring Keyring + localStorage billy.Filesystem } -func NewGoGitRepo(path string, clockLoaders []ClockLoader) (*GoGitRepo, error) { +func OpenGoGitRepo(path string, clockLoaders []ClockLoader) (*GoGitRepo, error) { path, err := detectGitPath(path) if err != nil { return nil, err @@ -54,10 +58,12 @@ func NewGoGitRepo(path string, clockLoaders []ClockLoader) (*GoGitRepo, error) { } repo := &GoGitRepo{ - r: r, - path: path, - clocks: make(map[string]lamport.Clock), - keyring: k, + r: r, + path: path, + isMemory: false, + clocks: make(map[string]lamport.Clock), + keyring: k, + localStorage: osfs.New(filepath.Join(path, "git-bug")), } for _, loader := range clockLoaders { @@ -79,6 +85,69 @@ func NewGoGitRepo(path string, clockLoaders []ClockLoader) (*GoGitRepo, error) { return repo, nil } +// InitGoGitRepo create a new empty git repo at the given path +func InitGoGitRepo(path string) (*GoGitRepo, error) { + r, err := gogit.PlainInit(path, false) + if err != nil { + return nil, err + } + + k, err := defaultKeyring() + if err != nil { + return nil, err + } + + return &GoGitRepo{ + r: r, + path: filepath.Join(path, ".git"), + isMemory: false, + clocks: make(map[string]lamport.Clock), + keyring: k, + localStorage: osfs.New(filepath.Join(path, ".git", "git-bug")), + }, nil +} + +// InitBareGoGitRepo create a new --bare empty git repo at the given path +func InitBareGoGitRepo(path string) (*GoGitRepo, error) { + r, err := gogit.PlainInit(path, true) + if err != nil { + return nil, err + } + + k, err := defaultKeyring() + if err != nil { + return nil, err + } + + return &GoGitRepo{ + r: r, + path: path, + isMemory: false, + clocks: make(map[string]lamport.Clock), + keyring: k, + localStorage: osfs.New(filepath.Join(path, "git-bug")), + }, nil +} + +func InitMemoryGoGitRepo() (*GoGitRepo, error) { + r, err := gogit.Init(memory.NewStorage(), nil) + if err != nil { + return nil, err + } + + k := keyring.NewArrayKeyring(nil) + + repo := &GoGitRepo{ + r: r, + isMemory: true, + clocks: make(map[string]lamport.Clock), + keyring: k, + localStorage: memfs.New(), + } + + return repo, nil +} + func detectGitPath(path string) (string, error) { // normalize the path path, err := filepath.Abs(path) @@ -87,12 +156,12 @@ func detectGitPath(path string) (string, error) { } for { - fi, err := os.Stat(stdpath.Join(path, ".git")) + fi, err := os.Stat(filepath.Join(path, ".git")) if err == nil { if !fi.IsDir() { return "", fmt.Errorf(".git exist but is not a directory") } - return stdpath.Join(path, ".git"), nil + return filepath.Join(path, ".git"), nil } if !os.IsNotExist(err) { // unknown error @@ -120,7 +189,7 @@ func isGitDir(path string) (bool, error) { markers := []string{"HEAD", "objects", "refs"} for _, marker := range markers { - _, err := os.Stat(stdpath.Join(path, marker)) + _, err := os.Stat(filepath.Join(path, marker)) if err == nil { continue } @@ -135,46 +204,6 @@ func isGitDir(path string) (bool, error) { return true, nil } -// InitGoGitRepo create a new empty git repo at the given path -func InitGoGitRepo(path string) (*GoGitRepo, error) { - r, err := gogit.PlainInit(path, false) - if err != nil { - return nil, err - } - - k, err := defaultKeyring() - if err != nil { - return nil, err - } - - return &GoGitRepo{ - r: r, - path: path + "/.git", - clocks: make(map[string]lamport.Clock), - keyring: k, - }, nil -} - -// InitBareGoGitRepo create a new --bare empty git repo at the given path -func InitBareGoGitRepo(path string) (*GoGitRepo, error) { - r, err := gogit.PlainInit(path, true) - if err != nil { - return nil, err - } - - k, err := defaultKeyring() - if err != nil { - return nil, err - } - - return &GoGitRepo{ - r: r, - path: path, - clocks: make(map[string]lamport.Clock), - keyring: k, - }, nil -} - // LocalConfig give access to the repository scoped configuration func (repo *GoGitRepo) LocalConfig() Config { return newGoGitLocalConfig(repo.r) @@ -182,10 +211,7 @@ func (repo *GoGitRepo) LocalConfig() Config { // GlobalConfig give access to the global scoped configuration func (repo *GoGitRepo) GlobalConfig() Config { - // TODO: replace that with go-git native implementation once it's supported - // see: https://github.com/go-git/go-git - // see: https://github.com/src-d/go-git/issues/760 - return newGoGitGlobalConfig(repo.r) + return newGoGitGlobalConfig() } // AnyConfig give access to a merged local/global configuration @@ -270,7 +296,7 @@ func (repo *GoGitRepo) GetRemotes() (map[string]string, error) { // LocalStorage return a billy.Filesystem giving access to $RepoPath/.git/git-bug func (repo *GoGitRepo) LocalStorage() billy.Filesystem { - return osfs.New(repo.path) + return repo.localStorage } // FetchRefs fetch git refs from a remote @@ -614,9 +640,7 @@ func (repo *GoGitRepo) GetOrCreateClock(name string) (lamport.Clock, error) { repo.clocksMutex.Lock() defer repo.clocksMutex.Unlock() - p := stdpath.Join(repo.path, clockPath, name+"-clock") - - c, err = lamport.NewPersistedClock(p) + c, err = lamport.NewPersistedClock(repo.localStorage, name+"-clock") if err != nil { return nil, err } @@ -633,9 +657,7 @@ func (repo *GoGitRepo) getClock(name string) (lamport.Clock, error) { return c, nil } - p := stdpath.Join(repo.path, clockPath, name+"-clock") - - c, err := lamport.LoadPersistedClock(p) + c, err := lamport.LoadPersistedClock(repo.localStorage, name+"-clock") if err == nil { repo.clocks[name] = c return c, nil @@ -664,6 +686,10 @@ func (repo *GoGitRepo) GetLocalRemote() string { // EraseFromDisk delete this repository entirely from the disk func (repo *GoGitRepo) EraseFromDisk() error { + if repo.isMemory { + return nil + } + path := filepath.Clean(strings.TrimSuffix(repo.path, string(filepath.Separator)+".git")) // fmt.Println("Cleaning repo:", path) -- cgit From be6e653f1566f5812e7c4442927eee02963dfb68 Mon Sep 17 00:00:00 2001 From: Michael Muré Date: Sat, 5 Dec 2020 21:40:48 +0100 Subject: repo: remove the memory-only repo for now --- repository/gogit.go | 35 +++-------------------------------- 1 file changed, 3 insertions(+), 32 deletions(-) (limited to 'repository/gogit.go') diff --git a/repository/gogit.go b/repository/gogit.go index 65c2ab54..0d34c00a 100644 --- a/repository/gogit.go +++ b/repository/gogit.go @@ -12,16 +12,13 @@ import ( "sync" "time" - "github.com/99designs/keyring" "github.com/go-git/go-billy/v5" - "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-billy/v5/osfs" gogit "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/filemode" "github.com/go-git/go-git/v5/plumbing/object" - "github.com/go-git/go-git/v5/storage/memory" "github.com/MichaelMure/git-bug/util/lamport" ) @@ -30,9 +27,8 @@ var _ ClockedRepo = &GoGitRepo{} var _ TestedRepo = &GoGitRepo{} type GoGitRepo struct { - r *gogit.Repository - path string - isMemory bool + r *gogit.Repository + path string clocksMutex sync.Mutex clocks map[string]lamport.Clock @@ -41,6 +37,7 @@ type GoGitRepo struct { localStorage billy.Filesystem } +// OpenGoGitRepo open an already existing repo at the given path func OpenGoGitRepo(path string, clockLoaders []ClockLoader) (*GoGitRepo, error) { path, err := detectGitPath(path) if err != nil { @@ -60,7 +57,6 @@ func OpenGoGitRepo(path string, clockLoaders []ClockLoader) (*GoGitRepo, error) repo := &GoGitRepo{ r: r, path: path, - isMemory: false, clocks: make(map[string]lamport.Clock), keyring: k, localStorage: osfs.New(filepath.Join(path, "git-bug")), @@ -100,7 +96,6 @@ func InitGoGitRepo(path string) (*GoGitRepo, error) { return &GoGitRepo{ r: r, path: filepath.Join(path, ".git"), - isMemory: false, clocks: make(map[string]lamport.Clock), keyring: k, localStorage: osfs.New(filepath.Join(path, ".git", "git-bug")), @@ -122,32 +117,12 @@ func InitBareGoGitRepo(path string) (*GoGitRepo, error) { return &GoGitRepo{ r: r, path: path, - isMemory: false, clocks: make(map[string]lamport.Clock), keyring: k, localStorage: osfs.New(filepath.Join(path, "git-bug")), }, nil } -func InitMemoryGoGitRepo() (*GoGitRepo, error) { - r, err := gogit.Init(memory.NewStorage(), nil) - if err != nil { - return nil, err - } - - k := keyring.NewArrayKeyring(nil) - - repo := &GoGitRepo{ - r: r, - isMemory: true, - clocks: make(map[string]lamport.Clock), - keyring: k, - localStorage: memfs.New(), - } - - return repo, nil -} - func detectGitPath(path string) (string, error) { // normalize the path path, err := filepath.Abs(path) @@ -686,10 +661,6 @@ func (repo *GoGitRepo) GetLocalRemote() string { // EraseFromDisk delete this repository entirely from the disk func (repo *GoGitRepo) EraseFromDisk() error { - if repo.isMemory { - return nil - } - path := filepath.Clean(strings.TrimSuffix(repo.path, string(filepath.Separator)+".git")) // fmt.Println("Cleaning repo:", path) -- cgit From 71e1303234ef227851998f08c98c7c69670c9966 Mon Sep 17 00:00:00 2001 From: Michael Muré Date: Tue, 8 Dec 2020 13:15:21 +0100 Subject: repo: simpler clock mutex locking --- repository/gogit.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'repository/gogit.go') diff --git a/repository/gogit.go b/repository/gogit.go index 0d34c00a..741982aa 100644 --- a/repository/gogit.go +++ b/repository/gogit.go @@ -604,6 +604,9 @@ func (repo *GoGitRepo) ListCommits(ref string) ([]Hash, error) { // GetOrCreateClock return a Lamport clock stored in the Repo. // If the clock doesn't exist, it's created. func (repo *GoGitRepo) GetOrCreateClock(name string) (lamport.Clock, error) { + repo.clocksMutex.Lock() + defer repo.clocksMutex.Unlock() + c, err := repo.getClock(name) if err == nil { return c, nil @@ -612,9 +615,6 @@ func (repo *GoGitRepo) GetOrCreateClock(name string) (lamport.Clock, error) { return nil, err } - repo.clocksMutex.Lock() - defer repo.clocksMutex.Unlock() - c, err = lamport.NewPersistedClock(repo.localStorage, name+"-clock") if err != nil { return nil, err @@ -625,9 +625,6 @@ func (repo *GoGitRepo) GetOrCreateClock(name string) (lamport.Clock, error) { } func (repo *GoGitRepo) getClock(name string) (lamport.Clock, error) { - repo.clocksMutex.Lock() - defer repo.clocksMutex.Unlock() - if c, ok := repo.clocks[name]; ok { return c, nil } -- cgit From c884d557bf5e0ebdbe6e3b20535af24c2e97d29f Mon Sep 17 00:00:00 2001 From: Michael Muré Date: Tue, 8 Dec 2020 14:42:13 +0100 Subject: repo: move bleve there --- repository/gogit.go | 76 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) (limited to 'repository/gogit.go') diff --git a/repository/gogit.go b/repository/gogit.go index 741982aa..74fe3fc5 100644 --- a/repository/gogit.go +++ b/repository/gogit.go @@ -12,6 +12,7 @@ import ( "sync" "time" + "github.com/blevesearch/bleve" "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/osfs" gogit "github.com/go-git/go-git/v5" @@ -33,6 +34,9 @@ type GoGitRepo struct { clocksMutex sync.Mutex clocks map[string]lamport.Clock + indexesMutex sync.Mutex + indexes map[string]bleve.Index + keyring Keyring localStorage billy.Filesystem } @@ -58,6 +62,7 @@ func OpenGoGitRepo(path string, clockLoaders []ClockLoader) (*GoGitRepo, error) r: r, path: path, clocks: make(map[string]lamport.Clock), + indexes: make(map[string]bleve.Index), keyring: k, localStorage: osfs.New(filepath.Join(path, "git-bug")), } @@ -97,6 +102,7 @@ func InitGoGitRepo(path string) (*GoGitRepo, error) { r: r, path: filepath.Join(path, ".git"), clocks: make(map[string]lamport.Clock), + indexes: make(map[string]bleve.Index), keyring: k, localStorage: osfs.New(filepath.Join(path, ".git", "git-bug")), }, nil @@ -118,6 +124,7 @@ func InitBareGoGitRepo(path string) (*GoGitRepo, error) { r: r, path: path, clocks: make(map[string]lamport.Clock), + indexes: make(map[string]bleve.Index), keyring: k, localStorage: osfs.New(filepath.Join(path, "git-bug")), }, nil @@ -179,6 +186,17 @@ func isGitDir(path string) (bool, error) { return true, nil } +func (repo *GoGitRepo) Close() error { + var firstErr error + for _, index := range repo.indexes { + err := index.Close() + if err != nil && firstErr == nil { + firstErr = err + } + } + return firstErr +} + // LocalConfig give access to the repository scoped configuration func (repo *GoGitRepo) LocalConfig() Config { return newGoGitLocalConfig(repo.r) @@ -274,6 +292,64 @@ func (repo *GoGitRepo) LocalStorage() billy.Filesystem { return repo.localStorage } +// GetBleveIndex return a bleve.Index that can be used to index documents +func (repo *GoGitRepo) GetBleveIndex(name string) (bleve.Index, error) { + repo.indexesMutex.Lock() + defer repo.indexesMutex.Unlock() + + if index, ok := repo.indexes[name]; ok { + return index, nil + } + + path := filepath.Join(repo.path, "git-bug", "indexes", name) + + index, err := bleve.Open(path) + if err == nil { + repo.indexes[name] = index + return index, nil + } + + err = os.MkdirAll(path, os.ModePerm) + if err != nil { + return nil, err + } + + mapping := bleve.NewIndexMapping() + mapping.DefaultAnalyzer = "en" + + index, err = bleve.New(path, mapping) + if err != nil { + return nil, err + } + + repo.indexes[name] = index + + return index, nil +} + +// ClearBleveIndex will wipe the given index +func (repo *GoGitRepo) ClearBleveIndex(name string) error { + repo.indexesMutex.Lock() + defer repo.indexesMutex.Unlock() + + path := filepath.Join(repo.path, "indexes", name) + + err := os.RemoveAll(path) + if err != nil { + return err + } + + if index, ok := repo.indexes[name]; ok { + err = index.Close() + if err != nil { + return err + } + delete(repo.indexes, name) + } + + return nil +} + // FetchRefs fetch git refs from a remote func (repo *GoGitRepo) FetchRefs(remote string, refSpec string) (string, error) { buf := bytes.NewBuffer(nil) -- cgit From 8128bb79b0db9023a98c356e4e173d846057c577 Mon Sep 17 00:00:00 2001 From: Michael Muré Date: Tue, 8 Dec 2020 15:09:58 +0100 Subject: repo: close before deleting --- repository/gogit.go | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'repository/gogit.go') diff --git a/repository/gogit.go b/repository/gogit.go index 74fe3fc5..bdac259d 100644 --- a/repository/gogit.go +++ b/repository/gogit.go @@ -734,6 +734,11 @@ func (repo *GoGitRepo) GetLocalRemote() string { // EraseFromDisk delete this repository entirely from the disk func (repo *GoGitRepo) EraseFromDisk() error { + err := repo.Close() + if err != nil { + return err + } + path := filepath.Clean(strings.TrimSuffix(repo.path, string(filepath.Separator)+".git")) // fmt.Println("Cleaning repo:", path) -- cgit