From 1e1a7d0623459807d6f1e871492147f971f7540c Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Thu, 30 Aug 2018 15:29:51 +0200 Subject: git: add Static option to PlainOpen Also adds Static configuration to Storage and DotGit. This option means that the git repository is not expected to be modified while open and enables some optimizations. Each time a file is accessed the storer tries to open an object file for the requested hash. When this is done for a lot of objects it is expensive. With Static option a list of object files is generated the first time an object is accessed and used to check if exists instead of using system calls. A similar optimization is done for packfiles. Signed-off-by: Javi Fontan --- storage/filesystem/storage.go | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) (limited to 'storage/filesystem/storage.go') diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go index 622bb4a..a969a1f 100644 --- a/storage/filesystem/storage.go +++ b/storage/filesystem/storage.go @@ -11,6 +11,8 @@ import ( // standard git format (this is, the .git directory). Zero values of this type // are not safe to use, see the NewStorage function below. type Storage struct { + StorageOptions + fs billy.Filesystem dir *dotgit.DotGit @@ -22,17 +24,36 @@ type Storage struct { ModuleStorage } +// StorageOptions holds configuration for the storage. +type StorageOptions struct { + // Static means that the filesystem is not modified while the repo is open. + Static bool +} + // NewStorage returns a new Storage backed by a given `fs.Filesystem` func NewStorage(fs billy.Filesystem) (*Storage, error) { - dir := dotgit.New(fs) + return NewStorageWithOptions(fs, StorageOptions{}) +} + +// NewStorageWithOptions returns a new Storage backed by a given `fs.Filesystem` +func NewStorageWithOptions( + fs billy.Filesystem, + ops StorageOptions, +) (*Storage, error) { + dOps := dotgit.DotGitOptions{ + Static: ops.Static, + } + + dir := dotgit.NewWithOptions(fs, dOps) o, err := NewObjectStorage(dir) if err != nil { return nil, err } return &Storage{ - fs: fs, - dir: dir, + StorageOptions: ops, + fs: fs, + dir: dir, ObjectStorage: o, ReferenceStorage: ReferenceStorage{dir: dir}, -- cgit From 82945e31dd8bce5fc51d4fd16d696a6d326e5f44 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Thu, 30 Aug 2018 18:33:37 +0200 Subject: git, storer: use a common storer.Options for storer and PlainOpen Signed-off-by: Javi Fontan --- storage/filesystem/storage.go | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) (limited to 'storage/filesystem/storage.go') diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go index a969a1f..24e6454 100644 --- a/storage/filesystem/storage.go +++ b/storage/filesystem/storage.go @@ -2,6 +2,7 @@ package filesystem import ( + "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" "gopkg.in/src-d/go-billy.v4" @@ -11,7 +12,7 @@ import ( // standard git format (this is, the .git directory). Zero values of this type // are not safe to use, see the NewStorage function below. type Storage struct { - StorageOptions + storer.Options fs billy.Filesystem dir *dotgit.DotGit @@ -24,36 +25,26 @@ type Storage struct { ModuleStorage } -// StorageOptions holds configuration for the storage. -type StorageOptions struct { - // Static means that the filesystem is not modified while the repo is open. - Static bool -} - // NewStorage returns a new Storage backed by a given `fs.Filesystem` func NewStorage(fs billy.Filesystem) (*Storage, error) { - return NewStorageWithOptions(fs, StorageOptions{}) + return NewStorageWithOptions(fs, storer.Options{}) } // NewStorageWithOptions returns a new Storage backed by a given `fs.Filesystem` func NewStorageWithOptions( fs billy.Filesystem, - ops StorageOptions, + ops storer.Options, ) (*Storage, error) { - dOps := dotgit.DotGitOptions{ - Static: ops.Static, - } - - dir := dotgit.NewWithOptions(fs, dOps) + dir := dotgit.NewWithOptions(fs, ops) o, err := NewObjectStorage(dir) if err != nil { return nil, err } return &Storage{ - StorageOptions: ops, - fs: fs, - dir: dir, + Options: ops, + fs: fs, + dir: dir, ObjectStorage: o, ReferenceStorage: ReferenceStorage{dir: dir}, -- cgit From 95acbf6c3958b7540a8549aa049051325fcecd8b Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Mon, 3 Sep 2018 11:17:22 +0200 Subject: storage/filesystem: make Storage options private Signed-off-by: Javi Fontan --- storage/filesystem/storage.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'storage/filesystem/storage.go') diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go index 24e6454..d2c5287 100644 --- a/storage/filesystem/storage.go +++ b/storage/filesystem/storage.go @@ -12,7 +12,7 @@ import ( // standard git format (this is, the .git directory). Zero values of this type // are not safe to use, see the NewStorage function below. type Storage struct { - storer.Options + options storer.Options fs billy.Filesystem dir *dotgit.DotGit @@ -42,7 +42,7 @@ func NewStorageWithOptions( } return &Storage{ - Options: ops, + options: ops, fs: fs, dir: dir, -- cgit From 874f669becc25489081306bbbcbbc27b970f6295 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Mon, 3 Sep 2018 19:40:22 +0200 Subject: storage/filesystem: move Options to filesytem and dotgit Signed-off-by: Javi Fontan --- storage/filesystem/storage.go | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) (limited to 'storage/filesystem/storage.go') diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go index d2c5287..25b3653 100644 --- a/storage/filesystem/storage.go +++ b/storage/filesystem/storage.go @@ -2,7 +2,6 @@ package filesystem import ( - "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" "gopkg.in/src-d/go-billy.v4" @@ -12,8 +11,6 @@ import ( // standard git format (this is, the .git directory). Zero values of this type // are not safe to use, see the NewStorage function below. type Storage struct { - options storer.Options - fs billy.Filesystem dir *dotgit.DotGit @@ -25,26 +22,36 @@ type Storage struct { ModuleStorage } +// Options holds configuration for the storage. +type Options struct { + // ExclusiveAccess means that the filesystem is not modified externally + // while the repo is open. + ExclusiveAccess bool +} + // NewStorage returns a new Storage backed by a given `fs.Filesystem` func NewStorage(fs billy.Filesystem) (*Storage, error) { - return NewStorageWithOptions(fs, storer.Options{}) + return NewStorageWithOptions(fs, Options{}) } // NewStorageWithOptions returns a new Storage backed by a given `fs.Filesystem` func NewStorageWithOptions( fs billy.Filesystem, - ops storer.Options, + ops Options, ) (*Storage, error) { - dir := dotgit.NewWithOptions(fs, ops) - o, err := NewObjectStorage(dir) + dirOps := dotgit.Options{ + ExclusiveAccess: ops.ExclusiveAccess, + } + + dir := dotgit.NewWithOptions(fs, dirOps) + o, err := NewObjectStorageWithOptions(dir, ops) if err != nil { return nil, err } return &Storage{ - options: ops, - fs: fs, - dir: dir, + fs: fs, + dir: dir, ObjectStorage: o, ReferenceStorage: ReferenceStorage{dir: dir}, -- cgit From 6384ab93a2dbac9045ee19099455cbcfe82d0201 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Thu, 30 Aug 2018 20:28:40 +0200 Subject: storage/dotgit: add KeepDescriptors option This option maintains packfile file descriptors opened after reading objects from them. It improves performance as it does not have to be opening packfiles each time an object is needed. Also adds Close to EncodedObjectStorer to close all the files manualy. Signed-off-by: Javi Fontan --- storage/filesystem/storage.go | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'storage/filesystem/storage.go') diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go index 25b3653..7fae789 100644 --- a/storage/filesystem/storage.go +++ b/storage/filesystem/storage.go @@ -27,6 +27,9 @@ type Options struct { // ExclusiveAccess means that the filesystem is not modified externally // while the repo is open. ExclusiveAccess bool + // KeepDescriptors makes the file descriptors to be reused but they will + // need to be manually closed calling Close(). + KeepDescriptors bool } // NewStorage returns a new Storage backed by a given `fs.Filesystem` @@ -41,6 +44,7 @@ func NewStorageWithOptions( ) (*Storage, error) { dirOps := dotgit.Options{ ExclusiveAccess: ops.ExclusiveAccess, + KeepDescriptors: ops.KeepDescriptors, } dir := dotgit.NewWithOptions(fs, dirOps) -- cgit From 8f6b3127c1ff7661113fff2662416c328971a285 Mon Sep 17 00:00:00 2001 From: kuba-- Date: Fri, 7 Sep 2018 09:27:35 +0200 Subject: Expose Storage cache. Signed-off-by: kuba-- --- storage/filesystem/storage.go | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) (limited to 'storage/filesystem/storage.go') diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go index 7fae789..14a772a 100644 --- a/storage/filesystem/storage.go +++ b/storage/filesystem/storage.go @@ -2,6 +2,7 @@ package filesystem import ( + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" "gopkg.in/src-d/go-billy.v4" @@ -32,38 +33,35 @@ type Options struct { KeepDescriptors bool } -// NewStorage returns a new Storage backed by a given `fs.Filesystem` -func NewStorage(fs billy.Filesystem) (*Storage, error) { - return NewStorageWithOptions(fs, Options{}) +// NewStorage returns a new Storage backed by a given `fs.Filesystem` and cache. +func NewStorage(fs billy.Filesystem, cache cache.Object) *Storage { + return NewStorageWithOptions(fs, cache, Options{}) } -// NewStorageWithOptions returns a new Storage backed by a given `fs.Filesystem` -func NewStorageWithOptions( - fs billy.Filesystem, - ops Options, -) (*Storage, error) { +// NewStorageWithOptions returns a new Storage with extra options, +// backed by a given `fs.Filesystem` and cache. +func NewStorageWithOptions(fs billy.Filesystem, cache cache.Object, ops Options) *Storage { dirOps := dotgit.Options{ ExclusiveAccess: ops.ExclusiveAccess, KeepDescriptors: ops.KeepDescriptors, } - dir := dotgit.NewWithOptions(fs, dirOps) - o, err := NewObjectStorageWithOptions(dir, ops) - if err != nil { - return nil, err - } return &Storage{ fs: fs, dir: dir, - ObjectStorage: o, + ObjectStorage: ObjectStorage{ + options: ops, + deltaBaseCache: cache, + dir: dir, + }, ReferenceStorage: ReferenceStorage{dir: dir}, IndexStorage: IndexStorage{dir: dir}, ShallowStorage: ShallowStorage{dir: dir}, ConfigStorage: ConfigStorage{dir: dir}, ModuleStorage: ModuleStorage{dir: dir}, - }, nil + } } // Filesystem returns the underlying filesystem @@ -71,6 +69,7 @@ func (s *Storage) Filesystem() billy.Filesystem { return s.fs } +// Init initializes .git directory func (s *Storage) Init() error { return s.dir.Initialize() } -- cgit