8 files changed, 165 insertions, 22 deletions

diff --git a/plumbing/format/packfile/packfile.go b/plumbing/format/packfile/packfile.go
index 1e7ef26..69b6e85 100644
--- a/plumbing/format/packfile/packfile.go
+++ b/plumbing/format/packfile/packfile.go
@@ -107,7 +107,7 @@ func (p *Packfile) GetSizeByOffset(o int64) (size int64, err error) {
 	if err != nil {
 		return 0, err
 	}
-	return h.Length, nil
+	return p.getObjectSize(h)
 }
 
 func (p *Packfile) objectHeaderAtOffset(offset int64) (*ObjectHeader, error) {
diff --git a/plumbing/format/packfile/packfile_test.go b/plumbing/format/packfile/packfile_test.go
index 05dc8a7..455fe65 100644
--- a/plumbing/format/packfile/packfile_test.go
+++ b/plumbing/format/packfile/packfile_test.go
@@ -277,3 +277,29 @@ func getIndexFromIdxFile(r io.Reader) idxfile.Index {
 
 	return idxf
 }
+
+func (s *PackfileSuite) TestSize(c *C) {
+	f := fixtures.Basic().ByTag("ref-delta").One()
+
+	index := getIndexFromIdxFile(f.Idx())
+	fs := osfs.New("")
+	pf, err := fs.Open(f.Packfile().Name())
+	c.Assert(err, IsNil)
+
+	packfile := packfile.NewPackfile(index, fs, pf)
+	defer packfile.Close()
+
+	// Get the size of binary.jpg, which is not delta-encoded.
+	offset, err := packfile.FindOffset(plumbing.NewHash("d5c0f4ab811897cadf03aec358ae60d21f91c50d"))
+	c.Assert(err, IsNil)
+	size, err := packfile.GetSizeByOffset(offset)
+	c.Assert(err, IsNil)
+	c.Assert(size, Equals, int64(76110))
+
+	// Get the size of the root commit, which is delta-encoded.
+	offset, err = packfile.FindOffset(f.Head)
+	c.Assert(err, IsNil)
+	size, err = packfile.GetSizeByOffset(offset)
+	c.Assert(err, IsNil)
+	c.Assert(size, Equals, int64(245))
+}
diff --git a/plumbing/revlist/revlist.go b/plumbing/revlist/revlist.go
index 0a9d1e8..7ad71ac 100644
--- a/plumbing/revlist/revlist.go
+++ b/plumbing/revlist/revlist.go
@@ -21,7 +21,20 @@ func Objects(
 	objs,
 	ignore []plumbing.Hash,
 ) ([]plumbing.Hash, error) {
-	ignore, err := objects(s, ignore, nil, true)
+	return ObjectsWithStorageForIgnores(s, s, objs, ignore)
+}
+
+// ObjectsWithStorageForIgnores is the same as Objects, but a
+// secondary storage layer can be provided, to be used to finding the
+// full set of objects to be ignored while finding the reachable
+// objects.  This is useful when the main `s` storage layer is slow
+// and/or remote, while the ignore list is available somewhere local.
+func ObjectsWithStorageForIgnores(
+	s, ignoreStore storer.EncodedObjectStorer,
+	objs,
+	ignore []plumbing.Hash,
+) ([]plumbing.Hash, error) {
+	ignore, err := objects(ignoreStore, ignore, nil, true)
 	if err != nil {
 		return nil, err
 	}
@@ -114,7 +127,6 @@ func reachableObjects(
 	i := object.NewCommitPreorderIter(commit, seen, ignore)
 	pending := make(map[plumbing.Hash]bool)
 	addPendingParents(pending, visited, commit)
-
 	for {
 		commit, err := i.Next()
 		if err == io.EOF {
diff --git a/plumbing/revlist/revlist_test.go b/plumbing/revlist/revlist_test.go
index dea1c73..ceae727 100644
--- a/plumbing/revlist/revlist_test.go
+++ b/plumbing/revlist/revlist_test.go
@@ -129,6 +129,32 @@ func (s *RevListSuite) TestRevListObjectsTagObject(c *C) {
 	c.Assert(len(hist), Equals, len(expected))
 }
 
+func (s *RevListSuite) TestRevListObjectsWithStorageForIgnores(c *C) {
+	sto := filesystem.NewStorage(
+		fixtures.ByTag("merge-conflict").One().DotGit(),
+		cache.NewObjectLRUDefault())
+
+	// The "merge-conflict" repo has one extra commit in it, with a
+	// two files modified in two different subdirs.
+	expected := map[string]bool{
+		"1980fcf55330d9d94c34abee5ab734afecf96aba": true, // commit
+		"73d9cf44e9045254346c73f6646b08f9302c8570": true, // root dir
+		"e8435d512a98586bd2e4fcfcdf04101b0bb1b500": true, // go/
+		"257cc5642cb1a054f08cc83f2d943e56fd3ebe99": true, // haskal.hs
+		"d499a1a0b79b7d87a35155afd0c1cce78b37a91c": true, // example.go
+		"d108adc364fb6f21395d011ae2c8a11d96905b0d": true, // haskal/
+	}
+
+	hist, err := ObjectsWithStorageForIgnores(sto, s.Storer, []plumbing.Hash{plumbing.NewHash("1980fcf55330d9d94c34abee5ab734afecf96aba")}, []plumbing.Hash{plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5")})
+	c.Assert(err, IsNil)
+
+	for _, h := range hist {
+		c.Assert(expected[h.String()], Equals, true)
+	}
+
+	c.Assert(len(hist), Equals, len(expected))
+}
+
 // ---
 // | |\
 // | | * b8e471f Creating changelog
diff --git a/plumbing/storer/object.go b/plumbing/storer/object.go
index 2ac9b09..98d1ec3 100644
--- a/plumbing/storer/object.go
+++ b/plumbing/storer/object.go
@@ -222,7 +222,7 @@ type MultiEncodedObjectIter struct {
 }
 
 // NewMultiEncodedObjectIter returns an object iterator for the given slice of
-// objects.
+// EncodedObjectIters.
 func NewMultiEncodedObjectIter(iters []EncodedObjectIter) EncodedObjectIter {
 	return &MultiEncodedObjectIter{iters: iters}
 }
diff --git a/plumbing/storer/reference.go b/plumbing/storer/reference.go
index 5e85a3b..cce72b4 100644
--- a/plumbing/storer/reference.go
+++ b/plumbing/storer/reference.go
@@ -131,9 +131,27 @@ func (iter *ReferenceSliceIter) Next() (*plumbing.Reference, error) {
 // an error happens or the end of the iter is reached. If ErrStop is sent
 // the iteration is stop but no error is returned. The iterator is closed.
 func (iter *ReferenceSliceIter) ForEach(cb func(*plumbing.Reference) error) error {
+	return forEachReferenceIter(iter, cb)
+}
+
+type bareReferenceIterator interface {
+	Next() (*plumbing.Reference, error)
+	Close()
+}
+
+func forEachReferenceIter(iter bareReferenceIterator, cb func(*plumbing.Reference) error) error {
 	defer iter.Close()
-	for _, r := range iter.series {
-		if err := cb(r); err != nil {
+	for {
+		obj, err := iter.Next()
+		if err != nil {
+			if err == io.EOF {
+				return nil
+			}
+
+			return err
+		}
+
+		if err := cb(obj); err != nil {
 			if err == ErrStop {
 				return nil
 			}
@@ -141,8 +159,6 @@ func (iter *ReferenceSliceIter) ForEach(cb func(*plumbing.Reference) error) erro
 			return err
 		}
 	}
-
-	return nil
 }
 
 // Close releases any resources used by the iterator.
@@ -150,6 +166,52 @@ func (iter *ReferenceSliceIter) Close() {
 	iter.pos = len(iter.series)
 }
 
+// MultiReferenceIter implements ReferenceIter. It iterates over several
+// ReferenceIter,
+//
+// The MultiReferenceIter must be closed with a call to Close() when it is no
+// longer needed.
+type MultiReferenceIter struct {
+	iters []ReferenceIter
+}
+
+// NewMultiReferenceIter returns an reference iterator for the given slice of
+// EncodedObjectIters.
+func NewMultiReferenceIter(iters []ReferenceIter) ReferenceIter {
+	return &MultiReferenceIter{iters: iters}
+}
+
+// Next returns the next reference from the iterator, if one iterator reach
+// io.EOF is removed and the next one is used.
+func (iter *MultiReferenceIter) Next() (*plumbing.Reference, error) {
+	if len(iter.iters) == 0 {
+		return nil, io.EOF
+	}
+
+	obj, err := iter.iters[0].Next()
+	if err == io.EOF {
+		iter.iters[0].Close()
+		iter.iters = iter.iters[1:]
+		return iter.Next()
+	}
+
+	return obj, err
+}
+
+// ForEach call the cb function for each reference contained on this iter until
+// an error happens or the end of the iter is reached. If ErrStop is sent
+// the iteration is stop but no error is returned. The iterator is closed.
+func (iter *MultiReferenceIter) ForEach(cb func(*plumbing.Reference) error) error {
+	return forEachReferenceIter(iter, cb)
+}
+
+// Close releases any resources used by the iterator.
+func (iter *MultiReferenceIter) Close() {
+	for _, i := range iter.iters {
+		i.Close()
+	}
+}
+
 // ResolveReference resolves a SymbolicReference to a HashReference.
 func ResolveReference(s ReferenceStorer, n plumbing.ReferenceName) (*plumbing.Reference, error) {
 	r, err := s.Reference(n)
diff --git a/plumbing/storer/reference_test.go b/plumbing/storer/reference_test.go
index 490ec95..1d02c22 100644
--- a/plumbing/storer/reference_test.go
+++ b/plumbing/storer/reference_test.go
@@ -172,3 +172,26 @@ func (s *ReferenceSuite) TestReferenceFilteredIterForEachStop(c *C) {
 
 	c.Assert(count, Equals, 1)
 }
+
+func (s *ReferenceSuite) TestMultiReferenceIterForEach(c *C) {
+	i := NewMultiReferenceIter(
+		[]ReferenceIter{
+			NewReferenceSliceIter([]*plumbing.Reference{
+				plumbing.NewReferenceFromStrings("foo", "foo"),
+			}),
+			NewReferenceSliceIter([]*plumbing.Reference{
+				plumbing.NewReferenceFromStrings("bar", "bar"),
+			}),
+		},
+	)
+
+	var result []string
+	err := i.ForEach(func(r *plumbing.Reference) error {
+		result = append(result, r.Name().String())
+		return nil
+	})
+
+	c.Assert(err, IsNil)
+	c.Assert(result, HasLen, 2)
+	c.Assert(result, DeepEquals, []string{"foo", "bar"})
+}
diff --git a/plumbing/transport/common.go b/plumbing/transport/common.go
index f7b882b..dcf9391 100644
--- a/plumbing/transport/common.go
+++ b/plumbing/transport/common.go
@@ -19,10 +19,10 @@ import (
 	"fmt"
 	"io"
 	"net/url"
-	"regexp"
 	"strconv"
 	"strings"
 
+	giturl "gopkg.in/src-d/go-git.v4/internal/url"
 	"gopkg.in/src-d/go-git.v4/plumbing"
 	"gopkg.in/src-d/go-git.v4/plumbing/protocol/packp"
 	"gopkg.in/src-d/go-git.v4/plumbing/protocol/packp/capability"
@@ -224,34 +224,28 @@ func getPath(u *url.URL) string {
 	return res
 }
 
-var (
-	isSchemeRegExp   = regexp.MustCompile(`^[^:]+://`)
-	scpLikeUrlRegExp = regexp.MustCompile(`^(?:(?P<user>[^@]+)@)?(?P<host>[^:\s]+):(?:(?P<port>[0-9]{1,5})/)?(?P<path>[^\\].*)$`)
-)
-
 func parseSCPLike(endpoint string) (*Endpoint, bool) {
-	if isSchemeRegExp.MatchString(endpoint) || !scpLikeUrlRegExp.MatchString(endpoint) {
+	if giturl.MatchesScheme(endpoint) || !giturl.MatchesScpLike(endpoint) {
 		return nil, false
 	}
 
-	m := scpLikeUrlRegExp.FindStringSubmatch(endpoint)
-
-	port, err := strconv.Atoi(m[3])
+	user, host, portStr, path := giturl.FindScpLikeComponents(endpoint)
+	port, err := strconv.Atoi(portStr)
 	if err != nil {
 		port = 22
 	}
 
 	return &Endpoint{
 		Protocol: "ssh",
-		User:     m[1],
-		Host:     m[2],
+		User:     user,
+		Host:     host,
 		Port:     port,
-		Path:     m[4],
+		Path:     path,
 	}, true
 }
 
 func parseFile(endpoint string) (*Endpoint, bool) {
-	if isSchemeRegExp.MatchString(endpoint) {
+	if giturl.MatchesScheme(endpoint) {
 		return nil, false
 	}