// Package revision extracts git revision from string
// More information about revision : https://www.kernel.org/pub/software/scm/git/docs/gitrevisions.html
package revision
import (
"bytes"
"fmt"
"io"
"regexp"
"strconv"
"time"
)
// ErrInvalidRevision is emitted if string doesn't match valid revision
type ErrInvalidRevision struct {
s string
}
func (e *ErrInvalidRevision) Error() string {
return "Revision invalid : " + e.s
}
// Revisioner represents a revision component.
// A revision is made of multiple revision components
// obtained after parsing a revision string,
// for instance revision "master~" will be converted in
// two revision components Ref and TildePath
type Revisioner any
// Ref represents a reference name : HEAD, master, <hash>
type Ref string
// TildePath represents ~, ~{n}
type TildePath struct {
Depth int
}
// CaretPath represents ^, ^{n}
type CaretPath struct {
Depth int
}
// CaretReg represents ^{/foo bar}
type CaretReg struct {
Regexp *regexp.Regexp
Negate bool
}
// CaretType represents ^{commit}
type CaretType struct {
ObjectType string
}
// AtReflog represents @{n}
type AtReflog struct {
Depth int
}
// AtCheckout represents @{-n}
type AtCheckout struct {
Depth int
}
// AtUpstream represents @{upstream}, @{u}
type AtUpstream struct {
BranchName string
}
// AtPush represents @{push}
type AtPush struct {
BranchName string
}
// AtDate represents @{"2006-01-02T15:04:05Z"}
type AtDate struct {
Date time.Time
}
// ColonReg represents :/foo bar
type ColonReg struct {
Regexp *regexp.Regexp
Negate bool
}
// ColonPath represents :./<path> :<path>
type ColonPath struct {
Path string
}
// ColonStagePath represents :<n>:/<path>
type ColonStagePath struct {
Path string
Stage int
}
// Parser represents a parser
// use to tokenize and transform to revisioner chunks
// a given string
type Parser struct {
s *scanner
currentParsedChar struct {
tok token
lit string
}
unreadLastChar bool
}
// NewParserFromString returns a new instance of parser from a string.
func NewParserFromString(s string) *Parser {
return NewParser(bytes.NewBufferString(s))
}
// NewParser returns a new instance of parser.
func NewParser(r io.Reader) *Parser {
return &Parser{s: newScanner(r)}
}
// scan returns the next token from the underlying scanner
// or the last scanned token if an unscan was requested
func (p *Parser) scan() (token, string, error) {
if p.unreadLastChar {
p.unreadLastChar = false
return p.currentParsedChar.tok, p.currentParsedChar.lit, nil
}
tok, lit, err := p.s.scan()
p.currentParsedChar.tok, p.currentParsedChar.lit = tok, lit
return tok, lit, err
}
// unscan pushes the previously read token back onto the buffer.
func (p *Parser) unscan() { p.unreadLastChar = true }
// Parse explode a revision string into revisioner chunks
func (p *Parser) Parse() ([]Revisioner, error) {
var rev Revisioner
var revs []Revisioner
var tok token
var err error
for {
tok, _, err = p.scan()
if err != nil {
return nil, err
}
switch tok {
case at:
rev, err = p.parseAt()
case tilde:
rev, err = p.parseTilde()
case caret:
rev, err = p.parseCaret()
case colon:
rev, err = p.parseColon()
case eof:
err = p.validateFullRevision(&revs)
if err != nil {
return []Revisioner{}, err
}
return revs, nil
default:
p.unscan()
rev, err = p.parseRef()
}
if err != nil {
return []Revisioner{}, err
}
revs = append(revs, rev)
}
}
// validateFullRevision ensures all revisioner chunks make a valid revision
func (p *Parser) validateFullRevision(chunks *[]Revisioner) error {
var hasReference bool
for i, chunk := range *chunks {
switch chunk.(type) {
case Ref:
if i == 0 {
hasReference = true
} else {
return &ErrInvalidRevision{`reference must be defined once at the beginning`}
}
case AtDate:
if len(*chunks) == 1 || hasReference && len(*chunks) == 2 {
return nil
}
return &ErrInvalidRevision{`"@" statement is not valid, could be : <refname>@{<ISO-8601 date>}, @{<ISO-8601 date>}`}
case AtReflog:
if len(*chunks) == 1 || hasReference && len(*chunks) == 2 {
return nil
}
return &ErrInvalidRevision{`"@" statement is not valid, could be : <refname>@{<n>}, @{<n>}`}
case AtCheckout:
if len(*chunks) == 1 {
return nil
}
return &ErrInvalidRevision{`"@" statement is not valid, could be : @{-<n>}`}
case AtUpstream:
if len(*chunks) == 1 || hasReference && len(*chunks) == 2 {
return nil
}
return &ErrInvalidRevision{`"@" statement is not valid, could be : <refname>@{upstream}, @{upstream}, <refname>@{u}, @{u}`}
case AtPush:
if len(*chunks) == 1 || hasReference && len(*chunks) == 2 {
return nil
}
return &ErrInvalidRevision{`"@" statement is not valid, could be : <refname>@{push}, @{push}`}
case TildePath, CaretPath, CaretReg:
if !hasReference {
return &ErrInvalidRevision{`"~" or "^" statement must have a reference defined at the beginning`}
}
case ColonReg:
if len(*chunks) == 1 {
return nil
}
return &ErrInvalidRevision{`":" statement is not valid, could be : :/<regexp>`}
case ColonPath:
if i == len(*chunks)-1 && hasReference || len(*chunks) == 1 {
return nil
}
return &ErrInvalidRevision{`":" statement is not valid, could be : <revision>:<path>`}
case ColonStagePath:
if len(*chunks) == 1 {
return nil
}
return &ErrInvalidRevision{`":" statement is not valid, could be : :<n>:<path>`}
}
}
return nil
}
// parseAt extract @ statements
func (p *Parser) parseAt() (Revisioner, error) {
var tok, nextTok token
var lit, nextLit string
var err error
tok, _, err = p.scan()
if err != nil {
return nil, err
}
if tok != obrace {
p.unscan()
return Ref("HEAD"), nil
}
tok, lit, err = p.scan()
if err != nil {
return nil, err
}
nextTok, nextLit, err = p.scan()
if err != nil {
return nil, err
}
switch {
case tok == word && (lit == "u" || lit == "upstream") && nextTok == cbrace:
return AtUpstream{}, nil
case tok == word && lit == "push" && nextTok == cbrace:
return AtPush{}, nil
case tok == number && nextTok == cbrace:
n, _ := strconv.Atoi(lit)
return AtReflog{n}, nil
case tok == minus && nextTok == number:
n, _ := strconv.Atoi(nextLit)
t, _, err := p.scan()
if err != nil {
return nil, err
}
if t != cbrace {
return nil, &ErrInvalidRevision{s: `missing "}" in @{-n} structure`}
}
return AtCheckout{n}, nil
default:
p.unscan()
date := lit
for {
tok, lit, err = p.scan()
if err != nil {
return nil, err
}
switch tok {
case cbrace:
t, err := time.Parse("2006-01-02T15:04:05Z", date)
if err != nil {
return nil, &ErrInvalidRevision{fmt.Sprintf(`wrong date "%s" must fit ISO-8601 format : 2006-01-02T15:04:05Z`, date)}
}
return AtDate{t}, nil
case eof:
return nil, &ErrInvalidRevision{s: `missing "}" in @{<data>} structure`}
default:
date += lit
}
}
}
}
// parseTilde extract ~ statements
func (p *Parser) parseTilde() (Revisioner, error) {
var tok token
var lit string
var err error
tok, lit, err = p.scan()
if err != nil {
return nil, err
}
switch tok {
case number:
n, _ := strconv.Atoi(lit)
return TildePath{n}, nil
default:
p.unscan()
return TildePath{1}, nil
}
}
// parseCaret extract ^ statements
func (p *Parser) parseCaret() (Revisioner, error) {
var tok token
var lit string
var err error
tok, lit, err = p.scan()
if err != nil {
return nil, err
}
switch tok {
case obrace:
r, err := p.parseCaretBraces()
if err != nil {
return nil, err
}
return r, nil
case number:
n, _ := strconv.Atoi(lit)
if n > 2 {
return nil, &ErrInvalidRevision{fmt.Sprintf(`"%s" found must be 0, 1 or 2 after "^"`, lit)}
}
return CaretPath{n}, nil
default:
p.unscan()
return CaretPath{1}, nil
}
}
// parseCaretBraces extract ^{<data>} statements
func (p *Parser) parseCaretBraces() (Revisioner, error) {
var tok, nextTok token
var lit, _ string
start := true
var re string
var negate bool
var err error
for {
tok, lit, err = p.scan()
if err != nil {
return nil, err
}
nextTok, _, err = p.scan()
if err != nil {
return nil, err
}
switch {
case tok == word && nextTok == cbrace && (lit == "commit" || lit == "tree" || lit == "blob" || lit == "tag" || lit == "object"):
return CaretType{lit}, nil
case re == "" && tok == cbrace:
return CaretType{"tag"}, nil
case re == "" && tok == emark && nextTok == emark:
re += lit
case re == "" && tok == emark && nextTok == minus:
negate = true
case re == "" && tok == emark:
return nil, &ErrInvalidRevision{s: `revision suffix brace component sequences starting with "/!" others than those defined are reserved`}
case re == "" && tok == slash:
p.unscan()
case tok != slash && start:
return nil, &ErrInvalidRevision{fmt.Sprintf(`"%s" is not a valid revision suffix brace component`, lit)}
case tok == eof:
return nil, &ErrInvalidRevision{s: `missing "}" in ^{<data>} structure`}
case tok != cbrace:
p.unscan()
re += lit
case tok == cbrace:
p.unscan()
reg, err := regexp.Compile(re)
if err != nil {
return CaretReg{}, &ErrInvalidRevision{fmt.Sprintf(`revision suffix brace component, %s`, err.Error())}
}
return CaretReg{reg, negate}, nil
}
start = false
}
}
// parseColon extract : statements
func (p *Parser) parseColon() (Revisioner, error) {
var tok token
var err error
tok, _, err = p.scan()
if err != nil {
return nil, err
}
switch tok {
case slash:
return p.parseColonSlash()
default:
p.unscan()
return p.parseColonDefault()
}
}
// parseColonSlash extract :/<data> statements
func (p *Parser) parseColonSlash() (Revisioner, error) {
var tok, nextTok token
var lit string
var re string
var negate bool
var err error
for {
tok, lit, err = p.scan()
if err != nil {
return nil, err
}
nextTok, _, err = p.scan()
if err != nil {
return nil, err
}
switch {
case tok == emark && nextTok == emark:
re += lit
case re == "" && tok == emark && nextTok == minus:
negate = true
case re == "" && tok == emark:
return nil, &ErrInvalidRevision{s: `revision suffix brace component sequences starting with "/!" others than those defined are reserved`}
case tok == eof:
p.unscan()
reg, err := regexp.Compile(re)
if err != nil {
return ColonReg{}, &ErrInvalidRevision{fmt.Sprintf(`revision suffix brace component, %s`, err.Error())}
}
return ColonReg{reg, negate}, nil
default:
p.unscan()
re += lit
}
}
}
// parseColonDefault extract :<data> statements
func (p *Parser) parseColonDefault() (Revisioner, error) {
var tok token
var lit string
var path string
var stage int
var err error
n := -1
tok, lit, err = p.scan()
if err != nil {
return nil, err
}
nextTok, _, err := p.scan()
if err != nil {
return nil, err
}
if tok == number && nextTok == colon {
n, _ = strconv.Atoi(lit)
}
switch n {
case 0, 1, 2, 3:
stage = n
default:
path += lit
p.unscan()
}
for {
tok, lit, err = p.scan()
if err != nil {
return nil, err
}
switch {
case tok == eof && n == -1:
return ColonPath{path}, nil
case tok == eof:
return ColonStagePath{path, stage}, nil
default:
path += lit
}
}
}
// parseRef extract reference name
func (p *Parser) parseRef() (Revisioner, error) {
var tok, prevTok token
var lit, buf string
var endOfRef bool
var err error
for {
tok, lit, err = p.scan()
if err != nil {
return nil, err
}
switch tok {
case eof, at, colon, tilde, caret:
endOfRef = true
}
err := p.checkRefFormat(tok, lit, prevTok, buf, endOfRef)
if err != nil {
return "", err
}
if endOfRef {
p.unscan()
if buf == "@" {
return Ref("HEAD"), nil
}
return Ref(buf), nil
}
buf += lit
prevTok = tok
}
}
// checkRefFormat ensure reference name follow rules defined here :
// https://git-scm.com/docs/git-check-ref-format
func (p *Parser) checkRefFormat(token token, literal string, previousToken token, buffer string, endOfRef bool) error {
switch token {
case aslash, space, control, qmark, asterisk, obracket:
return &ErrInvalidRevision{fmt.Sprintf(`must not contains "%s"`, literal)}
}
switch {
case (token == dot || token == slash) && buffer == "":
return &ErrInvalidRevision{fmt.Sprintf(`must not start with "%s"`, literal)}
case previousToken == slash && endOfRef:
return &ErrInvalidRevision{`must not end with "/"`}
case previousToken == dot && endOfRef:
return &ErrInvalidRevision{`must not end with "."`}
case token == dot && previousToken == slash:
return &ErrInvalidRevision{`must not contains "/."`}
case previousToken == dot && token == dot:
return &ErrInvalidRevision{`must not contains ".."`}
case previousToken == slash && token == slash:
return &ErrInvalidRevision{`must not contains consecutively "/"`}
case (token == slash || endOfRef) && len(buffer) > 4 && buffer[len(buffer)-5:] == ".lock":
return &ErrInvalidRevision{"cannot end with .lock"}
}
return nil
}
package revision
import (
"bufio"
"io"
"unicode"
)
// runeCategoryValidator takes a rune as input and
// validates it belongs to a rune category
type runeCategoryValidator func(r rune) bool
// tokenizeExpression aggregates a series of runes matching check predicate into a single
// string and provides given tokenType as token type
func tokenizeExpression(ch rune, tokenType token, check runeCategoryValidator, r *bufio.Reader) (token, string, error) {
var data []rune
data = append(data, ch)
for {
c, _, err := r.ReadRune()
if c == zeroRune {
break
}
if err != nil {
return tokenError, "", err
}
if check(c) {
data = append(data, c)
} else {
err := r.UnreadRune()
if err != nil {
return tokenError, "", err
}
return tokenType, string(data), nil
}
}
return tokenType, string(data), nil
}
// maxRevisionLength holds the maximum length that will be parsed for a
// revision. Git itself doesn't enforce a max length, but rather leans on
// the OS to enforce it via its ARG_MAX.
const maxRevisionLength = 128 * 1024 // 128kb
var zeroRune = rune(0)
// scanner represents a lexical scanner.
type scanner struct {
r *bufio.Reader
}
// newScanner returns a new instance of scanner.
func newScanner(r io.Reader) *scanner {
return &scanner{r: bufio.NewReader(io.LimitReader(r, maxRevisionLength))}
}
// Scan extracts tokens and their strings counterpart
// from the reader
func (s *scanner) scan() (token, string, error) {
ch, _, err := s.r.ReadRune()
if err != nil && err != io.EOF {
return tokenError, "", err
}
switch ch {
case zeroRune:
return eof, "", nil
case ':':
return colon, string(ch), nil
case '~':
return tilde, string(ch), nil
case '^':
return caret, string(ch), nil
case '.':
return dot, string(ch), nil
case '/':
return slash, string(ch), nil
case '{':
return obrace, string(ch), nil
case '}':
return cbrace, string(ch), nil
case '-':
return minus, string(ch), nil
case '@':
next, _, err := s.r.ReadRune()
if err == io.EOF || next == zeroRune {
return at, string(ch), nil
}
if err != nil {
return tokenError, "", err
}
if err := s.r.UnreadRune(); err != nil {
return tokenError, "", err
}
if next == '{' {
return at, string(ch), nil
}
return word, string(ch), nil
case '\\':
return aslash, string(ch), nil
case '?':
return qmark, string(ch), nil
case '*':
return asterisk, string(ch), nil
case '[':
return obracket, string(ch), nil
case '!':
return emark, string(ch), nil
}
if unicode.IsSpace(ch) {
return space, string(ch), nil
}
if unicode.IsControl(ch) {
return control, string(ch), nil
}
if unicode.IsLetter(ch) {
return tokenizeExpression(ch, word, unicode.IsLetter, s.r)
}
if unicode.IsNumber(ch) {
return tokenizeExpression(ch, number, unicode.IsNumber, s.r)
}
return tokenError, string(ch), nil
}
package commitgraph
import (
"bufio"
"io"
"path"
"github.com/go-git/go-billy/v6"
"github.com/go-git/go-git/v6/plumbing"
)
// OpenChainFile reads a commit chain file and returns a slice of the hashes within it
//
// Commit-Graph chains are described at https://git-scm.com/docs/commit-graph
// and are new line separated list of graph file hashes, oldest to newest.
//
// This function simply reads the file and returns the hashes as a slice.
func OpenChainFile(r io.Reader) ([]string, error) {
if r == nil {
return nil, io.ErrUnexpectedEOF
}
bufRd := bufio.NewReader(r)
chain := make([]string, 0, 8)
for {
line, err := bufRd.ReadSlice('\n')
if err != nil {
if err == io.EOF {
break
}
return nil, err
}
hashStr := string(line[:len(line)-1])
if !plumbing.IsHash(hashStr) {
return nil, ErrMalformedCommitGraphFile
}
chain = append(chain, hashStr)
}
return chain, nil
}
// OpenChainOrFileIndex expects a billy.Filesystem representing a .git directory.
// It will first attempt to read a commit-graph index file, before trying to read a
// commit-graph chain file and its index files. If neither are present, an error is returned.
// Otherwise an Index will be returned.
//
// See: https://git-scm.com/docs/commit-graph
func OpenChainOrFileIndex(fs billy.Filesystem) (Index, error) {
file, err := fs.Open(path.Join("objects", "info", "commit-graph"))
if err != nil {
// try to open a chain file
return OpenChainIndex(fs)
}
index, err := OpenFileIndex(file)
if err != nil {
// Ignore any file closing errors and return the error from OpenFileIndex instead
_ = file.Close()
return nil, err
}
return index, nil
}
// OpenChainIndex expects a billy.Filesystem representing a .git directory.
// It will read a commit-graph chain file and return a coalesced index.
// If the chain file or a graph in that chain is not present, an error is returned.
//
// See: https://git-scm.com/docs/commit-graph
func OpenChainIndex(fs billy.Filesystem) (Index, error) {
chainFile, err := fs.Open(path.Join("objects", "info", "commit-graphs", "commit-graph-chain"))
if err != nil {
return nil, err
}
chain, err := OpenChainFile(chainFile)
_ = chainFile.Close()
if err != nil {
return nil, err
}
var index Index
for _, hash := range chain {
file, err := fs.Open(path.Join("objects", "info", "commit-graphs", "graph-"+hash+".graph"))
if err != nil {
// Ignore all other file closing errors and return the error from opening the last file in the graph
_ = index.Close()
return nil, err
}
index, err = OpenFileIndexWithParent(file, index)
if err != nil {
// Ignore file closing errors and return the error from OpenFileIndex instead
_ = index.Close()
return nil, err
}
}
return index, nil
}
package commitgraph
import "bytes"
const (
szChunkSig = 4 // Length of a chunk signature
chunkSigOffset = 4 // Offset of each chunk signature in chunkSignatures
)
// chunkSignatures contains the coalesced byte signatures for each chunk type.
// The order of the signatures must match the order of the ChunkType constants.
// (When adding new chunk types you must avoid introducing ambiguity, and you may need to add padding separators to this list or reorder these signatures.)
// (i.e. it would not be possible to add a new chunk type with the signature "IDFO" without some reordering or the addition of separators.)
var chunkSignatures = []byte("OIDFOIDLCDATGDA2GDO2EDGEBIDXBDATBASE\000\000\000\000")
// ChunkType represents the type of a chunk in the commit graph file.
type ChunkType int
// Chunk types in the commit graph file.
const (
OIDFanoutChunk ChunkType = iota // "OIDF"
OIDLookupChunk // "OIDL"
CommitDataChunk // "CDAT"
GenerationDataChunk // "GDA2"
GenerationDataOverflowChunk // "GDO2"
ExtraEdgeListChunk // "EDGE"
BloomFilterIndexChunk // "BIDX"
BloomFilterDataChunk // "BDAT"
BaseGraphsListChunk // "BASE"
ZeroChunk // "\000\000\000\000"
)
const lenChunks = int(ZeroChunk) // ZeroChunk is not a valid chunk type, but it is used to determine the length of the chunk type list.
// Signature returns the byte signature for the chunk type.
func (ct ChunkType) Signature() []byte {
if ct >= BaseGraphsListChunk || ct < 0 { // not a valid chunk type just return ZeroChunk
return chunkSignatures[ZeroChunk*chunkSigOffset : ZeroChunk*chunkSigOffset+szChunkSig]
}
return chunkSignatures[ct*chunkSigOffset : ct*chunkSigOffset+szChunkSig]
}
// ChunkTypeFromBytes returns the chunk type for the given byte signature.
func ChunkTypeFromBytes(b []byte) (ChunkType, bool) {
idx := bytes.Index(chunkSignatures, b)
if idx == -1 || idx%chunkSigOffset != 0 { // not found, or not aligned at chunkSigOffset
return -1, false
}
return ChunkType(idx / chunkSigOffset), true
}
package commitgraph
import (
"io"
"math"
"time"
"github.com/go-git/go-git/v6/plumbing"
)
// CommitData is a reduced representation of Commit as presented in the commit graph
// file. It is merely useful as an optimization for walking the commit graphs.
type CommitData struct {
// TreeHash is the hash of the root tree of the commit.
TreeHash plumbing.Hash
// ParentIndexes are the indexes of the parent commits of the commit.
ParentIndexes []uint32
// ParentHashes are the hashes of the parent commits of the commit.
ParentHashes []plumbing.Hash
// Generation number is the pre-computed generation in the commit graph
// or zero if not available.
Generation uint64
// GenerationV2 stores the corrected commit date for the commits
// It combines the contents of the GDA2 and GDO2 sections of the commit-graph
// with the commit time portion of the CDAT section.
GenerationV2 uint64
// When is the timestamp of the commit.
When time.Time
}
// GenerationV2Data returns the corrected commit date for the commits
func (c *CommitData) GenerationV2Data() uint64 {
if c.GenerationV2 == 0 || c.GenerationV2 == math.MaxUint64 {
return 0
}
return c.GenerationV2 - uint64(c.When.Unix())
}
// Index represents a representation of commit graph that allows indexed
// access to the nodes using commit object hash
type Index interface {
// GetIndexByHash gets the index in the commit graph from commit hash, if available
GetIndexByHash(h plumbing.Hash) (uint32, error)
// GetHashByIndex gets the hash given an index in the commit graph
GetHashByIndex(i uint32) (plumbing.Hash, error)
// GetNodeByIndex gets the commit node from the commit graph using index
// obtained from child node, if available
GetCommitDataByIndex(i uint32) (*CommitData, error)
// Hashes returns all the hashes that are available in the index
Hashes() []plumbing.Hash
// HasGenerationV2 returns true if the commit graph has the corrected commit date data
HasGenerationV2() bool
// MaximumNumberOfHashes returns the maximum number of hashes within the index
MaximumNumberOfHashes() uint32
io.Closer
}
package commitgraph
import (
"crypto"
"fmt"
"io"
"math"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/hash"
"github.com/go-git/go-git/v6/utils/binary"
)
// Encoder writes MemoryIndex structs to an output stream.
type Encoder struct {
io.Writer
hash hash.Hash
}
// NewEncoder returns a new stream encoder that writes to w.
func NewEncoder(w io.Writer) *Encoder {
// TODO: Support passing an ObjectFormat (sha256)
h := hash.New(crypto.SHA1)
mw := io.MultiWriter(w, h)
return &Encoder{mw, h}
}
// Encode writes an index into the commit-graph file
func (e *Encoder) Encode(idx Index) error {
// Get all the hashes in the input index
hashes := idx.Hashes()
// Sort the input and prepare helper structures we'll need for encoding
hashToIndex, fanout, extraEdgesCount, generationV2OverflowCount, err := e.prepare(idx, hashes)
if err != nil {
return err
}
chunkSignatures := [][]byte{OIDFanoutChunk.Signature(), OIDLookupChunk.Signature(), CommitDataChunk.Signature()}
chunkSizes := []uint64{szUint32 * lenFanout, uint64(len(hashes) * e.hash.Size()), uint64(len(hashes) * (e.hash.Size() + szCommitData))}
if extraEdgesCount > 0 {
chunkSignatures = append(chunkSignatures, ExtraEdgeListChunk.Signature())
chunkSizes = append(chunkSizes, uint64(extraEdgesCount)*szUint32)
}
if idx.HasGenerationV2() {
chunkSignatures = append(chunkSignatures, GenerationDataChunk.Signature())
chunkSizes = append(chunkSizes, uint64(len(hashes))*szUint32)
if generationV2OverflowCount > 0 {
chunkSignatures = append(chunkSignatures, GenerationDataOverflowChunk.Signature())
chunkSizes = append(chunkSizes, uint64(generationV2OverflowCount)*szUint64)
}
}
if err := e.encodeFileHeader(len(chunkSignatures)); err != nil {
return err
}
if err := e.encodeChunkHeaders(chunkSignatures, chunkSizes); err != nil {
return err
}
if err := e.encodeFanout(fanout); err != nil {
return err
}
if err := e.encodeOidLookup(hashes); err != nil {
return err
}
extraEdges, generationV2Data, err := e.encodeCommitData(hashes, hashToIndex, idx)
if err != nil {
return err
}
if err = e.encodeExtraEdges(extraEdges); err != nil {
return err
}
if idx.HasGenerationV2() {
overflows, err := e.encodeGenerationV2Data(generationV2Data)
if err != nil {
return err
}
if err = e.encodeGenerationV2Overflow(overflows); err != nil {
return err
}
}
return e.encodeChecksum()
}
// lookupParentIndex resolves a parent hash to its position in the file being
// encoded. A bare map read yields index 0 for an absent hash, which would
// silently record an arbitrary commit as the parent, so report it instead.
func lookupParentIndex(hashToIndex map[plumbing.Hash]uint32, h plumbing.Hash) (uint32, error) {
i, ok := hashToIndex[h]
if !ok {
return 0, fmt.Errorf("%w: %s", ErrParentNotInIndex, h)
}
return i, nil
}
func (e *Encoder) prepare(idx Index, hashes []plumbing.Hash) (hashToIndex map[plumbing.Hash]uint32, fanout []uint32, extraEdgesCount, generationV2OverflowCount uint32, err error) {
// Sort the hashes and build our index
plumbing.HashesSort(hashes)
hashToIndex = make(map[plumbing.Hash]uint32)
fanout = make([]uint32, lenFanout)
for i, hash := range hashes {
hashToIndex[hash] = uint32(i)
fanout[hash.Bytes()[0]]++
}
// Convert the fanout to cumulative values
for i := 1; i < lenFanout; i++ {
fanout[i] += fanout[i-1]
}
hasGenerationV2 := idx.HasGenerationV2()
// Find out if we will need extra edge table. An index that cannot satisfy
// the lookup returns a nil CommitData, so the error has to be checked
// before v is dereferenced.
for i := range len(hashes) {
v, err := idx.GetCommitDataByIndex(uint32(i))
if err != nil {
return nil, nil, 0, 0, err
}
if len(v.ParentHashes) > 2 {
extraEdgesCount += uint32(len(v.ParentHashes) - 1)
}
if hasGenerationV2 && v.GenerationV2Data() > math.MaxUint32 {
generationV2OverflowCount++
}
}
return hashToIndex, fanout, extraEdgesCount, generationV2OverflowCount, nil
}
func (e *Encoder) encodeFileHeader(chunkCount int) (err error) {
if chunkCount > 255 {
return ErrTooManyChunks
}
if _, err = e.Write(commitFileSignature); err == nil {
version := byte(1)
if crypto.Hash(e.hash.Size()) == crypto.Hash(crypto.SHA256.Size()) {
version = byte(2)
}
_, err = e.Write([]byte{1, version, byte(chunkCount), 0})
}
return err
}
func (e *Encoder) encodeChunkHeaders(chunkSignatures [][]byte, chunkSizes []uint64) (err error) {
// 8 bytes of file header, 12 bytes for each chunk header and 12 byte for terminator
offset := uint64(szSignature + szHeader + (len(chunkSignatures)+1)*(szChunkSig+szUint64))
for i, signature := range chunkSignatures {
if _, err = e.Write(signature); err == nil {
err = binary.WriteUint64(e, offset)
}
if err != nil {
return err
}
offset += chunkSizes[i]
}
if _, err = e.Write(ZeroChunk.Signature()); err == nil {
err = binary.WriteUint64(e, offset)
}
return err
}
func (e *Encoder) encodeFanout(fanout []uint32) (err error) {
for i := 0; i <= 0xff; i++ {
if err = binary.WriteUint32(e, fanout[i]); err != nil {
return err
}
}
return err
}
func (e *Encoder) encodeOidLookup(hashes []plumbing.Hash) (err error) {
for _, hash := range hashes {
if _, err = e.Write(hash.Bytes()); err != nil {
return err
}
}
return err
}
func (e *Encoder) encodeCommitData(hashes []plumbing.Hash, hashToIndex map[plumbing.Hash]uint32, idx Index) (extraEdges []uint32, generationV2Data []uint64, err error) {
if idx.HasGenerationV2() {
generationV2Data = make([]uint64, 0, len(hashes))
}
for _, hash := range hashes {
// Both lookups can fail, and commitData is nil when the second one
// does.
origIndex, err := idx.GetIndexByHash(hash)
if err != nil {
return extraEdges, generationV2Data, err
}
commitData, err := idx.GetCommitDataByIndex(origIndex)
if err != nil {
return extraEdges, generationV2Data, err
}
if _, err = e.Write(commitData.TreeHash.Bytes()); err != nil {
return extraEdges, generationV2Data, err
}
var parent1, parent2 uint32
switch len(commitData.ParentHashes) {
case 0:
parent1 = parentNone
parent2 = parentNone
case 1:
if parent1, err = lookupParentIndex(hashToIndex, commitData.ParentHashes[0]); err != nil {
return extraEdges, generationV2Data, err
}
parent2 = parentNone
case 2:
if parent1, err = lookupParentIndex(hashToIndex, commitData.ParentHashes[0]); err != nil {
return extraEdges, generationV2Data, err
}
if parent2, err = lookupParentIndex(hashToIndex, commitData.ParentHashes[1]); err != nil {
return extraEdges, generationV2Data, err
}
default:
if parent1, err = lookupParentIndex(hashToIndex, commitData.ParentHashes[0]); err != nil {
return extraEdges, generationV2Data, err
}
parent2 = uint32(len(extraEdges)) | parentOctopusUsed
for _, parentHash := range commitData.ParentHashes[1:] {
extraEdge, err := lookupParentIndex(hashToIndex, parentHash)
if err != nil {
return extraEdges, generationV2Data, err
}
extraEdges = append(extraEdges, extraEdge)
}
extraEdges[len(extraEdges)-1] |= parentLast
}
if err = binary.WriteUint32(e, parent1); err == nil {
err = binary.WriteUint32(e, parent2)
}
if err != nil {
return extraEdges, generationV2Data, err
}
unixTime := uint64(commitData.When.Unix())
unixTime |= uint64(commitData.Generation) << 34
if err = binary.WriteUint64(e, unixTime); err != nil {
return extraEdges, generationV2Data, err
}
if generationV2Data != nil {
generationV2Data = append(generationV2Data, commitData.GenerationV2Data())
}
}
return extraEdges, generationV2Data, err
}
func (e *Encoder) encodeExtraEdges(extraEdges []uint32) (err error) {
for _, parent := range extraEdges {
if err = binary.WriteUint32(e, parent); err != nil {
return err
}
}
return err
}
func (e *Encoder) encodeGenerationV2Data(generationV2Data []uint64) (overflows []uint64, err error) {
head := 0
for _, data := range generationV2Data {
if data >= 0x80000000 {
// overflow
if err = binary.WriteUint32(e, uint32(head)|0x80000000); err != nil {
return nil, err
}
generationV2Data[head] = data
head++
continue
}
if err = binary.WriteUint32(e, uint32(data)); err != nil {
return nil, err
}
}
return generationV2Data[:head], nil
}
func (e *Encoder) encodeGenerationV2Overflow(overflows []uint64) (err error) {
for _, overflow := range overflows {
if err = binary.WriteUint64(e, overflow); err != nil {
return err
}
}
return err
}
func (e *Encoder) encodeChecksum() error {
_, err := e.Write(e.hash.Sum(nil)[:e.hash.Size()])
return err
}
package commitgraph
import (
"bytes"
"crypto"
encbin "encoding/binary"
"errors"
"io"
"math"
"time"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/format/config"
"github.com/go-git/go-git/v6/utils/binary"
)
var (
// ErrUnsupportedVersion is returned by OpenFileIndex when the commit graph
// file version is not supported.
ErrUnsupportedVersion = errors.New("unsupported version")
// ErrUnsupportedHash is returned by OpenFileIndex when the commit graph
// hash function is not supported. Currently only SHA-1 is defined and
// supported.
ErrUnsupportedHash = errors.New("unsupported hash algorithm")
// ErrMalformedCommitGraphFile is returned by OpenFileIndex when the commit
// graph file is corrupted.
ErrMalformedCommitGraphFile = errors.New("malformed commit graph file")
// ErrTooManyChunks is returned by Encoder.Encode when the assembled
// chunk-table configuration would not fit the uint8 the on-disk
// header stores at byte 6.
ErrTooManyChunks = errors.New("commitgraph: too many chunks")
// ErrParentNotInIndex is returned by Encoder.Encode when a commit names a
// parent that the index being written does not contain, so no edge can be
// recorded for it.
ErrParentNotInIndex = errors.New("commitgraph: parent is not part of the index being encoded")
commitFileSignature = []byte{'C', 'G', 'P', 'H'}
parentNone = uint32(0x70000000)
parentOctopusUsed = uint32(0x80000000)
parentOctopusMask = uint32(0x7fffffff)
parentLast = uint32(0x80000000)
)
const (
szUint32 = 4
szUint64 = 8
szSignature = 4
szHeader = 4
szCommitData = 2*szUint32 + szUint64
lenFanout = 256
)
type sizer interface {
Size() int64
}
// readerSize returns the byte length reachable from r. It honours bytes.Reader
// (Size()) and any io.Seeker (Seek to SeekEnd). When neither is available
// the size is reported as 0 with a non-nil error so callers can decide
// whether to skip the size-dependent checks.
func readerSize(r io.ReaderAt) (int64, error) {
if s, ok := r.(sizer); ok {
return s.Size(), nil
}
if s, ok := r.(io.Seeker); ok {
return s.Seek(0, io.SeekEnd)
}
return 0, errors.New("commitgraph: cannot determine reader size")
}
type fileIndex struct {
reader ReaderAtCloser
fanout [lenFanout]uint32
offsets [lenChunks]int64
sizes [lenChunks]int64 // byte length of each known chunk
parent Index
hasGenerationV2 bool
minimumNumberOfHashes uint32
objSize int
numChunks uint8
fileSize int64
}
// ReaderAtCloser is an interface that combines io.ReaderAt and io.Closer.
type ReaderAtCloser interface {
io.ReaderAt
io.Closer
}
// OpenFileIndex opens a serialized commit graph file in the format described at
// https://github.com/git/git/blob/v2.54.0/Documentation/technical/commit-graph-format.adoc
func OpenFileIndex(reader ReaderAtCloser) (Index, error) {
return OpenFileIndexWithParent(reader, nil)
}
// OpenFileIndexWithParent opens a serialized commit graph file in the format described at
// https://github.com/git/git/blob/v2.54.0/Documentation/technical/commit-graph-format.adoc
func OpenFileIndexWithParent(reader ReaderAtCloser, parent Index) (Index, error) {
if reader == nil {
return nil, io.ErrUnexpectedEOF
}
fi := &fileIndex{reader: reader, parent: parent, objSize: config.SHA1Size}
if err := fi.verifyFileHeader(); err != nil {
return nil, err
}
if err := fi.verifyFileSize(); err != nil {
return nil, err
}
if err := fi.readChunkHeaders(); err != nil {
return nil, err
}
if err := fi.verifyChunkSizes(); err != nil {
return nil, err
}
if err := fi.readFanout(); err != nil {
return nil, err
}
fi.hasGenerationV2 = fi.offsets[GenerationDataChunk] > 0
if fi.parent != nil {
fi.hasGenerationV2 = fi.hasGenerationV2 && fi.parent.HasGenerationV2()
}
if fi.parent != nil {
fi.minimumNumberOfHashes = fi.parent.MaximumNumberOfHashes()
}
return fi, nil
}
// Close closes the underlying reader and the parent index if it exists.
func (fi *fileIndex) Close() (err error) {
if fi.parent != nil {
defer func() {
parentErr := fi.parent.Close()
// only report the error from the parent if there is no error from the reader
if err == nil {
err = parentErr
}
}()
}
err = fi.reader.Close()
return err
}
func (fi *fileIndex) verifyFileHeader() error {
// Verify file signature
signature := make([]byte, szSignature)
if _, err := fi.reader.ReadAt(signature, 0); err != nil {
return err
}
if !bytes.Equal(signature, commitFileSignature) {
return ErrMalformedCommitGraphFile
}
// Read and verify the file header
header := make([]byte, szHeader)
if _, err := fi.reader.ReadAt(header, szHeader); err != nil {
return err
}
if header[0] != 1 {
return ErrUnsupportedVersion
}
if (fi.objSize != crypto.SHA1.Size() || header[1] != 1) &&
(fi.objSize != crypto.SHA256.Size() || header[1] != 2) {
// Unknown hash type / unsupported hash type
return ErrUnsupportedHash
}
fi.numChunks = header[2]
return nil
}
// verifyFileSize records the reader's byte length on fi.fileSize and
// mirrors canonical Git's parse_commit_graph_v1 check [1] that the
// file is large enough to hold the header, the full chunk table of
// contents (including the zero terminator), the fanout table, and
// the trailing hash trailer.
//
// If the reader satisfies neither sizer nor io.Seeker the size is
// left at zero and the precheck is skipped; the per-chunk reads in
// readChunkHeaders still detect truncation reactively.
//
// [1]: https://github.com/git/git/blob/v2.54.0/commit-graph.c#L419
func (fi *fileIndex) verifyFileSize() error {
size, err := readerSize(fi.reader)
if err != nil {
// Without a size we fall back on per-chunk reads to detect
// truncation. The reader interface only requires io.ReaderAt
// and io.Closer, so this branch is taken by exotic callers
// only; the in-tree filesystem and in-memory paths both
// satisfy sizer or io.Seeker.
return nil
}
fi.fileSize = size
minSize := int64(szSignature+szHeader) +
int64(uint16(fi.numChunks)+1)*int64(szChunkSig+szUint64) +
int64(lenFanout*szUint32) +
int64(fi.objSize)
if size < minSize {
return ErrMalformedCommitGraphFile
}
return nil
}
// chunkAssignment records the file offset of a known chunk type in
// table-of-contents order, so that readChunkHeaders can derive each
// chunk's byte length from adjacent offsets once the terminator is found.
type chunkAssignment struct {
ct ChunkType
offset int64
}
// readChunkHeaders parses the chunk table of contents. The number of
// non-terminating entries is taken from the file header (byte 6), mirroring
// canonical Git's parse_commit_graph_v1 [1] which passes that count to
// read_table_of_contents [2]; the latter iterates exactly num_chunks times
// and rejects both an early zero chunk-id and a non-zero terminator entry.
//
// After the terminator offset is known, the byte length of every known chunk
// is computed as the difference between its starting offset and that of the
// next entry in table-of-contents order (or the terminator for the last
// one). These lengths are stored in fi.sizes and used by GetCommitDataByIndex
// to bound the octopus extra-edge walk, mirroring canonical Git's
// chunk_extra_edges_size / sizeof(uint32_t) guard in fill_commit_in_graph.
//
// [1]: https://github.com/git/git/blob/v2.54.0/commit-graph.c#L414
// [2]: https://github.com/git/git/blob/v2.54.0/chunk-format.c#L117
func (fi *fileIndex) readChunkHeaders() error {
tocBase := int64(szSignature + szHeader)
const tocEntrySize = szChunkSig + szUint64
chunkID := make([]byte, szChunkSig) // reused across loop iterations
var prevOffset int64
// Canonical Git's read_table_of_contents [2] validates each chunk's
// offset against mfile_size - hash_size (upper bound) and the
// previous offset (monotonicity). Mirror that check here.
upperBound := fi.fileSize - int64(fi.objSize)
if fi.fileSize == 0 {
// verifyFileSize was unable to determine the file size; skip
// the upper-bound check, the per-chunk ReadAt will catch
// out-of-range offsets reactively.
upperBound = math.MaxInt64
}
// Track every chunk-id seen so far (known and unknown). Canonical
// Git's read_table_of_contents scans cf->chunks[0..chunks_nr-1] for
// the incoming id and returns -1 on the first match [2]. Use a map
// instead of a linear scan; the semantics are identical.
seen := make(map[[szChunkSig]byte]struct{}, int(fi.numChunks))
// assigned records, in file order, every chunk whose offset was stored in
// fi.offsets. After the terminator offset is known, a second pass derives
// fi.sizes from consecutive offset differences.
assigned := make([]chunkAssignment, 0, int(fi.numChunks))
for i := range int(fi.numChunks) {
entry := io.NewSectionReader(fi.reader, tocBase+int64(i)*tocEntrySize, tocEntrySize)
if _, err := io.ReadAtLeast(entry, chunkID, szChunkSig); err != nil {
return err
}
chunkOffset, err := binary.ReadUint64(entry)
if err != nil {
return err
}
// Validate the offset before classifying the chunk-id, mirroring
// canonical Git's read_table_of_contents which checks the offset
// against the previous one and the upper bound before any
// per-chunk dispatch.
if int64(chunkOffset) > upperBound || int64(chunkOffset) < prevOffset {
return ErrMalformedCommitGraphFile
}
prevOffset = int64(chunkOffset)
// Reject duplicate chunk-ids (known and unknown alike), matching
// canonical Git's "duplicate chunk ID" check [2].
var id [szChunkSig]byte
copy(id[:], chunkID)
if _, ok := seen[id]; ok {
return ErrMalformedCommitGraphFile
}
seen[id] = struct{}{}
chunkType, ok := ChunkTypeFromBytes(chunkID)
if !ok {
continue
}
// A zero chunk-id inside the declared count is the same condition
// canonical Git reports as "terminating chunk id appears earlier
// than expected".
if chunkType == ZeroChunk {
return ErrMalformedCommitGraphFile
}
if int(chunkType) >= len(fi.offsets) {
continue
}
fi.offsets[chunkType] = int64(chunkOffset)
assigned = append(assigned, chunkAssignment{ct: chunkType, offset: int64(chunkOffset)})
}
// The table is followed by a single terminator entry whose chunk-id
// is zero. Reading anything else means the declared count does not
// match the table contents.
terminator := io.NewSectionReader(fi.reader, tocBase+int64(fi.numChunks)*tocEntrySize, tocEntrySize)
if _, err := io.ReadAtLeast(terminator, chunkID, szChunkSig); err != nil {
return err
}
if !bytes.Equal(chunkID, ZeroChunk.Signature()) {
return ErrMalformedCommitGraphFile
}
// The terminator entry's offset marks the end of all chunk data. Use it
// together with the per-chunk start offsets to derive chunk byte lengths.
terminatorOffset, err := binary.ReadUint64(terminator)
if err != nil {
return err
}
for i, a := range assigned {
var end int64
if i+1 < len(assigned) {
end = assigned[i+1].offset
} else {
end = int64(terminatorOffset)
}
fi.sizes[a.ct] = end - a.offset
}
if fi.offsets[OIDFanoutChunk] <= 0 || fi.offsets[OIDLookupChunk] <= 0 || fi.offsets[CommitDataChunk] <= 0 {
return ErrMalformedCommitGraphFile
}
return nil
}
// verifyChunkSizes asserts the byte length of every required chunk
// against the fanout-derived commit count. Canonical Git applies the
// same cardinality checks at parse time so that truncated or
// hand-edited files fail once during OpenFileIndex rather than mid-
// walk (commit-graph.c v2.54.0, graph_read_oid_fanout [1],
// graph_read_oid_lookup [2], graph_read_commit_data [3], and
// graph_read_generation_data [4]).
//
// numCommits is fanout[255]; reading the single uint32 at the end of
// the fanout chunk avoids depending on readFanout's later pass.
//
// [1]: https://github.com/git/git/blob/v2.54.0/commit-graph.c#L288
// [2]: https://github.com/git/git/blob/v2.54.0/commit-graph.c#L311
// [3]: https://github.com/git/git/blob/v2.54.0/commit-graph.c#L320
// [4]: https://github.com/git/git/blob/v2.54.0/commit-graph.c#L330
func (fi *fileIndex) verifyChunkSizes() error {
if fi.sizes[OIDFanoutChunk] != lenFanout*szUint32 {
return ErrMalformedCommitGraphFile
}
var buf [szUint32]byte
off := fi.offsets[OIDFanoutChunk] + (lenFanout-1)*szUint32
if _, err := fi.reader.ReadAt(buf[:], off); err != nil {
return err
}
numCommits := int64(encbin.BigEndian.Uint32(buf[:]))
if numCommits > 0x7fffffff {
return ErrMalformedCommitGraphFile
}
hashSize := int64(fi.objSize)
if fi.sizes[OIDLookupChunk] != numCommits*hashSize {
return ErrMalformedCommitGraphFile
}
if fi.sizes[CommitDataChunk] != numCommits*(hashSize+szCommitData) {
return ErrMalformedCommitGraphFile
}
if fi.offsets[GenerationDataChunk] > 0 &&
fi.sizes[GenerationDataChunk] != numCommits*szUint32 {
return ErrMalformedCommitGraphFile
}
return nil
}
func (fi *fileIndex) readFanout() error {
// The Fanout table is a 256 entry table of the number (as uint32) of OIDs with first byte at most i.
// Thus F[255] stores the total number of commits (N)
fanoutReader := io.NewSectionReader(fi.reader, fi.offsets[OIDFanoutChunk], lenFanout*szUint32)
for i := range 256 {
fanoutValue, err := binary.ReadUint32(fanoutReader)
if err != nil {
return err
}
if fanoutValue > 0x7fffffff {
return ErrMalformedCommitGraphFile
}
fi.fanout[i] = fanoutValue
}
return nil
}
// GetIndexByHash looks up the provided hash in the commit-graph fanout and returns the index of the commit data for the given hash.
func (fi *fileIndex) GetIndexByHash(h plumbing.Hash) (uint32, error) {
var oid plumbing.Hash
// Find the hash in the oid lookup table
var low uint32
if h.Bytes()[0] == 0 {
low = 0
} else {
low = fi.fanout[h.Bytes()[0]-1]
}
high := fi.fanout[h.Bytes()[0]]
for low < high {
mid := (low + high) >> 1
offset := fi.offsets[OIDLookupChunk] + int64(mid)*int64(fi.objSize)
if _, err := oid.ReadFrom(io.NewSectionReader(fi.reader, offset, int64(oid.Size()))); err != nil {
return 0, err
}
cmp := h.Compare(oid.Bytes())
switch {
case cmp < 0:
high = mid
case cmp == 0:
return mid + fi.minimumNumberOfHashes, nil
default:
low = mid + 1
}
}
if fi.parent != nil {
idx, err := fi.parent.GetIndexByHash(h)
if err != nil {
return 0, err
}
return idx, nil
}
return 0, plumbing.ErrObjectNotFound
}
// GetCommitDataByIndex returns the commit data for the given index in the commit-graph.
func (fi *fileIndex) GetCommitDataByIndex(idx uint32) (*CommitData, error) {
if idx < fi.minimumNumberOfHashes {
if fi.parent != nil {
data, err := fi.parent.GetCommitDataByIndex(idx)
if err != nil {
return nil, err
}
return data, nil
}
return nil, plumbing.ErrObjectNotFound
}
idx -= fi.minimumNumberOfHashes
if idx >= fi.fanout[0xff] {
return nil, plumbing.ErrObjectNotFound
}
offset := fi.offsets[CommitDataChunk] + int64(idx)*int64(fi.objSize+szCommitData)
commitDataReader := io.NewSectionReader(fi.reader, offset, int64(fi.objSize+szCommitData))
// TODO: Add support for SHA256
var treeHash plumbing.Hash
_, err := treeHash.ReadFrom(commitDataReader)
if err != nil {
return nil, err
}
parent1, err := binary.ReadUint32(commitDataReader)
if err != nil {
return nil, err
}
parent2, err := binary.ReadUint32(commitDataReader)
if err != nil {
return nil, err
}
genAndTime, err := binary.ReadUint64(commitDataReader)
if err != nil {
return nil, err
}
var parentIndexes []uint32
switch {
case parent2&parentOctopusUsed == parentOctopusUsed:
// Octopus merge — look up extra parents from the EDGE chunk. Canonical
// Git's fill_commit_in_graph bounds parent_data_pos against
// chunk_extra_edges_size / sizeof(uint32_t) on every iteration
// (commit-graph.c v2.54.0); mirror that to refuse out-of-range
// pointers and sentinel-less runaway reads.
edgeCount := fi.sizes[ExtraEdgeListChunk] / szUint32
pos := int64(parent2 & parentOctopusMask)
if pos >= edgeCount {
return nil, ErrMalformedCommitGraphFile
}
parentIndexes = []uint32{parent1 & parentOctopusMask}
offset := fi.offsets[ExtraEdgeListChunk] + szUint32*pos
buf := make([]byte, szUint32)
for {
if pos >= edgeCount {
return nil, ErrMalformedCommitGraphFile
}
_, err := fi.reader.ReadAt(buf, offset)
if err != nil {
return nil, err
}
parent := encbin.BigEndian.Uint32(buf)
offset += szUint32
pos++
parentIndexes = append(parentIndexes, parent&parentOctopusMask)
if parent&parentLast == parentLast {
break
}
}
case parent2 != parentNone:
parentIndexes = []uint32{parent1 & parentOctopusMask, parent2 & parentOctopusMask}
case parent1 != parentNone:
parentIndexes = []uint32{parent1 & parentOctopusMask}
}
parentHashes, err := fi.getHashesFromIndexes(parentIndexes)
if err != nil {
return nil, err
}
generationV2 := uint64(0)
if fi.hasGenerationV2 {
// set the GenerationV2 result to the commit time
generationV2 = uint64(genAndTime & 0x3FFFFFFFF)
// Next read the generation (offset) data from the generation data chunk
offset := fi.offsets[GenerationDataChunk] + int64(idx)*szUint32
buf := make([]byte, szUint32)
if _, err := fi.reader.ReadAt(buf, offset); err != nil {
return nil, err
}
genV2Data := encbin.BigEndian.Uint32(buf)
// check if the data is an overflow that needs to be looked up in the overflow chunk
if genV2Data&0x80000000 > 0 {
// Overflow — look up the corrected commit date from the GDO2
// chunk. Canonical Git's fill_commit_graph_info refuses an
// offset_pos that falls past
// chunk_generation_data_overflow_size / sizeof(uint64_t)
// (commit-graph.c v2.54.0); mirror that to keep an
// out-of-range pointer from reading adjacent chunk bytes
// or past EOF.
pos := int64(genV2Data & 0x7fffffff)
overflowCount := fi.sizes[GenerationDataOverflowChunk] / szUint64
if pos >= overflowCount {
return nil, ErrMalformedCommitGraphFile
}
offset := fi.offsets[GenerationDataOverflowChunk] + pos*szUint64
buf := make([]byte, 8)
if _, err := fi.reader.ReadAt(buf, offset); err != nil {
return nil, err
}
generationV2 += encbin.BigEndian.Uint64(buf)
} else {
generationV2 += uint64(genV2Data)
}
}
return &CommitData{
TreeHash: treeHash,
ParentIndexes: parentIndexes,
ParentHashes: parentHashes,
Generation: genAndTime >> 34,
GenerationV2: generationV2,
When: time.Unix(int64(genAndTime&0x3FFFFFFFF), 0),
}, nil
}
// GetHashByIndex looks up the hash for the given index in the commit-graph.
func (fi *fileIndex) GetHashByIndex(idx uint32) (found plumbing.Hash, err error) {
if idx < fi.minimumNumberOfHashes {
if fi.parent != nil {
return fi.parent.GetHashByIndex(idx)
}
return found, ErrMalformedCommitGraphFile
}
idx -= fi.minimumNumberOfHashes
if idx >= fi.fanout[0xff] {
return found, ErrMalformedCommitGraphFile
}
offset := fi.offsets[OIDLookupChunk] + int64(idx)*int64(fi.objSize)
if _, err := found.ReadFrom(io.NewSectionReader(fi.reader, offset, int64(found.Size()))); err != nil {
return found, err
}
return found, nil
}
func (fi *fileIndex) getHashesFromIndexes(indexes []uint32) ([]plumbing.Hash, error) {
hashes := make([]plumbing.Hash, len(indexes))
for i, idx := range indexes {
if idx < fi.minimumNumberOfHashes {
if fi.parent != nil {
hash, err := fi.parent.GetHashByIndex(idx)
if err != nil {
return nil, err
}
hashes[i] = hash
continue
}
return nil, ErrMalformedCommitGraphFile
}
idx -= fi.minimumNumberOfHashes
if idx >= fi.fanout[0xff] {
return nil, ErrMalformedCommitGraphFile
}
offset := fi.offsets[OIDLookupChunk] + int64(idx)*int64(fi.objSize)
if _, err := hashes[i].ReadFrom(io.NewSectionReader(fi.reader, offset, int64(hashes[i].Size()))); err != nil {
return nil, err
}
}
return hashes, nil
}
// Hashes returns all the hashes that are available in the index.
func (fi *fileIndex) Hashes() []plumbing.Hash {
hashes := make([]plumbing.Hash, fi.fanout[0xff]+fi.minimumNumberOfHashes)
for i := uint32(0); i < fi.minimumNumberOfHashes; i++ {
hash, err := fi.parent.GetHashByIndex(i)
if err != nil {
return nil
}
hashes[i] = hash
}
for i := uint32(0); i < fi.fanout[0xff]; i++ {
h := &hashes[i+fi.minimumNumberOfHashes]
offset := fi.offsets[OIDLookupChunk] + int64(i)*int64(h.Size())
n, err := h.ReadFrom(io.NewSectionReader(fi.reader, offset, int64(h.Size())))
if err != nil || n < int64(h.Size()) {
return nil
}
}
return hashes
}
func (fi *fileIndex) HasGenerationV2() bool {
return fi.hasGenerationV2
}
func (fi *fileIndex) MaximumNumberOfHashes() uint32 {
return fi.minimumNumberOfHashes + fi.fanout[0xff]
}
package commitgraph
import (
"math"
"github.com/go-git/go-git/v6/plumbing"
)
// MemoryIndex provides a way to build the commit-graph in memory
// for later encoding to file.
type MemoryIndex struct {
commitData []commitData
indexMap map[plumbing.Hash]uint32
hasGenerationV2 bool
}
type commitData struct {
Hash plumbing.Hash
*CommitData
}
// NewMemoryIndex creates in-memory commit graph representation
func NewMemoryIndex() *MemoryIndex {
return &MemoryIndex{
indexMap: make(map[plumbing.Hash]uint32),
hasGenerationV2: true,
}
}
// GetIndexByHash gets the index in the commit graph from commit hash, if available
func (mi *MemoryIndex) GetIndexByHash(h plumbing.Hash) (uint32, error) {
i, ok := mi.indexMap[h]
if ok {
return i, nil
}
return 0, plumbing.ErrObjectNotFound
}
// GetHashByIndex gets the hash given an index in the commit graph
func (mi *MemoryIndex) GetHashByIndex(i uint32) (plumbing.Hash, error) {
if i >= uint32(len(mi.commitData)) {
return plumbing.ZeroHash, plumbing.ErrObjectNotFound
}
return mi.commitData[i].Hash, nil
}
// GetCommitDataByIndex gets the commit node from the commit graph using index
// obtained from child node, if available
func (mi *MemoryIndex) GetCommitDataByIndex(i uint32) (*CommitData, error) {
if i >= uint32(len(mi.commitData)) {
return nil, plumbing.ErrObjectNotFound
}
commitData := mi.commitData[i]
// Map parent hashes to parent indexes
if commitData.ParentIndexes == nil {
parentIndexes := make([]uint32, len(commitData.ParentHashes))
for i, parentHash := range commitData.ParentHashes {
var err error
if parentIndexes[i], err = mi.GetIndexByHash(parentHash); err != nil {
return nil, err
}
}
commitData.ParentIndexes = parentIndexes
}
return commitData.CommitData, nil
}
// Hashes returns all the hashes that are available in the index
func (mi *MemoryIndex) Hashes() []plumbing.Hash {
hashes := make([]plumbing.Hash, 0, len(mi.indexMap))
for k := range mi.indexMap {
hashes = append(hashes, k)
}
return hashes
}
// Add adds new node to the memory index
func (mi *MemoryIndex) Add(hash plumbing.Hash, data *CommitData) {
// The parent indexes are calculated lazily in GetNodeByIndex
// which allows adding nodes out of order as long as all parents
// are eventually resolved
data.ParentIndexes = nil
mi.indexMap[hash] = uint32(len(mi.commitData))
mi.commitData = append(mi.commitData, commitData{Hash: hash, CommitData: data})
if data.GenerationV2 == math.MaxUint64 { // if GenerationV2 is not available reset it to zero
data.GenerationV2 = 0
}
mi.hasGenerationV2 = mi.hasGenerationV2 && data.GenerationV2 != 0
}
// HasGenerationV2 returns true if the index has generation v2 data.
func (mi *MemoryIndex) HasGenerationV2() bool {
return mi.hasGenerationV2
}
// Close closes the index
func (mi *MemoryIndex) Close() error {
return nil
}
// MaximumNumberOfHashes returns the maximum number of hashes in the index.
func (mi *MemoryIndex) MaximumNumberOfHashes() uint32 {
return uint32(len(mi.indexMap))
}
package config
import "slices"
// New creates a new config instance.
func New() *Config {
return &Config{}
}
// Config contains all the sections, comments and includes from a config file.
type Config struct {
Comment *Comment
Sections Sections
Includes Includes
}
// Includes is a list of Includes in a config file.
type Includes []*Include
// Include is a reference to an included config file.
type Include struct {
Path string
Config *Config
}
// Comment string without the prefix '#' or ';'.
type Comment string
const (
// NoSubsection token is passed to Config.Section and Config.SetSection to
// represent the absence of a section.
NoSubsection = ""
)
// Section returns a existing section with the given name or creates a new one.
func (c *Config) Section(name string) *Section {
for _, s := range slices.Backward(c.Sections) {
if s.IsName(name) {
return s
}
}
s := &Section{Name: name}
c.Sections = append(c.Sections, s)
return s
}
// HasSection checks if the Config has a section with the specified name.
func (c *Config) HasSection(name string) bool {
for _, s := range c.Sections {
if s.IsName(name) {
return true
}
}
return false
}
// RemoveSection removes a section from a config file.
func (c *Config) RemoveSection(name string) *Config {
result := Sections{}
for _, s := range c.Sections {
if !s.IsName(name) {
result = append(result, s)
}
}
c.Sections = result
return c
}
// RemoveSubsection remove a subsection from a config file.
func (c *Config) RemoveSubsection(section, subsection string) *Config {
for _, s := range c.Sections {
if s.IsName(section) {
result := Subsections{}
for _, ss := range s.Subsections {
if !ss.IsName(subsection) {
result = append(result, ss)
}
}
s.Subsections = result
}
}
return c
}
// AddOption adds an option to a given section and subsection. Use the
// NoSubsection constant for the subsection argument if no subsection is wanted.
func (c *Config) AddOption(section, subsection, key, value string) *Config {
if subsection == "" {
c.Section(section).AddOption(key, value)
} else {
c.Section(section).Subsection(subsection).AddOption(key, value)
}
return c
}
// SetOption sets an option to a given section and subsection. Use the
// NoSubsection constant for the subsection argument if no subsection is wanted.
func (c *Config) SetOption(section, subsection, key, value string) *Config {
if subsection == "" {
c.Section(section).SetOption(key, value)
} else {
c.Section(section).Subsection(subsection).SetOption(key, value)
}
return c
}
package config
import (
"io"
"github.com/go-git/gcfg/v2"
)
// A Decoder reads and decodes config files from an input stream.
type Decoder struct {
io.Reader
}
// NewDecoder returns a new decoder that reads from r.
func NewDecoder(r io.Reader) *Decoder {
return &Decoder{r}
}
// Decode reads the whole config from its input and stores it in the
// value pointed to by config.
func (d *Decoder) Decode(config *Config) error {
cb := func(s, ss, k, v string, _ bool) error {
if ss == "" && k == "" {
config.Section(s)
return nil
}
if ss != "" && k == "" {
config.Section(s).Subsection(ss)
return nil
}
config.AddOption(s, ss, k, v)
return nil
}
return gcfg.ReadWithCallback(d, cb)
}
package config
import (
"fmt"
"io"
"strings"
)
// An Encoder writes config files to an output stream.
type Encoder struct {
w io.Writer
}
var (
subsectionReplacer = strings.NewReplacer(`"`, `\"`, `\`, `\\`)
valueReplacer = strings.NewReplacer(`"`, `\"`, `\`, `\\`, "\n", `\n`, "\t", `\t`, "\b", `\b`)
)
// NewEncoder returns a new encoder that writes to w.
func NewEncoder(w io.Writer) *Encoder {
return &Encoder{w}
}
// Encode writes the config in git config format to the stream of the encoder.
func (e *Encoder) Encode(cfg *Config) error {
for _, s := range cfg.Sections {
if err := e.encodeSection(s); err != nil {
return err
}
}
return nil
}
func (e *Encoder) encodeSection(s *Section) error {
if len(s.Options) > 0 {
if err := e.printf("[%s]\n", s.Name); err != nil {
return err
}
if err := e.encodeOptions(s.Options); err != nil {
return err
}
}
for _, ss := range s.Subsections {
if err := e.encodeSubsection(s.Name, ss); err != nil {
return err
}
}
return nil
}
func (e *Encoder) encodeSubsection(sectionName string, s *Subsection) error {
if err := e.printf("[%s \"%s\"]\n", sectionName, subsectionReplacer.Replace(s.Name)); err != nil {
return err
}
return e.encodeOptions(s.Options)
}
func (e *Encoder) encodeOptions(opts Options) error {
for _, o := range opts {
var value string
if strings.ContainsAny(o.Value, "#;\"\t\n\\") || strings.HasPrefix(o.Value, " ") || strings.HasSuffix(o.Value, " ") {
value = `"` + valueReplacer.Replace(o.Value) + `"`
} else {
value = o.Value
}
if err := e.printf("\t%s = %s\n", o.Key, value); err != nil {
return err
}
}
return nil
}
func (e *Encoder) printf(msg string, args ...any) error {
_, err := fmt.Fprintf(e.w, msg, args...)
return err
}
package config
import "errors"
// RepositoryFormatVersion represents the repository format version,
// as per defined at:
//
// https://git-scm.com/docs/repository-version
type RepositoryFormatVersion string
const (
// Version0 is the format defined by the initial version of git,
// including but not limited to the format of the repository
// directory, the repository configuration file, and the object
// and ref storage.
//
// Specifying the complete behavior of git is beyond the scope
// of this document.
Version0 = "0"
// Version1 is identical to version 0, with the following exceptions:
//
// 1. When reading the core.repositoryformatversion variable, a git
// implementation which supports version 1 MUST also read any
// configuration keys found in the extensions section of the
// configuration file.
//
// 2. If a version-1 repository specifies any extensions.* keys that
// the running git has not implemented, the operation MUST NOT proceed.
// Similarly, if the value of any known key is not understood by the
// implementation, the operation MUST NOT proceed.
//
// Note that if no extensions are specified in the config file, then
// core.repositoryformatversion SHOULD be set to 0 (setting it to 1 provides
// no benefit, and makes the repository incompatible with older
// implementations of git).
Version1 = "1"
// DefaultRepositoryFormatVersion holds the default repository format version.
DefaultRepositoryFormatVersion = Version0
)
// ObjectFormat defines the object format.
type ObjectFormat string
const (
// UnsetObjectFormat indicates no object format has been set.
UnsetObjectFormat ObjectFormat = ""
// SHA1 represents the object format used for SHA1.
SHA1 ObjectFormat = "sha1"
// SHA256 represents the object format used for SHA256.
SHA256 ObjectFormat = "sha256"
// DefaultObjectFormat holds the default object format.
DefaultObjectFormat = SHA1
)
// String returns the string representation of the ObjectFormat.
func (f ObjectFormat) String() string {
return string(f)
}
// Size returns the hash size of the ObjectFormat.
func (f ObjectFormat) Size() int {
switch f {
case SHA1:
return SHA1Size
case SHA256:
return SHA256Size
default:
return DefaultObjectFormat.Size()
}
}
// HexSize returns the hash size in hexadecimal format of the ObjectFormat.
func (f ObjectFormat) HexSize() int {
switch f {
case SHA1:
return SHA1HexSize
case SHA256:
return SHA256HexSize
default:
return DefaultObjectFormat.HexSize()
}
}
// ErrInvalidObjectFormat is returned when an invalid ObjectFormat is used.
var ErrInvalidObjectFormat = errors.New("invalid object format")
const (
// SHA1Size is the size of SHA1 hash.
SHA1Size = 20
// SHA256Size is the size of SHA256 hash.
SHA256Size = 32
// SHA1HexSize is the size of SHA1 hash in hexadecimal format.
SHA1HexSize = SHA1Size * 2
// SHA256HexSize is the size of SHA256 hash in hexadecimal format.
SHA256HexSize = SHA256Size * 2
)
package config
import (
"fmt"
"slices"
"strings"
)
// Option defines a key/value entity in a config file.
type Option struct {
// Key preserving original caseness.
// Use IsKey instead to compare key regardless of caseness.
Key string
// Original value as string, could be not normalized.
Value string
}
// Options is a collection of Option.
type Options []*Option
// IsKey returns true if the given key matches
// this option's key in a case-insensitive comparison.
func (o *Option) IsKey(key string) bool {
return strings.EqualFold(o.Key, key)
}
// GoString returns a Go-syntax representation of Options.
func (opts Options) GoString() string {
strs := make([]string, 0, len(opts))
for _, opt := range opts {
strs = append(strs, fmt.Sprintf("%#v", opt))
}
return strings.Join(strs, ", ")
}
// Get gets the value for the given key if set,
// otherwise it returns the empty string.
//
// # Note that there is no difference
//
// This matches git behaviour since git v1.8.1-rc1,
// if there are multiple definitions of a key, the
// last one wins.
//
// See: http://article.gmane.org/gmane.linux.kernel/1407184
//
// In order to get all possible values for the same key,
// use GetAll.
func (opts Options) Get(key string) string {
for _, o := range slices.Backward(opts) {
if o.IsKey(key) {
return o.Value
}
}
return ""
}
// Has checks if an Option exist with the given key.
func (opts Options) Has(key string) bool {
for _, o := range opts {
if o.IsKey(key) {
return true
}
}
return false
}
// GetAll returns all possible values for the same key.
func (opts Options) GetAll(key string) []string {
result := []string{}
for _, o := range opts {
if o.IsKey(key) {
result = append(result, o.Value)
}
}
return result
}
func (opts Options) withoutOption(key string) Options {
result := Options{}
for _, o := range opts {
if !o.IsKey(key) {
result = append(result, o)
}
}
return result
}
func (opts Options) withAddedOption(key, value string) Options {
return append(opts, &Option{key, value})
}
func (opts Options) withSettedOption(key string, values ...string) Options {
var result Options
var added []string
for _, o := range opts {
if !o.IsKey(key) {
result = append(result, o)
continue
}
if slices.Contains(values, o.Value) {
added = append(added, o.Value)
result = append(result, o)
continue
}
}
for _, value := range values {
if slices.Contains(added, value) {
continue
}
result = result.withAddedOption(key, value)
}
return result
}
package config
import (
"fmt"
"slices"
"strings"
)
// Section is the representation of a section inside git configuration files.
// Each Section contains Options that are used by both the Git plumbing
// and the porcelains.
// Sections can be further divided into subsections. To begin a subsection
// put its name in double quotes, separated by space from the section name,
// in the section header, like in the example below:
//
// [section "subsection"]
//
// All the other lines (and the remainder of the line after the section header)
// are recognized as option variables, in the form "name = value" (or just name,
// which is a short-hand to say that the variable is the boolean "true").
// The variable names are case-insensitive, allow only alphanumeric characters
// and -, and must start with an alphabetic character:
//
// [section "subsection1"]
// option1 = value1
// option2
// [section "subsection2"]
// option3 = value2
type Section struct {
Name string
Options Options
Subsections Subsections
}
// Subsection is a subsection of a Section.
type Subsection struct {
Name string
Options Options
}
// Sections is a collection of Section.
type Sections []*Section
// GoString returns a Go-syntax representation of Sections.
func (s Sections) GoString() string {
strs := make([]string, 0, len(s))
for _, ss := range s {
strs = append(strs, fmt.Sprintf("%#v", ss))
}
return strings.Join(strs, ", ")
}
// Subsections is a collection of Subsection.
type Subsections []*Subsection
// GoString returns a Go-syntax representation of Subsections.
func (s Subsections) GoString() string {
strs := make([]string, 0, len(s))
for _, ss := range s {
strs = append(strs, fmt.Sprintf("%#v", ss))
}
return strings.Join(strs, ", ")
}
// IsName checks if the name provided is equals to the Section name, case insensitive.
func (s *Section) IsName(name string) bool {
return strings.EqualFold(s.Name, name)
}
// Subsection returns a Subsection from the specified Section. If the
// Subsection does not exists, new one is created and added to Section.
func (s *Section) Subsection(name string) *Subsection {
for _, ss := range slices.Backward(s.Subsections) {
if ss.IsName(name) {
return ss
}
}
ss := &Subsection{Name: name}
s.Subsections = append(s.Subsections, ss)
return ss
}
// HasSubsection checks if the Section has a Subsection with the specified name.
func (s *Section) HasSubsection(name string) bool {
for _, ss := range s.Subsections {
if ss.IsName(name) {
return true
}
}
return false
}
// RemoveSubsection removes a subsection from a Section.
func (s *Section) RemoveSubsection(name string) *Section {
result := Subsections{}
for _, s := range s.Subsections {
if !s.IsName(name) {
result = append(result, s)
}
}
s.Subsections = result
return s
}
// Option returns the value for the specified key. Empty string is returned if
// key does not exists.
func (s *Section) Option(key string) string {
return s.Options.Get(key)
}
// OptionAll returns all possible values for an option with the specified key.
// If the option does not exists, an empty slice will be returned.
func (s *Section) OptionAll(key string) []string {
return s.Options.GetAll(key)
}
// HasOption checks if the Section has an Option with the given key.
func (s *Section) HasOption(key string) bool {
return s.Options.Has(key)
}
// AddOption adds a new Option to the Section. The updated Section is returned.
func (s *Section) AddOption(key, value string) *Section {
s.Options = s.Options.withAddedOption(key, value)
return s
}
// SetOption adds a new Option to the Section. If the option already exists, is replaced.
// The updated Section is returned.
func (s *Section) SetOption(key, value string) *Section {
s.Options = s.Options.withSettedOption(key, value)
return s
}
// RemoveOption removes an option with the specified key. The updated Section is returned.
func (s *Section) RemoveOption(key string) *Section {
s.Options = s.Options.withoutOption(key)
return s
}
// IsName checks if the name of the subsection is exactly the specified name.
func (s *Subsection) IsName(name string) bool {
return s.Name == name
}
// Option returns an option with the specified key. If the option does not exists,
// empty spring will be returned.
func (s *Subsection) Option(key string) string {
return s.Options.Get(key)
}
// OptionAll returns all possible values for an option with the specified key.
// If the option does not exists, an empty slice will be returned.
func (s *Subsection) OptionAll(key string) []string {
return s.Options.GetAll(key)
}
// HasOption checks if the Subsection has an Option with the given key.
func (s *Subsection) HasOption(key string) bool {
return s.Options.Has(key)
}
// AddOption adds a new Option to the Subsection. The updated Subsection is returned.
func (s *Subsection) AddOption(key, value string) *Subsection {
s.Options = s.Options.withAddedOption(key, value)
return s
}
// SetOption adds a new Option to the Subsection. If the option already exists, is replaced.
// The updated Subsection is returned.
func (s *Subsection) SetOption(key string, value ...string) *Subsection {
s.Options = s.Options.withSettedOption(key, value...)
return s
}
// RemoveOption removes the option with the specified key. The updated Subsection is returned.
func (s *Subsection) RemoveOption(key string) *Subsection {
s.Options = s.Options.withoutOption(key)
return s
}
package gitignore
import (
"bufio"
"bytes"
"io"
"os"
"strings"
"github.com/go-git/go-billy/v6"
"github.com/go-git/go-git/v6/internal/pathutil"
"github.com/go-git/go-git/v6/plumbing/format/config"
gioutil "github.com/go-git/go-git/v6/utils/ioutil"
)
// IgnoreFile is the name of the per-directory ignore file. A walk driven by a
// Scope needs it to tell, from a directory listing it already holds, whether
// that directory declares patterns of its own.
const IgnoreFile = ".gitignore"
const (
commentPrefix = "#"
coreSection = "core"
excludesfile = "excludesfile"
gitDir = ".git"
gitignoreFile = IgnoreFile
gitconfigFile = ".gitconfig"
systemFile = "/etc/gitconfig"
infoExcludeFile = gitDir + "/info/exclude"
)
// readIgnoreFile reads a specific git ignore file.
func readIgnoreFile(fs billy.Filesystem, path []string, ignoreFile string) (ps []Pattern, err error) {
ignoreFile, _ = pathutil.ReplaceTildeWithHome(ignoreFile)
f, err := fs.Open(fs.Join(append(path, ignoreFile)...))
if err == nil {
defer func() { _ = f.Close() }()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
s := scanner.Text()
if !strings.HasPrefix(s, commentPrefix) && len(strings.TrimSpace(s)) > 0 {
ps = append(ps, ParsePattern(s, path))
}
}
} else if !os.IsNotExist(err) {
return nil, err
}
return ps, err
}
// ReadPatterns reads the .git/info/exclude and then the gitignore patterns
// recursively traversing through the directory structure. The result is in
// the ascending order of priority (last higher).
//
// .git/info/exclude is only consulted at the root of the given filesystem,
// matching reference git which reads $GIT_DIR/info/exclude of the
// repository being walked. Ignore files are opened only when present in
// the directory listing, so directories without them cost a single ReadDir.
//
// Deprecated: use Scope, which evaluates rules per directory as a walk
// descends. The flat list returned here cannot express that a parent
// directory is excluded, so a Matcher built from it may re-include a path
// that git reports as ignored, and rules below an excluded directory are
// collected even though reference git never reads them.
func ReadPatterns(fs billy.Filesystem, path []string) (ps []Pattern, err error) {
fis, err := fs.ReadDir(fs.Join(path...))
if err != nil {
return nil, err
}
var hasGitDir, hasGitignore bool
for _, fi := range fis {
switch fi.Name() {
case gitDir:
hasGitDir = true
case gitignoreFile:
hasGitignore = true
}
}
if len(path) == 0 && hasGitDir {
ps, _ = readIgnoreFile(fs, path, infoExcludeFile)
}
if hasGitignore {
subps, _ := readIgnoreFile(fs, path, gitignoreFile)
ps = append(ps, subps...)
}
for _, fi := range fis {
if fi.IsDir() && fi.Name() != gitDir {
if NewMatcher(ps).Match(append(path, fi.Name()), true) {
continue
}
var subps []Pattern
subps, err = ReadPatterns(fs, append(path, fi.Name()))
if err != nil {
return ps, err
}
if len(subps) > 0 {
ps = append(ps, subps...)
}
}
}
return ps, err
}
func loadPatterns(fs billy.Filesystem, path string) (ps []Pattern, err error) {
f, err := fs.Open(path)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
defer gioutil.CheckClose(f, &err)
b, err := io.ReadAll(f)
if err != nil {
return ps, err
}
d := config.NewDecoder(bytes.NewBuffer(b))
raw := config.New()
if err = d.Decode(raw); err != nil {
return ps, err
}
s := raw.Section(coreSection)
efo := s.Options.Get(excludesfile)
if efo == "" {
return nil, nil
}
ps, err = readIgnoreFile(fs, nil, efo)
if os.IsNotExist(err) {
return nil, nil
}
return ps, err
}
// LoadGlobalPatterns loads gitignore patterns from the gitignore file
// declared in a user's ~/.gitconfig file. If the ~/.gitconfig file does not
// exist the function will return nil. If the core.excludesfile property
// is not declared, the function will return nil. If the file pointed to by
// the core.excludesfile property does not exist, the function will return nil.
//
// The function assumes fs is rooted at the root filesystem.
func LoadGlobalPatterns(fs billy.Filesystem) (ps []Pattern, err error) {
home, err := os.UserHomeDir()
if err != nil {
return ps, err
}
return loadPatterns(fs, fs.Join(home, gitconfigFile))
}
// LoadSystemPatterns loads gitignore patterns from the gitignore file
// declared in a system's /etc/gitconfig file. If the /etc/gitconfig file does
// not exist the function will return nil. If the core.excludesfile property
// is not declared, the function will return nil. If the file pointed to by
// the core.excludesfile property does not exist, the function will return nil.
//
// The function assumes fs is rooted at the root filesystem.
func LoadSystemPatterns(fs billy.Filesystem) (ps []Pattern, err error) {
return loadPatterns(fs, systemFile)
}
package gitignore
// Matcher defines a global multi-pattern matcher for gitignore patterns.
type Matcher interface {
// Match reports whether path is excluded by the highest-priority matching
// pattern. Path is an ordered sequence of logical path components. Patterns
// created with ParsePattern match only paths beginning with their domain.
// isDir reports whether the final path component is a directory. For a
// pattern ending in "/", isDir only restricts a match at the candidate
// endpoint; descendants of a matched directory may still match.
Match(path []string, isDir bool) bool
}
// NewMatcher constructs a new global matcher from patterns in increasing
// priority order. Match evaluates them from last to first and uses the first
// Exclude or Include result. Generic settings files should come first, followed
// by the repository .gitignore, .gitignore files in successively deeper
// directories, and command-line arguments.
func NewMatcher(ps []Pattern) Matcher {
return &matcher{ps}
}
type matcher struct {
patterns []Pattern
}
func (m *matcher) Match(path []string, isDir bool) bool {
n := len(m.patterns)
for i := n - 1; i >= 0; i-- {
if match := m.patterns[i].Match(path, isDir); match > NoMatch {
return match == Exclude
}
}
return false
}
package gitignore
import (
"strings"
)
// MatchResult defines outcomes of a match, no match, exclusion or inclusion.
type MatchResult int
const (
// NoMatch defines the no match outcome of a match check
NoMatch MatchResult = iota
// Exclude defines an exclusion of a file as a result of a match check
Exclude
// Include defines an explicit inclusion of a file as a result of a match check
Include
)
const (
inclusionPrefix = "!"
zeroToManyDirs = "**"
patternDirSep = "/"
)
// Pattern defines a single gitignore pattern.
type Pattern interface {
// Match reports how the pattern applies to path. Path is an ordered sequence
// of logical path components. Patterns created with ParsePattern match only
// paths beginning with their domain. isDir reports whether the final path
// component is a directory. For a pattern ending in "/", isDir only
// restricts a match at the candidate endpoint; descendants of a matched
// directory may still match.
Match(path []string, isDir bool) MatchResult
}
type pattern struct {
domain []string
pattern []string
inclusion bool
dirOnly bool
isGlob bool
}
// ParsePattern parses a gitignore pattern string into a Pattern. The domain is
// an ordered prefix of logical path components that scopes the pattern.
// Matching applies to the components after that prefix. A nil or empty domain
// applies the pattern without a prefix.
//
// ReadPatterns uses the path of the directory containing a .gitignore file as
// its domain. When the filesystem is rooted at a repository, that path is
// repository-relative.
func ParsePattern(p string, domain []string) Pattern {
// storing domain, copy it to ensure it isn't changed externally
domain = append([]string(nil), domain...)
res := pattern{domain: domain}
if strings.HasPrefix(p, inclusionPrefix) {
res.inclusion = true
p = p[1:]
}
if !strings.HasSuffix(p, "\\ ") {
p = strings.TrimRight(p, " ")
}
if strings.HasSuffix(p, patternDirSep) {
res.dirOnly = true
p = p[:len(p)-1]
}
if strings.Contains(p, patternDirSep) {
res.isGlob = true
}
res.pattern = strings.Split(p, patternDirSep)
return &res
}
func (p *pattern) Match(path []string, isDir bool) MatchResult {
if len(path) <= len(p.domain) {
return NoMatch
}
for i, e := range p.domain {
if path[i] != e {
return NoMatch
}
}
path = path[len(p.domain):]
if p.isGlob && !p.globMatch(path, isDir) {
return NoMatch
} else if !p.isGlob && !p.simpleNameMatch(path, isDir) {
return NoMatch
}
if p.inclusion {
return Include
}
return Exclude
}
// The wildmatch implementation below ports the matcher from canonical Git's
// wildmatch.c at tag v2.54.0[1]. The algorithm is preserved exactly; the Go
// shape trades C idioms (raw pointers, NUL-terminated strings, goto-based
// control flow) for string slicing, explicit bounds checks, and a regular
// switch. Returned codes match upstream so callers can prune recursion the
// same way.
//
// [1]: https://github.com/git/git/blob/v2.54.0/wildmatch.c
// wildmatch return codes mirror the WM_* constants from upstream wildmatch.h.
// wmAbortToStarStar lets a recursive call signal to its caller that it hit a
// '/' boundary while expanding a non-'**' star, so the outer '*' can prune
// further alternatives instead of re-trying them.
const (
wmMatch = 0
wmNoMatch = 1
wmAbortAll = -1
wmAbortToStarStar = -2
)
// wildmatch flags mirror the WM_* flag bits in upstream wildmatch.h. The
// current go-git API does not expose case-insensitive matching, and the
// matcher splits paths on '/' before dispatching here, so wmCasefold and
// wmPathname code paths are kept for upstream parity but never exercised by
// the public Match.
const (
wmCasefold = 1
wmPathname = 2
)
// wildmatch reports whether text matches the wildcard pattern. It is a thin
// wrapper over dowild; the gitignore matcher splits paths on '/' before
// dispatching, so dowild always operates on a single pattern/text segment
// with flags=0.
func wildmatch(pattern, text string) bool {
return dowild(pattern, text, 0) == wmMatch
}
// dowild walks pattern and text in lock-step, recursing at each '*' to try
// every text suffix and propagating wmMatch, wmNoMatch, wmAbortAll, or
// wmAbortToStarStar back up so callers can prune work the same way the
// upstream C implementation does (wildmatch.c#L59-L283).
func dowild(p, text string, flags int) int {
pi, ti := 0, 0
for pi < len(p) {
pCh := p[pi]
var tCh byte
atEndOfText := ti >= len(text)
if !atEndOfText {
tCh = text[ti]
}
if atEndOfText && pCh != '*' {
return wmAbortAll
}
if flags&wmCasefold != 0 && isASCIIUpper(tCh) {
tCh += 'a' - 'A'
}
if flags&wmCasefold != 0 && isASCIIUpper(pCh) {
pCh += 'a' - 'A'
}
switch pCh {
case '\\':
// Literal match with the following character. A trailing '\'
// has no character to escape; canonical Git reads NUL (the C
// string terminator) into p_ch and the default-case compare
// fails because t_ch can never be NUL (the surrounding check
// returned wmAbortAll when text was exhausted). We mirror that
// by returning wmNoMatch directly.
if pi+1 >= len(p) {
return wmNoMatch
}
pi++
pCh = p[pi]
if tCh != pCh {
return wmNoMatch
}
pi++
ti++
case '?':
// Match any character except '/'.
if flags&wmPathname != 0 && tCh == '/' {
return wmNoMatch
}
pi++
ti++
case '*':
pi++
var matchSlash bool
if pi < len(p) && p[pi] == '*' {
prevPi := pi
for pi < len(p) && p[pi] == '*' {
pi++
}
switch {
case flags&wmPathname == 0:
// Without WM_PATHNAME, '*' == '**'.
matchSlash = true
case (prevPi < 2 || p[prevPi-2] == '/') &&
(pi >= len(p) || p[pi] == '/' ||
(pi+1 < len(p) && p[pi] == '\\' && p[pi+1] == '/')):
// At a '/<**>/' boundary: optionally match the slash as
// nothing, recursing past it so that foo/<*><*>/bar
// matches both foo/bar and foo/a/bar.
if pi < len(p) && p[pi] == '/' &&
dowild(p[pi+1:], text[ti:], flags) == wmMatch {
return wmMatch
}
matchSlash = true
}
} else {
// Single '*': without WM_PATHNAME crosses '/'; with it,
// does not.
matchSlash = flags&wmPathname == 0
}
if pi >= len(p) {
// Trailing "**" matches everything; trailing "*" matches only
// when no '/' remains in text.
if !matchSlash && strings.IndexByte(text[ti:], '/') >= 0 {
return wmAbortToStarStar
}
return wmMatch
} else if !matchSlash && p[pi] == '/' {
// One '*' followed by '/' with WM_PATHNAME: advance text to
// the next '/' so the outer loop consumes it.
slash := strings.IndexByte(text[ti:], '/')
if slash < 0 {
return wmAbortAll
}
ti += slash
// Fall through to the outer-loop advance.
pi++
ti++
continue
}
for {
if ti >= len(text) {
return wmAbortAll
}
tCh = text[ti]
// Try to advance faster when '*' is followed by a literal.
// Everything before the next occurrence of that literal
// must belong to '*'. With matchSlash=false, stop at the
// first '/'.
if !isGlobSpecial(p[pi]) {
pCh = p[pi]
if flags&wmCasefold != 0 && isASCIIUpper(pCh) {
pCh += 'a' - 'A'
}
for ti < len(text) {
tCh = text[ti]
if !matchSlash && tCh == '/' {
break
}
if flags&wmCasefold != 0 && isASCIIUpper(tCh) {
tCh += 'a' - 'A'
}
if tCh == pCh {
break
}
ti++
}
if ti >= len(text) || tCh != pCh {
if matchSlash {
return wmAbortAll
}
return wmAbortToStarStar
}
}
matched := dowild(p[pi:], text[ti:], flags)
if matched != wmNoMatch {
if !matchSlash || matched != wmAbortToStarStar {
return matched
}
} else if !matchSlash && tCh == '/' {
return wmAbortToStarStar
}
ti++
}
case '[':
pi++
if pi >= len(p) {
return wmAbortAll
}
pCh = p[pi]
if pCh == '^' {
pCh = '!'
}
negated := pCh == '!'
if negated {
pi++
if pi >= len(p) {
return wmAbortAll
}
pCh = p[pi]
}
var prevCh byte
matched := false
// The C source uses a do/while loop terminating when p_ch == ']';
// each iteration ends with prev_ch = p_ch and p_ch = *++p. NUL
// from the C string is detected here with explicit pi bounds
// checks before every read.
for {
switch {
case pCh == '\\':
pi++
if pi >= len(p) {
return wmAbortAll
}
pCh = p[pi]
if tCh == pCh {
matched = true
}
case pCh == '-' && prevCh != 0 &&
pi+1 < len(p) && p[pi+1] != ']':
pi++
pCh = p[pi]
if pCh == '\\' {
pi++
if pi >= len(p) {
return wmAbortAll
}
pCh = p[pi]
}
if tCh <= pCh && tCh >= prevCh {
matched = true
} else if flags&wmCasefold != 0 && isASCIILower(tCh) {
tUpper := tCh - ('a' - 'A')
if tUpper <= pCh && tUpper >= prevCh {
matched = true
}
}
pCh = 0 // resets prev_ch for next iteration
case pCh == '[' && pi+1 < len(p) && p[pi+1] == ':':
// POSIX class [:name:]. Walk forward to the next ']';
// if it isn't preceded by ':' the construct is not a
// class, so rewind and treat the '[' as a literal.
s := pi + 2
pi = s
for pi < len(p) && p[pi] != ']' {
pi++
}
if pi >= len(p) {
return wmAbortAll
}
nameLen := pi - s - 1
if nameLen < 0 || p[pi-1] != ':' {
pi = s - 2
pCh = '['
if tCh == pCh {
matched = true
}
// Fall through to the loop tail with pCh='[' so the
// post-step records it as prev_ch.
break
}
classMatched, valid := matchPOSIXClass(p[s:pi-1], tCh, flags)
if !valid {
return wmAbortAll
}
if classMatched {
matched = true
}
pCh = 0 // resets prev_ch
default:
if tCh == pCh {
matched = true
}
}
prevCh = pCh
pi++
if pi >= len(p) {
return wmAbortAll
}
if p[pi] == ']' {
break
}
pCh = p[pi]
}
if matched == negated ||
(flags&wmPathname != 0 && tCh == '/') {
return wmNoMatch
}
pi++
ti++
default:
if tCh != pCh {
return wmNoMatch
}
pi++
ti++
}
}
if ti < len(text) {
return wmNoMatch
}
return wmMatch
}
// isGlobSpecial mirrors is_glob_special() from upstream ctype.c. Bytes that
// can start or modify a wildmatch sub-pattern are "special"; everything else
// is literal text and may be fast-skipped in the '*' loop.
func isGlobSpecial(c byte) bool {
switch c {
case '*', '?', '[', '\\':
return true
}
return false
}
// matchPOSIXClass evaluates a [:name:] character-class entry within a bracket
// expression. Classification is ASCII-only to mirror sane-ctype.h: bytes
// with the high bit set never satisfy any class. valid is false when the
// class name is unrecognized — wildmatch.c propagates that as wmAbortAll
// ("malformed [:class:] string").
func matchPOSIXClass(name string, ch byte, flags int) (matched, valid bool) {
switch name {
case "alnum":
return isASCIIAlpha(ch) || isASCIIDigit(ch), true
case "alpha":
return isASCIIAlpha(ch), true
case "blank":
return ch == ' ' || ch == '\t', true
case "cntrl":
return ch < 0x20 || ch == 0x7f, true
case "digit":
return isASCIIDigit(ch), true
case "graph":
return ch > ' ' && ch < 0x7f, true
case "lower":
return ch >= 'a' && ch <= 'z', true
case "print":
return ch >= ' ' && ch < 0x7f, true
case "punct":
return isASCIIPunct(ch), true
case "space":
return ch == ' ' || ch == '\t' || ch == '\n' ||
ch == '\v' || ch == '\f' || ch == '\r', true
case "upper":
if ch >= 'A' && ch <= 'Z' {
return true, true
}
if flags&wmCasefold != 0 && isASCIILower(ch) {
return true, true
}
return false, true
case "xdigit":
return isASCIIDigit(ch) || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F'), true
default:
return false, false
}
}
func isASCIIAlpha(ch byte) bool {
return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z')
}
func isASCIIDigit(ch byte) bool {
return ch >= '0' && ch <= '9'
}
func isASCIIUpper(ch byte) bool {
return ch >= 'A' && ch <= 'Z'
}
func isASCIILower(ch byte) bool {
return ch >= 'a' && ch <= 'z'
}
func isASCIIPunct(ch byte) bool {
return (ch >= '!' && ch <= '/') ||
(ch >= ':' && ch <= '@') ||
(ch >= '[' && ch <= '`') ||
(ch >= '{' && ch <= '~')
}
func (p *pattern) simpleNameMatch(path []string, isDir bool) bool {
for i, name := range path {
if !wildmatch(p.pattern[0], name) {
continue
}
if p.dirOnly && !isDir && i == len(path)-1 {
return false
}
return true
}
return false
}
func (p *pattern) globMatch(path []string, isDir bool) bool {
matched := false
canTraverse := false
trailingStar := false
for i, pattern := range p.pattern {
if pattern == "" {
canTraverse = false
continue
}
if pattern == zeroToManyDirs {
if i == len(p.pattern)-1 {
// A trailing `**` matches the entries below whatever the
// earlier segments consumed, so it needs either a remaining
// component or a directory candidate standing in for them.
// Assigning matched rather than only raising it stops an
// exhausted path from inheriting the previous segment's
// result, which would make `a/**/*/**` match `a/f.txt`.
matched = len(path) > 0 || isDir
trailingStar = matched
break
}
canTraverse = true
continue
}
// Note: If pattern contains ** but isn't exactly **, it's treated as a regular wildcard pattern
// (e.g., foo** or **bar) and wildmatch will handle it
if len(path) == 0 {
return false
}
if canTraverse {
canTraverse = false
for len(path) > 0 {
e := path[0]
path = path[1:]
if wildmatch(pattern, e) {
matched = true
break
}
if len(path) == 0 {
// A `**` that never finds the segment following it is a
// definitive non-match. Returning here rather than
// clearing matched keeps a trailing `**` from reviving
// the pattern once the path is exhausted, which would
// make `**/bar/**` match directories containing no bar.
return false
}
}
} else {
if !wildmatch(pattern, path[0]) {
return false
}
matched = true
path = path[1:]
// files matching dir globs, don't match
if len(path) == 0 && i < len(p.pattern)-1 {
matched = false
}
}
}
// Check dirOnly: either we consumed all path (len(path) == 0) or we matched a trailing **
if matched && p.dirOnly && !isDir && (len(path) == 0 || trailingStar) {
matched = false
}
return matched
}
package gitignore
import (
"os"
"slices"
"github.com/go-git/go-billy/v6"
)
// Scope is the set of ignore patterns in effect for the entries of a single
// directory: those inherited from its ancestors, in ascending order of
// priority, followed by those the directory declares itself.
//
// A Scope also records whether the directory it belongs to is excluded. That
// is what a flat []Pattern cannot express. gitignore(5) states that a file
// cannot be re-included once a parent directory of it is excluded, so once a
// directory is excluded every entry below it is ignored whatever the patterns
// there say. Reference git tracks the same state in its walk rather than
// recomputing it per query: prep_exclude stops at the excluded directory and
// last_matching_pattern returns its pattern directly (dir.c).
//
// A Scope is immutable and safe for concurrent use. Descend derives the Scope
// for a subdirectory; the zero Scope matches nothing and is a valid root for a
// walk with no patterns at all.
type Scope struct {
// patterns is ancestors-then-own, in ascending order of priority.
patterns []Pattern
matcher Matcher
// excluded reports whether this directory, or an ancestor of it, matched
// an exclude rule.
excluded bool
}
// NewScope returns the root Scope for a walk. base holds the patterns that
// apply to the whole tree before any .gitignore is read: typically
// .git/info/exclude, the core.excludesfile patterns, and any supplied by the
// caller. They are ordered by ascending priority, as for NewMatcher.
func NewScope(base []Pattern) *Scope {
// Cloned so the Scope really is immutable, as documented above: without it
// a caller mutating its own slice afterwards would change this Scope and
// every Scope derived from it, including concurrently. Descend already
// copies, so this is the only place the caller's memory could be shared.
base = slices.Clone(base)
return &Scope{patterns: base, matcher: NewMatcher(base)}
}
// Descend returns the Scope for the subdirectory of s at dir. dir is the full
// path of the subdirectory from the root of the walk, so that patterns keep
// matching against complete paths.
//
// readOwn supplies the patterns the subdirectory declares itself. It is called
// only when the subdirectory is not excluded, because git does not read ignore
// files below an excluded directory and nothing in one could change an
// outcome. Passing it as a function rather than a slice keeps that decision
// here, so callers cannot pay for a read that is then discarded. It may be nil
// when the subdirectory has no ignore file, which callers already know from
// the directory listing they hold.
func (s *Scope) Descend(dir []string, readOwn func() ([]Pattern, error)) (*Scope, error) {
if s.excluded || s.matches(dir, true) {
// Nothing below an excluded directory can change an outcome, so the
// pattern set is frozen here and readOwn is never called.
return &Scope{patterns: s.patterns, matcher: s.matcher, excluded: true}, nil
}
if readOwn == nil {
return s, nil
}
own, err := readOwn()
if err != nil {
return nil, err
}
if len(own) == 0 {
return s, nil
}
patterns := slices.Concat(s.patterns, own)
return &Scope{patterns: patterns, matcher: NewMatcher(patterns)}, nil
}
// Excluded reports whether the directory this Scope belongs to, or an ancestor
// of it, is excluded. Every entry below such a directory is ignored.
func (s *Scope) Excluded() bool {
return s.excluded
}
// Match reports whether path is excluded by the patterns in effect. path is an
// ordered sequence of the components of a complete path from the root of the
// walk, and isDir reports whether its final component is a directory.
//
// Unlike a Matcher built from a flat pattern list, Match honours excluded
// ancestors: below an excluded directory it reports true without consulting
// any pattern, so a negation there cannot re-include the entry.
func (s *Scope) Match(path []string, isDir bool) bool {
if s.excluded {
return true
}
return s.matches(path, isDir)
}
// Patterns returns the patterns in effect, ancestors first. The result must
// not be modified.
func (s *Scope) Patterns() []Pattern {
return s.patterns
}
func (s *Scope) matches(path []string, isDir bool) bool {
if s.matcher == nil {
return false
}
return s.matcher.Match(path, isDir)
}
// DirPatterns returns the patterns declared by the .gitignore of a single
// directory, without recursing. A missing file is not an error and yields no
// patterns. Use it to build the readOwn argument of Scope.Descend while walking
// a tree; ReadPatterns is the eager whole-tree equivalent.
func DirPatterns(fs billy.Filesystem, path []string) ([]Pattern, error) {
ps, err := readIgnoreFile(fs, path, gitignoreFile)
if err != nil && !os.IsNotExist(err) {
return nil, err
}
return ps, nil
}
// RootPatterns returns the patterns that apply to a whole worktree before any
// .gitignore is consulted: .git/info/exclude followed by the .gitignore at the
// root, in ascending order of priority. Missing files are not an error.
func RootPatterns(fs billy.Filesystem) ([]Pattern, error) {
// Both files are opened directly rather than confirmed against a listing of
// the root. Two opens cost less than enumerating a directory that may hold
// thousands of entries, and the walk lists the root for its own purposes
// anyway, so a listing here would be the second of two.
//
// Errors from the exclude file are dropped, as ReadPatterns drops them: it
// is optional, and a worktree filesystem may refuse to open a path inside
// .git outright rather than reporting it as missing.
ps, _ := readIgnoreFile(fs, nil, infoExcludeFile)
root, err := DirPatterns(fs, nil)
if err != nil {
return nil, err
}
return append(ps, root...), nil
}
package idxfile
import (
"bytes"
"encoding/hex"
"errors"
"fmt"
"io"
"io/fs"
"github.com/go-git/go-git/v6/plumbing/hash"
"github.com/go-git/go-git/v6/utils/binary"
)
var (
// ErrUnsupportedVersion is returned by Decode when the idx file version
// is not supported.
ErrUnsupportedVersion = errors.New("unsupported version")
// ErrMalformedIdxFile is returned by Decode when the idx file is corrupted.
ErrMalformedIdxFile = errors.New("malformed idx file")
)
const (
fanout = 256
)
// Byte sizes of the idx v2 layout elements, used by the size formula
// in [validateIdxV2Size]. See [gitformat-pack] for the canonical
// layout.
//
// [gitformat-pack]: https://git-scm.com/docs/gitformat-pack
const (
headerLen = 8 // magic + version
fanoutLen = fanout * 4 // uint32 per bucket
crc32Len = 4 // CRC32 per object
offset32Len = 4 // 32-bit offset per object
offset64Len = 8 // 64-bit overflow offset
trailerHashes = 2 // pack checksum + idx checksum, each hashsz
)
// Input is the input to a [Decoder]. The decoder reads loose-object
// bytes from it and calls Stat to learn the on-disk length, which it
// uses to validate the canonical-Git size formula before any
// allocations driven by the fanout table.
//
// [os.File] and the go-billy [File] type satisfy Input directly.
//
// [File]: https://pkg.go.dev/github.com/go-git/go-billy/v6#File
type Input interface {
io.Reader
Stat() (fs.FileInfo, error)
}
// Decoder reads and decodes idx files from an [Input].
type Decoder struct {
in Input
h hash.Hash
}
// NewDecoder builds a new idx decoder that reads from in.
func NewDecoder(in Input, h hash.Hash) *Decoder {
return &Decoder{in, h}
}
// Decode reads from the input and decodes the content into idx.
func (d *Decoder) Decode(idx *MemoryIndex) error {
fi, err := d.in.Stat()
if err != nil {
return fmt.Errorf("%w: stat input: %w", ErrMalformedIdxFile, err)
}
idxSize := fi.Size()
d.h.Reset()
r := io.TeeReader(d.in, d.h)
if err := validateHeader(r); err != nil {
return err
}
headerFlow := []func(*MemoryIndex, io.Reader) error{
readVersion,
readFanout,
}
for _, f := range headerFlow {
if err := f(idx, r); err != nil {
return err
}
}
if err := validateIdxV2Size(idx, idxSize); err != nil {
return err
}
bodyFlow := []func(*MemoryIndex, io.Reader) error{
readObjectNames,
readCRC32,
readOffsets,
readPackChecksum,
}
for _, f := range bodyFlow {
if err := f(idx, r); err != nil {
return err
}
}
actual := d.h.Sum(nil)
if err := readIdxChecksum(idx, r); err != nil {
return err
}
if idx.IdxChecksum.Compare(actual) != 0 {
return fmt.Errorf("%w: checksum mismatch: %q instead of %q",
ErrMalformedIdxFile, idx.IdxChecksum.String(), hex.EncodeToString(actual))
}
return nil
}
func validateHeader(r io.Reader) error {
h := make([]byte, 4)
if _, err := io.ReadFull(r, h); err != nil {
return err
}
if !bytes.Equal(h, idxHeader) {
return ErrMalformedIdxFile
}
return nil
}
func readVersion(idx *MemoryIndex, r io.Reader) error {
v, err := binary.ReadUint32(r)
if err != nil {
return err
}
if v != VersionSupported {
return fmt.Errorf("%w: v%d", ErrUnsupportedVersion, v)
}
idx.Version = v
return nil
}
func readFanout(idx *MemoryIndex, r io.Reader) error {
for k := range fanout {
n, err := binary.ReadUint32(r)
if err != nil {
return err
}
if k > 0 && n < idx.Fanout[k-1] {
return fmt.Errorf("%w: fanout table is not monotonically non-decreasing at entry %d", ErrMalformedIdxFile, k)
}
idx.Fanout[k] = n
idx.FanoutMapping[k] = noMapping
}
return nil
}
func readObjectNames(idx *MemoryIndex, r io.Reader) error {
idSize := uint32(idx.idSize())
for k := range fanout {
var buckets uint32
if k == 0 {
buckets = idx.Fanout[k]
} else {
buckets = idx.Fanout[k] - idx.Fanout[k-1]
}
if buckets == 0 {
continue
}
idx.FanoutMapping[k] = len(idx.Names)
nameLen := int(buckets * idSize)
bin := make([]byte, nameLen)
if _, err := io.ReadFull(r, bin); err != nil {
return err
}
idx.Names = append(idx.Names, bin)
idx.Offset32 = append(idx.Offset32, make([]byte, buckets*4))
idx.CRC32 = append(idx.CRC32, make([]byte, buckets*4))
}
return nil
}
func readCRC32(idx *MemoryIndex, r io.Reader) error {
for k := range fanout {
if pos := idx.FanoutMapping[k]; pos != noMapping {
if _, err := io.ReadFull(r, idx.CRC32[pos]); err != nil {
return err
}
}
}
return nil
}
func readOffsets(idx *MemoryIndex, r io.Reader) error {
var o64cnt int64
for k := range fanout {
if pos := idx.FanoutMapping[k]; pos != noMapping {
if _, err := io.ReadFull(r, idx.Offset32[pos]); err != nil {
return err
}
for p := 0; p < len(idx.Offset32[pos]); p += 4 {
if idx.Offset32[pos][p]&(byte(1)<<7) > 0 {
o64cnt++
}
}
}
}
if o64cnt > 0 {
idx.Offset64 = make([]byte, o64cnt*8)
if _, err := io.ReadFull(r, idx.Offset64); err != nil {
return err
}
}
return nil
}
func readPackChecksum(idx *MemoryIndex, r io.Reader) error {
idx.PackfileChecksum.ResetBySize(idx.idSize())
if _, err := idx.PackfileChecksum.ReadFrom(r); err != nil {
return err
}
return nil
}
func readIdxChecksum(idx *MemoryIndex, r io.Reader) error {
idx.IdxChecksum.ResetBySize(idx.idSize())
if _, err := idx.IdxChecksum.ReadFrom(r); err != nil {
return err
}
return nil
}
// validateIdxV2Size enforces the size formula used by canonical Git
// load_idx for idx v2 files: the on-disk length must lie within
// [minSize, maxSize] where
//
// perObject = hashsz + crc32Len + offset32Len
// minSize = headerLen + fanoutLen + trailerHashes*hashsz + nr*perObject
// maxSize = minSize + (nr-1)*offset64Len when nr > 0
//
// with nr taken from the last fanout entry and hashsz from the
// configured object ID size. Multiplications use a self-checking
// overflow guard so inputs whose claimed object count overflows the
// formula are rejected rather than wrapping into a smaller value.
func validateIdxV2Size(idx *MemoryIndex, idxSize int64) error {
nr := int64(idx.Fanout[fanout-1])
hashsz := int64(idx.idSize())
minSize := minIdxV2Size(nr, hashsz)
maxSize := maxIdxV2Size(nr, hashsz)
if minSize < 0 || maxSize < 0 {
return fmt.Errorf("%w: object count %d is inconsistent with file size", ErrMalformedIdxFile, nr)
}
if idxSize < minSize || idxSize > maxSize {
return fmt.Errorf("%w: file size %d is inconsistent with object count %d", ErrMalformedIdxFile, idxSize, nr)
}
return nil
}
// minIdxV2Size returns the minimum on-disk size of an idx v2 file
// holding nr objects with the given hash size, mirroring the
// computation in canonical Git load_idx. Returns -1 when any
// intermediate multiplication or addition would overflow int64.
func minIdxV2Size(nr, hashsz int64) int64 {
perObject := hashsz + crc32Len + offset32Len
fixed := int64(headerLen+fanoutLen) + trailerHashes*hashsz
objects, ok := mulInt64(nr, perObject)
if !ok {
return -1
}
sum, ok := addInt64(fixed, objects)
if !ok {
return -1
}
return sum
}
// maxIdxV2Size returns the maximum on-disk size of an idx v2 file
// holding nr objects with the given hash size, mirroring the
// computation in canonical Git load_idx. Returns -1 on overflow.
func maxIdxV2Size(nr, hashsz int64) int64 {
minSize := minIdxV2Size(nr, hashsz)
if minSize < 0 {
return -1
}
if nr == 0 {
return minSize
}
overflow, ok := mulInt64(nr-1, offset64Len)
if !ok {
return -1
}
sum, ok := addInt64(minSize, overflow)
if !ok {
return -1
}
return sum
}
// mulInt64 returns a*b and whether the result fits in an int64 without
// overflow. Negative operands or overflow yield ok=false. The overflow
// check uses the standard self-inverse identity: a*b/b == a only when
// the multiplication did not wrap.
func mulInt64(a, b int64) (int64, bool) {
if a < 0 || b < 0 {
return 0, false
}
if a == 0 || b == 0 {
return 0, true
}
c := a * b
if c/b != a {
return 0, false
}
return c, true
}
// addInt64 returns a+b and whether the result fits in an int64 without
// overflow. Negative operands or overflow yield ok=false.
func addInt64(a, b int64) (int64, bool) {
if a < 0 || b < 0 {
return 0, false
}
c := a + b
if c < a {
return 0, false
}
return c, true
}
package idxfile
import (
"fmt"
"hash"
"io"
"github.com/go-git/go-git/v6/utils/binary"
)
// encoder is the internal state for encoding an idx file.
// It is not exported to prevent reuse - each Encode call creates fresh state.
type encoder struct {
writer io.Writer
hashSum func() []byte
idx *MemoryIndex
}
// stateFnEncode defines each individual state within the state machine that
// represents encoding an idxfile.
type stateFnEncode func(*encoder) (stateFnEncode, error)
// Encode encodes a MemoryIndex to the writer.
// This function is safe to call concurrently with different parameters.
func Encode(w io.Writer, h hash.Hash, idx *MemoryIndex) error {
if w == nil {
return fmt.Errorf("nil writer")
}
if idx == nil {
return fmt.Errorf("nil index")
}
e := &encoder{
writer: io.MultiWriter(w, h),
hashSum: func() []byte { return h.Sum(nil) },
idx: idx,
}
for state := writeHeader; state != nil; {
var err error
state, err = state(e)
if err != nil {
return err
}
}
return nil
}
func writeHeader(e *encoder) (stateFnEncode, error) {
if e.idx.Version != VersionSupported {
return nil, ErrUnsupportedVersion
}
_, err := e.writer.Write(idxHeader)
if err != nil {
return nil, err
}
err = binary.WriteUint32(e.writer, e.idx.Version)
if err != nil {
return nil, err
}
return writeFanout, nil
}
func writeFanout(e *encoder) (stateFnEncode, error) {
for _, c := range e.idx.Fanout {
if err := binary.WriteUint32(e.writer, c); err != nil {
return nil, err
}
}
return writeHashes, nil
}
func writeHashes(e *encoder) (stateFnEncode, error) {
for k := range fanout {
pos := e.idx.FanoutMapping[k]
if pos == noMapping {
continue
}
if pos >= len(e.idx.Names) {
return nil, fmt.Errorf("%w: invalid position %d", ErrMalformedIdxFile, pos)
}
_, err := e.writer.Write(e.idx.Names[pos])
if err != nil {
return nil, err
}
}
return writeCRC32, nil
}
func writeCRC32(e *encoder) (stateFnEncode, error) {
for k := range fanout {
pos := e.idx.FanoutMapping[k]
if pos == noMapping {
continue
}
if pos >= len(e.idx.CRC32) {
return nil, fmt.Errorf("%w: invalid CRC32 index %d", ErrMalformedIdxFile, pos)
}
_, err := e.writer.Write(e.idx.CRC32[pos])
if err != nil {
return nil, err
}
}
return writeOffsets, nil
}
func writeOffsets(e *encoder) (stateFnEncode, error) {
for k := range fanout {
pos := e.idx.FanoutMapping[k]
if pos == noMapping {
continue
}
if pos >= len(e.idx.Offset32) {
return nil, fmt.Errorf("%w: invalid offset32 index %d", ErrMalformedIdxFile, pos)
}
_, err := e.writer.Write(e.idx.Offset32[pos])
if err != nil {
return nil, err
}
}
if len(e.idx.Offset64) > 0 {
_, err := e.writer.Write(e.idx.Offset64)
if err != nil {
return nil, err
}
}
return writeChecksums, nil
}
func writeChecksums(e *encoder) (stateFnEncode, error) {
_, err := e.writer.Write(e.idx.PackfileChecksum.Bytes())
if err != nil {
return nil, err
}
checksum := e.hashSum()
if _, err := e.idx.IdxChecksum.Write(checksum); err != nil {
return nil, err
}
_, err = e.writer.Write(e.idx.IdxChecksum.Bytes())
if err != nil {
return nil, err
}
return nil, nil
}
package idxfile
import (
"bytes"
"crypto/sha1"
"encoding/binary"
"github.com/go-git/go-git/v6/plumbing"
)
// buildMinimalIdx constructs a minimal valid idx v2 file with the given
// number of objects and hash size. Used by fuzz seed corpus generation.
func buildMinimalIdx(count, hashSize int) []byte {
var buf bytes.Buffer
buf.Write([]byte{0xff, 't', 'O', 'c'})
_ = binary.Write(&buf, binary.BigEndian, uint32(2))
for range 256 {
_ = binary.Write(&buf, binary.BigEndian, uint32(count))
}
for i := range count {
h := make([]byte, hashSize)
// Ensure all hashes start with 0x00 (match fanout bucket 0).
h[1] = byte(i >> 8)
h[2] = byte(i)
buf.Write(h)
}
// CRC32: count * 4 bytes (all zeros).
buf.Write(make([]byte, count*4))
// Offset32: count * 4 bytes (sequential small offsets).
for i := range count {
_ = binary.Write(&buf, binary.BigEndian, uint32(i*100))
}
// No offset64 entries.
packChecksum := make([]byte, hashSize)
packChecksum[0] = 0xAA // recognizable
buf.Write(packChecksum)
buf.Write(make([]byte, hashSize)) // idx checksum
return buf.Bytes()
}
// buildMinimalRev constructs a minimal valid .rev file for the given
// number of objects and hash size. Used by fuzz seed corpus generation.
func buildMinimalRev(count, hashSize int) []byte {
var buf bytes.Buffer
buf.Write([]byte{'R', 'I', 'D', 'X'})
_ = binary.Write(&buf, binary.BigEndian, uint32(1)) // version
hashID := uint32(1) // sha1
if hashSize == 32 {
hashID = 2 // sha256
}
_ = binary.Write(&buf, binary.BigEndian, hashID)
// Entries: identity mapping (already sorted by offset).
for i := range count {
_ = binary.Write(&buf, binary.BigEndian, uint32(i))
}
buf.Write(make([]byte, hashSize*2))
return buf.Bytes()
}
// buildOOBOffset64Idx constructs a structurally valid v2 idx file
// whose 32-bit offset entry for the first object is marked as a
// 64-bit overflow (MSB set) but whose lower 31 bits point past the
// only allocated 64-bit offset slot. The idx decodes successfully;
// using it must fail with [ErrMalformedIdxFile] rather than reach
// the post-decode lookups in an inconsistent state.
//
// The returned hash is the name of the first object — pass it to
// FindOffset to exercise the malformed-input path.
//
// The fixture has two objects so the on-disk length satisfies the
// idx v2 size formula applied during Decode; only with `nr > 1`
// does the formula permit any 8-byte offset64 slots.
//
// Lives outside `_test.go` so the OSS-Fuzz harness, which does not
// see other test files when extracting fuzz targets, can reach it
// from the seed corpus. The harness finds a target by grepping the
// package for its name, so naming one here would break the build.
// See `make validate-fuzz`.
func buildOOBOffset64Idx() ([]byte, plumbing.Hash) {
const hashSize = 20
var buf bytes.Buffer
buf.Write(idxHeader)
_ = binary.Write(&buf, binary.BigEndian, uint32(2))
// Fanout: two objects whose first bytes are 0x00, so all 256
// entries hold the cumulative count 2.
for range 256 {
_ = binary.Write(&buf, binary.BigEndian, uint32(2))
}
// Two names (both valid 20-byte hashes with first byte 0x00,
// in ascending order so the table remains well-formed).
name1 := make([]byte, hashSize)
name1[hashSize-1] = 0x01
name2 := make([]byte, hashSize)
name2[hashSize-1] = 0x02
buf.Write(name1)
buf.Write(name2)
// CRC32 (two entries, values irrelevant).
buf.Write(make([]byte, 8))
// Offset32 for object 1: MSB set, lower 31 bits = 5 → references
// Offset64[40:48]; only one 8-byte slot exists, so this is out
// of range.
_ = binary.Write(&buf, binary.BigEndian, uint32(0x80000005))
// Offset32 for object 2: a small in-range value.
_ = binary.Write(&buf, binary.BigEndian, uint32(0))
// Offset64: a single 8-byte slot — the lookup above is out of range.
_ = binary.Write(&buf, binary.BigEndian, uint64(0x12345678))
// Pack checksum (zeros — the LazyIndex test passes the same value
// in as the expected pack hash so it matches).
buf.Write(make([]byte, hashSize))
// Idx checksum: SHA1 of everything written so far.
sum := sha1.Sum(buf.Bytes())
buf.Write(sum[:])
var h plumbing.Hash
h.ResetBySize(hashSize)
_, _ = h.Write(name1)
return buf.Bytes(), h
}
// nopCloserReaderAt wraps a bytes.Reader to satisfy ReadAtCloser.
type nopCloserReaderAt struct {
*bytes.Reader
}
func (nopCloserReaderAt) Close() error { return nil }
package idxfile
import (
"bytes"
"crypto"
encbin "encoding/binary"
"fmt"
"io"
"sort"
"sync"
"github.com/go-git/go-git/v6/plumbing"
)
const (
// VersionSupported is the only idx version supported.
VersionSupported = 2
noMapping = -1
)
var idxHeader = []byte{255, 't', 'O', 'c'}
// Index represents an index of a packfile.
//
// Implementations satisfy a [io.Closer] contract via [Index.Close]:
// on-disk implementations release file descriptors, pure
// in-memory implementations return nil. Downstream callers
// holding their own concrete [Index] implementations must
// supply a [Close] method to satisfy this interface; a no-op
// `func (*MyIndex) Close() error { return nil }` is sufficient
// for in-memory backends.
type Index interface {
// Contains checks whether the given hash is in the index.
Contains(h plumbing.Hash) (bool, error)
// FindOffset finds the offset in the packfile for the object with
// the given hash.
FindOffset(h plumbing.Hash) (int64, error)
// FindCRC32 finds the CRC32 of the object with the given hash.
FindCRC32(h plumbing.Hash) (uint32, error)
// FindHash finds the hash for the object with the given offset.
FindHash(o int64) (plumbing.Hash, error)
// Count returns the number of entries in the index.
Count() (int64, error)
// Entries returns an iterator to retrieve all index entries.
Entries() (EntryIter, error)
// EntriesByOffset returns an iterator to retrieve all index entries ordered
// by offset.
EntriesByOffset() (EntryIter, error)
// EntriesWithPrefix returns an iterator over index entries whose
// hashes start with prefix. Implementations use the fanout table
// to bound the search when len(prefix) >= 1; an empty prefix
// returns all entries (equivalent to Entries). The returned
// iterator must be Closed by the caller to release any held
// resources.
EntriesWithPrefix(prefix []byte) (EntryIter, error)
// MayContain reports whether the index might contain h. A false
// return is authoritative ("h is definitely not in this pack")
// based on the idx fanout table; true means the caller should
// call Contains or FindOffset for a definitive answer.
//
// Implementations must be O(1) and I/O-free. Callers route
// every read through MayContain to gate further index work
// (see storage/filesystem.ObjectStorage.findObjectInPackfile);
// an implementation that performs I/O or scales with index
// size silently regresses every storage-level read.
MayContain(h plumbing.Hash) bool
// Close releases any resources held by the index. Implementations
// backed by on-disk files must close their file descriptors; pure
// in-memory implementations must return nil. Close is idempotent.
Close() error
}
// MemoryIndex is the in memory representation of an idx file.
//
// The use of MemoryIndex for large repositories is discouraged.
// Use [LazyIndex] instead.
type MemoryIndex struct {
// Version is the version of the index file.
Version uint32
// Fanout is a table where the Nth entry is the cumulative count of objects with the first byte of their name <= N.
Fanout [256]uint32
// FanoutMapping maps the position in the fanout table to the position
// in the Names, Offset32 and CRC32 slices. This improves the memory
// usage by not needing an array with unnecessary empty slots.
FanoutMapping [256]int
// Names is the list of object names.
Names [][]byte
// Offset32 is the list of 32-bit offsets.
Offset32 [][]byte
// CRC32 is the list of CRC32 checksums.
CRC32 [][]byte
// Offset64 is the list of 64-bit offsets.
Offset64 []byte
// PackfileChecksum is the checksum of the packfile.
PackfileChecksum plumbing.Hash
// IdxChecksum is the checksum of the index file.
IdxChecksum plumbing.Hash
offsetHash map[int64]plumbing.Hash
offsetBuildOnce sync.Once
mu sync.RWMutex
objectIDSize int
}
var _ Index = (*MemoryIndex)(nil)
// Close is a no-op. MemoryIndex holds no external resources.
func (idx *MemoryIndex) Close() error { return nil }
// NewMemoryIndex returns an instance of a new MemoryIndex.
func NewMemoryIndex(objectIDSize int) *MemoryIndex {
m := &MemoryIndex{objectIDSize: objectIDSize}
m.IdxChecksum.ResetBySize(objectIDSize)
m.PackfileChecksum.ResetBySize(objectIDSize)
return m
}
func (idx *MemoryIndex) findHashIndex(h plumbing.Hash) (int, bool) {
k := idx.FanoutMapping[h.Bytes()[0]]
if k == noMapping {
return 0, false
}
if len(idx.Names) <= k {
return 0, false
}
data := idx.Names[k]
high := uint64(len(idx.Offset32[k])) >> 2
if high == 0 {
return 0, false
}
low := uint64(0)
for {
mid := (low + high) >> 1
offset := mid * uint64(idx.idSize())
cmp := h.Compare(data[offset : offset+uint64(idx.idSize())])
switch {
case cmp < 0:
high = mid
case cmp == 0:
return int(mid), true
default:
low = mid + 1
}
if low >= high {
break
}
}
return 0, false
}
// MayContain implements the Index interface. It reports whether the
// index might contain h using the in-memory fanout mapping. Returns
// false iff h's first byte falls in an empty fanout bucket.
func (idx *MemoryIndex) MayContain(h plumbing.Hash) bool {
return idx.FanoutMapping[h.Bytes()[0]] != noMapping
}
// Contains implements the Index interface.
func (idx *MemoryIndex) Contains(h plumbing.Hash) (bool, error) {
_, ok := idx.findHashIndex(h)
return ok, nil
}
// FindOffset implements the Index interface.
func (idx *MemoryIndex) FindOffset(h plumbing.Hash) (int64, error) {
fo := h.Bytes()[0]
if len(idx.FanoutMapping) <= int(fo) {
return 0, plumbing.ErrObjectNotFound
}
k := idx.FanoutMapping[fo]
i, ok := idx.findHashIndex(h)
if !ok {
return 0, plumbing.ErrObjectNotFound
}
offset, err := idx.getOffset(k, i)
if err != nil {
return 0, err
}
// Save the offset for reverse lookup
idx.mu.Lock()
if idx.offsetHash == nil {
idx.offsetHash = make(map[int64]plumbing.Hash)
}
idx.offsetHash[int64(offset)] = h
idx.mu.Unlock()
return int64(offset), nil
}
const isO64Mask = uint64(1) << 31
func (idx *MemoryIndex) getOffset(firstLevel, secondLevel int) (uint64, error) {
offset := secondLevel << 2
ofs := encbin.BigEndian.Uint32(idx.Offset32[firstLevel][offset : offset+4])
if (uint64(ofs) & isO64Mask) != 0 {
offset := 8 * (uint64(ofs) & ^isO64Mask)
if l := uint64(len(idx.Offset64)); l < 8 || offset > l-8 {
return 0, fmt.Errorf("%w: offset64 index out of range", ErrMalformedIdxFile)
}
return encbin.BigEndian.Uint64(idx.Offset64[offset : offset+8]), nil
}
return uint64(ofs), nil
}
// FindCRC32 implements the Index interface.
func (idx *MemoryIndex) FindCRC32(h plumbing.Hash) (uint32, error) {
k := idx.FanoutMapping[h.Bytes()[0]]
i, ok := idx.findHashIndex(h)
if !ok {
return 0, plumbing.ErrObjectNotFound
}
return idx.getCRC32(k, i), nil
}
func (idx *MemoryIndex) getCRC32(firstLevel, secondLevel int) uint32 {
offset := secondLevel << 2
return encbin.BigEndian.Uint32(idx.CRC32[firstLevel][offset : offset+4])
}
// FindHash implements the Index interface.
func (idx *MemoryIndex) FindHash(o int64) (plumbing.Hash, error) {
var hash plumbing.Hash
var ok bool
idx.mu.RLock()
if idx.offsetHash != nil {
if hash, ok = idx.offsetHash[o]; ok {
idx.mu.RUnlock()
return hash, nil
}
}
idx.mu.RUnlock()
var genErr error
idx.offsetBuildOnce.Do(func() {
genErr = idx.genOffsetHash()
})
if genErr != nil {
return plumbing.ZeroHash, genErr
}
idx.mu.RLock()
hash, ok = idx.offsetHash[o]
idx.mu.RUnlock()
if !ok {
return plumbing.ZeroHash, plumbing.ErrObjectNotFound
}
return hash, nil
}
// genOffsetHash generates the offset/hash mapping for reverse search.
func (idx *MemoryIndex) genOffsetHash() error {
count, err := idx.Count()
if err != nil {
return err
}
offsetHash := make(map[int64]plumbing.Hash, count)
var hash plumbing.Hash
hash.ResetBySize(idx.objectIDSize)
i := uint32(0)
for firstLevel, fanoutValue := range idx.Fanout {
mappedFirstLevel := idx.FanoutMapping[firstLevel]
for secondLevel := uint32(0); i < fanoutValue; i++ {
_, err = hash.Write(idx.Names[mappedFirstLevel][secondLevel*uint32(idx.idSize()):])
if err != nil {
return fmt.Errorf("cannot write name to hash: %w", err)
}
off, err := idx.getOffset(mappedFirstLevel, int(secondLevel))
if err != nil {
return err
}
offsetHash[int64(off)] = hash
secondLevel++
}
}
idx.mu.Lock()
idx.offsetHash = offsetHash
idx.mu.Unlock()
return nil
}
// Count implements the Index interface.
func (idx *MemoryIndex) Count() (int64, error) {
return int64(idx.Fanout[fanout-1]), nil
}
// Entries implements the Index interface.
func (idx *MemoryIndex) Entries() (EntryIter, error) {
return &idxfileEntryIter{idx, 0, 0, 0}, nil
}
// EntriesWithPrefix implements the Index interface. It returns an
// iterator over entries whose hashes start with prefix. When prefix
// is empty the call is equivalent to Entries; otherwise the
// iterator visits only the fanout bucket selected by prefix[0] and
// stops as soon as the sorted-by-hash bucket walks past prefix.
//
// For a multi-byte prefix the matching entries form a contiguous
// run somewhere within the bucket; binary-search positions the
// iterator at the start of that run so the linear walk only spans
// matches. This mirrors upstream Git's for_each_prefixed_object_in_pack
// which calls bsearch_pack to position before walking forward.
func (idx *MemoryIndex) EntriesWithPrefix(prefix []byte) (EntryIter, error) {
if len(prefix) == 0 {
return idx.Entries()
}
bucket := idx.FanoutMapping[prefix[0]]
if bucket == noMapping {
return &idxfilePrefixIter{done: true}, nil
}
idSize := idx.idSize()
names := idx.Names[bucket]
n := len(names) / idSize
// Find the leftmost entry whose hash is >= prefix (padded with
// zeros to hash size). All matching entries, if any, start at
// this position; the iterator's stop-on-first-mismatch then
// terminates correctly once the run ends.
target := make([]byte, idSize)
copy(target, prefix)
lo, hi := 0, n
for lo < hi {
mid := (lo + hi) >> 1
slot := names[mid*idSize : (mid+1)*idSize]
if bytes.Compare(slot, target) < 0 {
lo = mid + 1
} else {
hi = mid
}
}
return &idxfilePrefixIter{
idSize: idSize,
prefix: prefix,
names: names,
offset32: idx.Offset32[bucket],
crc32: idx.CRC32[bucket],
offset64: idx.Offset64,
pos: lo,
}, nil
}
// EntriesByOffset implements the Index interface.
func (idx *MemoryIndex) EntriesByOffset() (EntryIter, error) {
count, err := idx.Count()
if err != nil {
return nil, err
}
iter := &idxfileEntryOffsetIter{
entries: make(entriesByOffset, count),
}
entries, err := idx.Entries()
if err != nil {
return nil, err
}
for pos := 0; int64(pos) < count; pos++ {
entry, err := entries.Next()
if err != nil {
return nil, err
}
iter.entries[pos] = entry
}
sort.Sort(iter.entries)
return iter, nil
}
func (idx *MemoryIndex) idSize() int {
if idx.objectIDSize != 0 {
return idx.objectIDSize
}
return crypto.SHA1.Size()
}
// EntryIter is an iterator that will return the entries in a packfile index.
type EntryIter interface {
// Next returns the next entry in the packfile index.
Next() (*Entry, error)
// Close closes the iterator.
Close() error
}
type idxfileEntryIter struct {
idx *MemoryIndex
total int
firstLevel, secondLevel int
}
func (i *idxfileEntryIter) Next() (*Entry, error) {
for {
if i.firstLevel >= fanout {
return nil, io.EOF
}
if i.total >= int(i.idx.Fanout[i.firstLevel]) {
i.firstLevel++
i.secondLevel = 0
continue
}
mappedFirstLevel := i.idx.FanoutMapping[i.firstLevel]
entry := new(Entry)
entry.Hash.ResetBySize(i.idx.idSize())
_, err := entry.Hash.Write(i.idx.Names[mappedFirstLevel][i.secondLevel*i.idx.idSize():])
if err != nil {
return nil, fmt.Errorf("cannot write entry hash: %w", err)
}
entry.Offset, err = i.idx.getOffset(mappedFirstLevel, i.secondLevel)
if err != nil {
return nil, err
}
entry.CRC32 = i.idx.getCRC32(mappedFirstLevel, i.secondLevel)
i.secondLevel++
i.total++
return entry, nil
}
}
func (i *idxfileEntryIter) Close() error {
i.firstLevel = fanout
return nil
}
// idxfilePrefixIter walks a single fanout bucket, yielding entries
// whose hash starts with prefix. The bucket is sorted by hash, so
// once a name is read whose first bytes do not match prefix the
// iterator stops.
//
// The iterator references the bucket's per-slot slices directly
// (names, offset32, crc32) plus the shared offset64 table, so it
// does not retain a reference to the parent MemoryIndex. This keeps
// the iterator footprint to just the cursor state and the slice
// headers it actually reads from.
//
// Lifetime: the slice headers are views into the parent
// MemoryIndex's per-bucket storage. The iterator is invalid after
// the parent Index is closed or reindexed — callers must consume
// (or Close) the iterator before discarding the Index.
type idxfilePrefixIter struct {
idSize int
prefix []byte
names []byte // bucket's hash bytes
offset32 []byte // bucket's 32-bit offset table
crc32 []byte // bucket's CRC32 table
offset64 []byte // shared 64-bit offset overflow table
pos int // entries already yielded
done bool
}
func (i *idxfilePrefixIter) Next() (*Entry, error) {
if i.done {
return nil, io.EOF
}
offset := i.pos * i.idSize
if offset+i.idSize > len(i.names) {
i.done = true
return nil, io.EOF
}
hashBytes := i.names[offset : offset+i.idSize]
if !bytes.HasPrefix(hashBytes, i.prefix) {
// Bucket is sorted by hash, so the first mismatch ends the run.
i.done = true
return nil, io.EOF
}
entry := new(Entry)
entry.Hash.ResetBySize(i.idSize)
if _, err := entry.Hash.Write(hashBytes); err != nil {
return nil, fmt.Errorf("cannot write entry hash: %w", err)
}
o, err := i.bucketOffset(i.pos)
if err != nil {
return nil, err
}
entry.Offset = o
entry.CRC32 = i.bucketCRC32(i.pos)
i.pos++
return entry, nil
}
// bucketOffset mirrors MemoryIndex.getOffset using only the per-
// bucket Offset32/Offset64 slices the iterator holds, so callers
// do not need to retain a reference to the parent MemoryIndex.
func (i *idxfilePrefixIter) bucketOffset(pos int) (uint64, error) {
off := pos << 2
ofs := encbin.BigEndian.Uint32(i.offset32[off : off+4])
if (uint64(ofs) & isO64Mask) != 0 {
o64 := 8 * (uint64(ofs) & ^isO64Mask)
if l := uint64(len(i.offset64)); l < 8 || o64 > l-8 {
return 0, fmt.Errorf("%w: offset64 index out of range", ErrMalformedIdxFile)
}
return encbin.BigEndian.Uint64(i.offset64[o64 : o64+8]), nil
}
return uint64(ofs), nil
}
// bucketCRC32 mirrors MemoryIndex.getCRC32 using only the per-bucket
// CRC32 slice the iterator holds.
func (i *idxfilePrefixIter) bucketCRC32(pos int) uint32 {
off := pos << 2
return encbin.BigEndian.Uint32(i.crc32[off : off+4])
}
func (i *idxfilePrefixIter) Close() error {
i.done = true
return nil
}
// Entry is the in memory representation of an object entry in the idx file.
type Entry struct {
Hash plumbing.Hash
CRC32 uint32
Offset uint64
}
type idxfileEntryOffsetIter struct {
entries entriesByOffset
pos int
}
func (i *idxfileEntryOffsetIter) Next() (*Entry, error) {
if i.pos >= len(i.entries) {
return nil, io.EOF
}
entry := i.entries[i.pos]
i.pos++
return entry, nil
}
func (i *idxfileEntryOffsetIter) Close() error {
i.pos = len(i.entries) + 1
return nil
}
type entriesByOffset []*Entry
func (o entriesByOffset) Len() int {
return len(o)
}
func (o entriesByOffset) Less(i, j int) bool {
return o[i].Offset < o[j].Offset
}
func (o entriesByOffset) Swap(i, j int) {
o[i], o[j] = o[j], o[i]
}
package idxfile
import (
"bytes"
"encoding/binary"
"errors"
"fmt"
"io"
"time"
"github.com/go-git/go-git/v6/internal/sharedfile"
"github.com/go-git/go-git/v6/plumbing"
gsync "github.com/go-git/go-git/v6/utils/sync"
"github.com/go-git/go-git/v6/x/fdpool"
)
const defaultCloseGracePeriod = time.Second
const (
idxHeaderSize = 8 // 4 magic + 4 version
idxFanoutSize = 256 * 4
off32Size = 4
off64Size = 8
revHeaderSize = 12 // 4 magic + 4 version + 4 hash function
is64bitsMask = uint64(1) << 31
)
// ReadAtCloser is the interface required for files used by LazyIndex.
// It is an alias for [sharedfile.ReadAtCloser]; both names refer
// to the same type at compile time.
type ReadAtCloser = sharedfile.ReadAtCloser
// LazyIndex implements the Index interface by reading directly from
// .idx and .rev files via ReadAt, without loading all data into memory.
//
// File descriptors are managed automatically via reference-counted
// shared handles: opened lazily on first use, shared across concurrent
// readers, and closed when no readers remain. This avoids holding
// descriptors open indefinitely while still sharing a single FD across
// concurrent operations.
type LazyIndex struct {
hashSize int
count int
count64 int
// Section byte offsets within the idx file.
fanoutStart int
namesStart int
crcStart int
off32Start int
off64Start int
idx *sharedfile.SharedFile
rev *sharedfile.SharedFile
fanout [256]uint32 // cached from idx; small enough to keep in memory
}
var _ Index = (*LazyIndex)(nil)
// NewLazyIndex creates a LazyIndex from opener functions for .idx and
// .rev files.
//
// The openers are called to obtain file handles on demand. Each call
// must return a fresh, independently closeable handle. File descriptors
// are shared across concurrent readers and released automatically when
// idle.
func NewLazyIndex(openIdx, openRev func() (ReadAtCloser, error), packHash plumbing.Hash) (*LazyIndex, error) {
return NewLazyIndexWithPool(openIdx, openRev, packHash, nil)
}
// NewLazyIndexWithPool is like [NewLazyIndex] but registers the
// idx and rev [sharedfile.SharedFile]s with the given
// [*fdpool.Pool]. The pool governs LRU eviction across many
// LazyIndexes so a storage-wide FD budget covers the .idx and
// .rev descriptors. Pass nil to disable pooling (equivalent to
// [NewLazyIndex]).
//
// When pool is non-nil the [defaultCloseGracePeriod] timer is
// inert: each FD stays open and registered with the pool until
// the LRU evicts it (or [LazyIndex.Close] tears it down). When
// pool is nil the grace timer governs FD lifetime as in
// [NewLazyIndex].
//
// Neither this constructor nor the [Index] methods accept a
// [context.Context]. Index lookups are pure ReadAt I/O without
// cancellation hooks, matching the context-free convention of
// the storage, plumbing/format, and plumbing/storer layers;
// callers requiring cancellation enforce it at the call-site
// in the layer above.
func NewLazyIndexWithPool(openIdx, openRev func() (ReadAtCloser, error), packHash plumbing.Hash, pool *fdpool.Pool) (*LazyIndex, error) {
if openIdx == nil {
return nil, errors.New("idx opener is nil")
}
if openRev == nil {
return nil, errors.New("rev opener is nil")
}
s := &LazyIndex{
idx: sharedfile.NewWithPool(openIdx, defaultCloseGracePeriod, pool),
rev: sharedfile.NewWithPool(openRev, defaultCloseGracePeriod, pool),
}
if err := s.init(packHash); err != nil {
_ = s.Close()
return nil, err
}
return s, nil
}
// init reads and validates headers, caches the fanout table and
// computes section offsets. It acquires file handles through the
// sharedFile so the grace period keeps them warm for the first real
// operation.
func (s *LazyIndex) init(packHash plumbing.Hash) error {
idxRA, err := s.idx.Acquire()
if err != nil {
return fmt.Errorf("cannot open idx: %w", err)
}
defer s.idx.Release()
revRA, err := s.rev.Acquire()
if err != nil {
return fmt.Errorf("cannot open rev: %w", err)
}
defer s.rev.Release()
var hdr [idxHeaderSize]byte
if _, err := idxRA.ReadAt(hdr[:], 0); err != nil {
return fmt.Errorf("cannot read idx header: %w", err)
}
if !bytes.Equal(hdr[:4], idxHeader) {
return fmt.Errorf("%w: %s", ErrMalformedIdxFile, "header mismatch")
}
v := binary.BigEndian.Uint32(hdr[4:])
if v != VersionSupported {
return ErrUnsupportedVersion
}
var revHdr [revHeaderSize]byte
if _, err := revRA.ReadAt(revHdr[:], 0); err != nil {
return fmt.Errorf("cannot read rev header: %w", err)
}
if !bytes.Equal(revHdr[:4], []byte{'R', 'I', 'D', 'X'}) {
return fmt.Errorf("%w: rev file magic mismatch", ErrMalformedIdxFile)
}
if v := binary.BigEndian.Uint32(revHdr[4:]); v != 1 {
return fmt.Errorf("%w: unsupported rev file version %d", ErrMalformedIdxFile, v)
}
s.fanoutStart = idxHeaderSize
var fanoutBuf [idxFanoutSize]byte
if _, err := idxRA.ReadAt(fanoutBuf[:], int64(s.fanoutStart)); err != nil {
return fmt.Errorf("cannot read idx fanout: %w", err)
}
for i := range 256 {
s.fanout[i] = binary.BigEndian.Uint32(fanoutBuf[i*4:])
if i > 0 && s.fanout[i] < s.fanout[i-1] {
return fmt.Errorf("%w: fanout table is not monotonically non-decreasing at entry %d",
ErrMalformedIdxFile, i)
}
}
s.count = int(s.fanout[255])
s.hashSize = packHash.Size()
s.namesStart = s.fanoutStart + idxFanoutSize
s.crcStart = s.namesStart + (s.count * s.hashSize)
s.off32Start = s.crcStart + (s.count * 4)
s.off64Start = s.off32Start + (s.count * off32Size)
// Count 64-bit offset entries by scanning the 32-bit offset table
// for entries with the MSB set.
n64, err := s.count64bitOffsets(idxRA)
if err != nil {
return err
}
s.count64 = n64
// The pack checksum sits right after the 64-bit offset table.
packBuf := make([]byte, s.hashSize)
packHashOff := int64(s.off64Start + n64*off64Size)
if _, err := idxRA.ReadAt(packBuf, packHashOff); err != nil {
return fmt.Errorf("cannot read pack checksum: %w", err)
}
if packHash.Compare(packBuf) != 0 {
var got plumbing.Hash
got.ResetBySize(s.hashSize)
_, _ = got.Write(packBuf)
return fmt.Errorf("%w: packfile mismatch: got %q instead of %q",
ErrMalformedIdxFile, got.String(), packHash.String())
}
return nil
}
// Contains reports whether the given hash exists in the index by
// binary-searching the idx names table.
func (s *LazyIndex) Contains(h plumbing.Hash) (bool, error) {
idx, err := s.idx.Acquire()
if err != nil {
return false, err
}
defer s.idx.Release()
_, found, err := s.findHashPos(idx, h)
return found, err
}
// MayContain implements the Index interface. It reports whether the
// index might contain h, using the cached fanout table loaded at
// construction time. No I/O, no lock. False is authoritative ("h is
// not in this pack"); true means call Contains or FindOffset for a
// definitive answer.
func (s *LazyIndex) MayContain(h plumbing.Hash) bool {
first := int(h.Bytes()[0])
var prev uint32
if first > 0 {
prev = s.fanout[first-1]
}
return s.fanout[first] > prev
}
// FindOffset returns the packfile offset for the object with the given hash.
// It returns plumbing.ErrObjectNotFound if the hash is not in the index.
func (s *LazyIndex) FindOffset(h plumbing.Hash) (int64, error) {
idx, err := s.idx.Acquire()
if err != nil {
return 0, err
}
defer s.idx.Release()
pos, found, err := s.findHashPos(idx, h)
if err != nil {
return 0, err
}
if !found {
return 0, plumbing.ErrObjectNotFound
}
off, err := s.offset(idx, pos)
if err != nil {
return 0, err
}
return int64(off), nil
}
// FindCRC32 returns the CRC32 checksum of the object with the given hash.
// It returns plumbing.ErrObjectNotFound if the hash is not in the index.
func (s *LazyIndex) FindCRC32(h plumbing.Hash) (uint32, error) {
idx, err := s.idx.Acquire()
if err != nil {
return 0, err
}
defer s.idx.Release()
pos, found, err := s.findHashPos(idx, h)
if err != nil {
return 0, err
}
if !found {
return 0, plumbing.ErrObjectNotFound
}
return s.crc32(idx, pos)
}
// FindHash returns the object hash stored at the given packfile offset
// by binary-searching the .rev reverse index.
// It returns plumbing.ErrObjectNotFound if no object exists at that offset.
func (s *LazyIndex) FindHash(o int64) (plumbing.Hash, error) {
idx, err := s.idx.Acquire()
if err != nil {
return plumbing.ZeroHash, err
}
defer s.idx.Release()
rev, err := s.rev.Acquire()
if err != nil {
return plumbing.ZeroHash, err
}
defer s.rev.Release()
return s.findHashViaRev(idx, rev, o)
}
// Count returns the total number of objects in the index.
func (s *LazyIndex) Count() (int64, error) {
return int64(s.count), nil
}
// Entries returns an iterator over all index entries in hash order.
// The caller must call Close on the returned iterator to release the
// underlying file reference.
func (s *LazyIndex) Entries() (EntryIter, error) {
idx, err := s.idx.Acquire()
if err != nil {
return nil, err
}
return &scannerEntryIter{s: s, idx: idx}, nil
}
// EntriesWithPrefix implements the Index interface. It returns an
// iterator over entries whose hashes start with prefix. When prefix
// is empty the call is equivalent to Entries; otherwise the
// iterator visits only the fanout-bounded names-table slice
// selected by prefix[0] and stops as soon as a name without prefix
// is read (the names table is sorted by hash).
//
// For a multi-byte prefix the matching entries form a contiguous
// run somewhere within the bucket; binary-search positions the
// iterator at the start of that run so the linear walk only spans
// matches. This mirrors upstream Git's for_each_prefixed_object_in_pack
// which calls bsearch_pack to position before walking forward.
//
// The returned iterator holds an acquired reference to the idx
// SharedFile which is released on Close.
func (s *LazyIndex) EntriesWithPrefix(prefix []byte) (EntryIter, error) {
if len(prefix) == 0 {
return s.Entries()
}
first := int(prefix[0])
var lo int
if first > 0 {
lo = int(s.fanout[first-1])
}
hi := int(s.fanout[first])
if lo >= hi {
return &lazyPrefixIter{}, nil
}
idx, err := s.idx.Acquire()
if err != nil {
return nil, err
}
// Find the leftmost entry in [lo, hi) whose hash is >= prefix
// (padded with zeros to hash size). All matching entries, if
// any, start at this position; the iterator's
// stop-on-first-mismatch then terminates correctly once the
// run ends.
target := make([]byte, s.hashSize)
copy(target, prefix)
var arr [32]byte
buf := arr[:s.hashSize]
bsLo, bsHi := lo, hi
for bsLo < bsHi {
mid := (bsLo + bsHi) >> 1
nameOff := int64(s.namesStart + mid*s.hashSize)
if _, err := idx.ReadAt(buf, nameOff); err != nil {
s.idx.Release()
return nil, fmt.Errorf("read name at pos %d: %w", mid, err)
}
if bytes.Compare(buf, target) < 0 {
bsLo = mid + 1
} else {
bsHi = mid
}
}
if bsLo >= hi {
s.idx.Release()
return &lazyPrefixIter{}, nil
}
return &lazyPrefixIter{
s: s,
idx: idx,
prefix: prefix,
pos: bsLo,
end: hi,
}, nil
}
// EntriesByOffset returns an iterator over all index entries sorted by
// their packfile offset. It reads positions from the .rev file on each
// call to Next, avoiding any up-front allocation or sorting.
//
// The caller must call Close on the returned iterator to release the
// underlying file references.
func (s *LazyIndex) EntriesByOffset() (EntryIter, error) {
idx, err := s.idx.Acquire()
if err != nil {
return nil, err
}
rev, err := s.rev.Acquire()
if err != nil {
s.idx.Release()
return nil, err
}
return &revEntryIter{s: s, idx: idx, rev: rev}, nil
}
// Close releases the underlying shared file handles, preventing future
// operations. If there are active readers they will finish normally;
// the file descriptors close when the last reader is done.
func (s *LazyIndex) Close() error {
return errors.Join(s.idx.Close(), s.rev.Close())
}
// CloseIdleDescriptors releases the idx and rev file descriptors
// without disabling the [LazyIndex]. The FDs close inline when no
// readers are active; otherwise each [sharedfile.SharedFile]
// latches an immediate close on the next refs==0 transition.
// In-flight readers complete normally; subsequent operations
// reopen the FDs on demand and resume normal grace-timer
// behaviour.
//
// Returns the joined error of the inline closes; latched closes
// that fire later are not reported.
func (s *LazyIndex) CloseIdleDescriptors() error {
return errors.Join(s.idx.ReleaseNow(), s.rev.ReleaseNow())
}
// --- internal helpers; all take an io.ReaderAt so the caller controls
// the acquire/release lifecycle. ---
// findHashPos binary-searches the names table for h, returning the flat
// position (0..count-1) if found.
func (s *LazyIndex) findHashPos(idx io.ReaderAt, h plumbing.Hash) (int, bool, error) {
if h.Size() != s.hashSize {
return 0, false, fmt.Errorf("hash size mismatch: %d %d", h.Size(), s.hashSize)
}
first := int(h.Bytes()[0])
var lo int
if first > 0 {
lo = int(s.fanout[first-1])
}
hi := int(s.fanout[first])
if lo >= hi {
return 0, false, nil
}
target := h.Bytes()[:s.hashSize]
var arr [32]byte
buf := arr[:s.hashSize]
for lo < hi {
mid := (lo + hi) >> 1
nameOff := int64(s.namesStart + mid*s.hashSize)
if _, err := idx.ReadAt(buf, nameOff); err != nil {
return 0, false, fmt.Errorf("read name at pos %d: %w", mid, err)
}
cmp := bytes.Compare(target, buf)
switch {
case cmp < 0:
hi = mid
case cmp > 0:
lo = mid + 1
default:
return mid, true, nil
}
}
return 0, false, nil
}
// offset returns the pack offset for the object at position pos.
func (s *LazyIndex) offset(idx io.ReaderAt, pos int) (uint64, error) {
var buf [off32Size]byte
off := int64(s.off32Start + pos*off32Size)
if _, err := idx.ReadAt(buf[:], off); err != nil {
return 0, fmt.Errorf("%w: cannot read offset32: %v", ErrMalformedIdxFile, err)
}
off32 := binary.BigEndian.Uint32(buf[:])
if uint64(off32)&is64bitsMask != 0 {
loIndex := int(uint64(off32) & ^is64bitsMask)
if loIndex >= s.count64 {
return 0, fmt.Errorf("%w: offset64 index %d out of range (have %d entries)",
ErrMalformedIdxFile, loIndex, s.count64)
}
var buf64 [off64Size]byte
off64Pos := int64(s.off64Start + loIndex*off64Size)
if _, err := idx.ReadAt(buf64[:], off64Pos); err != nil {
return 0, fmt.Errorf("%w: cannot read offset64: %v", ErrMalformedIdxFile, err)
}
return binary.BigEndian.Uint64(buf64[:]), nil
}
return uint64(off32), nil
}
// count64bitOffsets scans the 32-bit offset table and returns the number
// of entries whose MSB is set (i.e. that use the 64-bit overflow table).
func (s *LazyIndex) count64bitOffsets(idx io.ReaderAt) (int, error) {
bufp := gsync.GetByteSlice()
defer gsync.PutByteSlice(bufp)
buf := *bufp
// Round down to a multiple of off32Size so we always read whole entries.
buf = buf[:len(buf)&^(off32Size-1)]
var n int
remaining := s.count
pos := int64(s.off32Start)
for remaining > 0 {
chunk := min(remaining*off32Size, len(buf))
if _, err := idx.ReadAt(buf[:chunk], pos); err != nil {
return 0, fmt.Errorf("%w: cannot read offset32 table: %v", ErrMalformedIdxFile, err)
}
for i := 0; i < chunk; i += off32Size {
if binary.BigEndian.Uint32(buf[i:])&uint32(is64bitsMask) != 0 {
n++
}
}
pos += int64(chunk)
remaining -= chunk / off32Size
}
return n, nil
}
// crc32 returns the CRC32 for the object at position pos.
func (s *LazyIndex) crc32(idx io.ReaderAt, pos int) (uint32, error) {
var buf [4]byte
off := int64(s.crcStart + pos*4)
if _, err := idx.ReadAt(buf[:], off); err != nil {
return 0, fmt.Errorf("%w: cannot read CRC32: %v", ErrMalformedIdxFile, err)
}
return binary.BigEndian.Uint32(buf[:]), nil
}
// hashAtPos reads the hash at the given flat position.
func (s *LazyIndex) hashAtPos(idx io.ReaderAt, pos int) (plumbing.Hash, error) {
var arr [32]byte
buf := arr[:s.hashSize]
off := int64(s.namesStart + pos*s.hashSize)
if _, err := idx.ReadAt(buf, off); err != nil {
return plumbing.ZeroHash, fmt.Errorf("read name at pos %d: %w", pos, err)
}
var h plumbing.Hash
h.ResetBySize(s.hashSize)
_, _ = h.Write(buf)
return h, nil
}
func (s *LazyIndex) findHashViaRev(idx, rev io.ReaderAt, want int64) (plumbing.Hash, error) {
lo, hi := 0, s.count
var buf [4]byte
for lo < hi {
mid := (lo + hi) >> 1
revOff := int64(revHeaderSize + mid*4)
if _, err := rev.ReadAt(buf[:], revOff); err != nil {
return plumbing.ZeroHash, fmt.Errorf("read rev entry: %w", err)
}
idxPos := int(binary.BigEndian.Uint32(buf[:]))
if idxPos < 0 || idxPos >= s.count {
return plumbing.ZeroHash, fmt.Errorf("%w: rev entry %d out of range (count %d)",
ErrMalformedIdxFile, idxPos, s.count)
}
got, err := s.offset(idx, idxPos)
if err != nil {
return plumbing.ZeroHash, err
}
switch {
case int64(got) < want:
lo = mid + 1
case int64(got) > want:
hi = mid
default:
return s.hashAtPos(idx, idxPos)
}
}
return plumbing.ZeroHash, plumbing.ErrObjectNotFound
}
// entryAt reads a complete entry at the given flat position.
func (s *LazyIndex) entryAt(idx io.ReaderAt, pos int) (*Entry, error) {
h, err := s.hashAtPos(idx, pos)
if err != nil {
return nil, err
}
off, err := s.offset(idx, pos)
if err != nil {
return nil, err
}
crc, err := s.crc32(idx, pos)
if err != nil {
return nil, err
}
return &Entry{Hash: h, Offset: off, CRC32: crc}, nil
}
// scannerEntryIter iterates over entries in hash order.
// It holds an acquired reference to the idx sharedFile which is
// released when Close is called.
type scannerEntryIter struct {
s *LazyIndex
idx io.ReaderAt // acquired from s.idx
pos int
}
func (it *scannerEntryIter) Next() (*Entry, error) {
if it.idx == nil {
return nil, sharedfile.ErrClosed
}
if it.pos >= it.s.count {
return nil, io.EOF
}
e, err := it.s.entryAt(it.idx, it.pos)
if err != nil {
return nil, err
}
it.pos++
return e, nil
}
func (it *scannerEntryIter) Close() error {
it.pos = it.s.count
if it.idx != nil {
it.s.idx.Release()
it.idx = nil
}
return nil
}
// revEntryIter iterates over entries in packfile-offset order by
// walking the .rev file sequentially. It holds acquired references to
// both the idx and rev sharedFiles, released on Close.
type revEntryIter struct {
s *LazyIndex
idx io.ReaderAt
rev io.ReaderAt
pos int
}
func (it *revEntryIter) Next() (*Entry, error) {
if it.idx == nil || it.rev == nil {
return nil, sharedfile.ErrClosed
}
if it.pos >= it.s.count {
return nil, io.EOF
}
var buf [4]byte
revOff := int64(revHeaderSize + it.pos*4)
if _, err := it.rev.ReadAt(buf[:], revOff); err != nil {
return nil, fmt.Errorf("read rev entry at %d: %w", it.pos, err)
}
idxPos := int(binary.BigEndian.Uint32(buf[:]))
if idxPos < 0 || idxPos >= it.s.count {
return nil, fmt.Errorf("%w: rev entry %d out of range (count %d)",
ErrMalformedIdxFile, idxPos, it.s.count)
}
e, err := it.s.entryAt(it.idx, idxPos)
if err != nil {
return nil, err
}
it.pos++
return e, nil
}
func (it *revEntryIter) Close() error {
it.pos = it.s.count
if it.idx != nil {
it.s.idx.Release()
it.idx = nil
}
if it.rev != nil {
it.s.rev.Release()
it.rev = nil
}
return nil
}
// lazyPrefixIter walks the LazyIndex names table from pos to end,
// yielding entries whose hash starts with prefix. It stops the run
// when a hash without the prefix is read (the table is sorted). It
// holds an acquired reference to the idx SharedFile released on
// Close.
//
// Lifetime: Next may release the iterator's SharedFile reference
// eagerly when the first prefix-mismatched entry is observed —
// further matches are impossible in the sorted table, so holding
// the reference would only add pool pressure. Callers should
// defer Close unconditionally; it is idempotent and the eager
// release is purely an optimisation.
type lazyPrefixIter struct {
s *LazyIndex
idx io.ReaderAt
prefix []byte
pos int
end int
}
func (it *lazyPrefixIter) Next() (*Entry, error) {
if it.idx == nil {
return nil, io.EOF
}
if it.pos >= it.end {
return nil, io.EOF
}
e, err := it.s.entryAt(it.idx, it.pos)
if err != nil {
return nil, err
}
if !bytes.HasPrefix(e.Hash.Bytes(), it.prefix) {
// Past the prefix in the sorted names table; close out so the
// SharedFile reference is released eagerly.
_ = it.Close()
return nil, io.EOF
}
it.pos++
return e, nil
}
func (it *lazyPrefixIter) Close() error {
if it.idx != nil {
it.s.idx.Release()
it.idx = nil
}
return nil
}
package idxfile
import (
"bytes"
"fmt"
"math"
"sort"
"sync"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/utils/binary"
)
// objects implements sort.Interface and uses hash as sorting key.
type objects []Entry
// Writer implements a packfile Observer interface and is used to generate
// indexes.
type Writer struct {
m sync.Mutex
count uint32
checksum plumbing.Hash
objects objects
offset64 uint32
finished bool
index *MemoryIndex
added map[plumbing.Hash]struct{}
}
// Index returns a previously created MemoryIndex or creates a new one if
// needed.
func (w *Writer) Index() (*MemoryIndex, error) {
w.m.Lock()
defer w.m.Unlock()
if w.index == nil {
return w.createIndex()
}
return w.index, nil
}
// Add appends new object data.
func (w *Writer) Add(h plumbing.Hash, pos uint64, crc uint32) {
// Skip unmaterialised delta objects.
if h.IsZero() {
return
}
w.m.Lock()
defer w.m.Unlock()
if w.added == nil {
w.added = make(map[plumbing.Hash]struct{})
}
if _, ok := w.added[h]; !ok {
w.added[h] = struct{}{}
w.objects = append(w.objects, Entry{h, crc, pos})
}
}
// Finished returns true if the writer has finished writing.
func (w *Writer) Finished() bool {
return w.finished
}
// OnHeader implements packfile.Observer interface.
func (w *Writer) OnHeader(count uint32) error {
w.count = count
w.objects = make(objects, 0, count)
return nil
}
// OnInflatedObjectHeader implements packfile.Observer interface.
func (w *Writer) OnInflatedObjectHeader(_ plumbing.ObjectType, _, _ int64) error {
return nil
}
// OnInflatedObjectContent implements packfile.Observer interface.
func (w *Writer) OnInflatedObjectContent(h plumbing.Hash, pos int64, crc uint32, _ []byte) error {
w.Add(h, uint64(pos), crc)
return nil
}
// OnFooter implements packfile.Observer interface.
func (w *Writer) OnFooter(h plumbing.Hash) error {
w.checksum = h
w.finished = true
_, err := w.createIndex()
return err
}
// creatIndex returns a filled MemoryIndex with the information filled by
// the observer callbacks.
func (w *Writer) createIndex() (*MemoryIndex, error) {
if !w.finished {
return nil, fmt.Errorf("the index still hasn't finished building")
}
idx := NewMemoryIndex(w.checksum.Size())
w.index = idx
sort.Sort(w.objects)
// unmap all fans by default
for i := range idx.FanoutMapping {
idx.FanoutMapping[i] = noMapping
}
// Pre-allocate underlying array based on expected number
// of objects.
idx.Names = make([][]byte, 0, len(w.objects))
idx.Offset32 = make([][]byte, 0, len(w.objects))
idx.CRC32 = make([][]byte, 0, len(w.objects))
buf := new(bytes.Buffer)
last := -1
bucket := -1
for i, o := range w.objects {
if o.Hash.Size() != w.checksum.Size() {
return nil, fmt.Errorf("object hash size mismatch: %d instead of %d", o.Hash.Size(), w.checksum.Size())
}
fan := o.Hash.Bytes()[0]
// fill the gaps between fans
for j := last + 1; j < int(fan); j++ {
idx.Fanout[j] = uint32(i)
}
// update the number of objects for this position
idx.Fanout[fan] = uint32(i + 1)
// we move from one bucket to another, update counters and allocate
// memory
if last != int(fan) {
bucket++
idx.FanoutMapping[fan] = bucket
last = int(fan)
idx.Names = append(idx.Names, make([]byte, 0))
idx.Offset32 = append(idx.Offset32, make([]byte, 0))
idx.CRC32 = append(idx.CRC32, make([]byte, 0))
}
idx.Names[bucket] = append(idx.Names[bucket], o.Hash.Bytes()...)
offset := o.Offset
if offset > math.MaxInt32 {
var err error
offset, err = w.addOffset64(offset)
if err != nil {
return nil, err
}
}
buf.Truncate(0)
if err := binary.WriteUint32(buf, uint32(offset)); err != nil {
return nil, err
}
idx.Offset32[bucket] = append(idx.Offset32[bucket], buf.Bytes()...)
buf.Truncate(0)
if err := binary.WriteUint32(buf, o.CRC32); err != nil {
return nil, err
}
idx.CRC32[bucket] = append(idx.CRC32[bucket], buf.Bytes()...)
}
for j := last + 1; j < 256; j++ {
idx.Fanout[j] = uint32(len(w.objects))
}
idx.Version = VersionSupported
idx.PackfileChecksum = w.checksum
return idx, nil
}
func (w *Writer) addOffset64(pos uint64) (uint64, error) {
buf := new(bytes.Buffer)
if err := binary.WriteUint64(buf, pos); err != nil {
return 0, err
}
w.index.Offset64 = append(w.index.Offset64, buf.Bytes()...)
index := uint64(w.offset64 | (1 << 31))
w.offset64++
return index, nil
}
func (o objects) Len() int {
return len(o)
}
func (o objects) Less(i, j int) bool {
cmp := o[i].Hash.Compare(o[j].Hash.Bytes())
return cmp < 0
}
func (o objects) Swap(i, j int) {
o[i], o[j] = o[j], o[i]
}
package index
import (
"bufio"
"bytes"
"errors"
"fmt"
"io"
"strconv"
"time"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/hash"
"github.com/go-git/go-git/v6/utils/binary"
"github.com/go-git/go-git/v6/utils/trace"
)
var (
// DecodeVersionSupported is the range of supported index versions.
DecodeVersionSupported = struct{ Min, Max uint32 }{Min: 2, Max: 4}
// ErrMalformedSignature is returned by Decode when the index header file is
// malformed.
ErrMalformedSignature = errors.New("index decoder: malformed index signature file")
// ErrInvalidChecksum is returned by Decode if the SHA1/SHA256 hash mismatch with
// the read content.
ErrInvalidChecksum = errors.New("index decoder: invalid checksum")
// ErrUnknownExtension is returned when an index extension is encountered that is considered mandatory.
ErrUnknownExtension = errors.New("index decoder: unknown extension")
// ErrMalformedIndexFile is returned when the index file contents are
// structurally invalid.
ErrMalformedIndexFile = errors.New("index decoder: malformed index file")
)
const (
entryHeaderLength = 42
entryExtended = 0x4000
nameMask = 0xfff
intentToAddMask = 1 << 13
skipWorkTreeMask = 1 << 14
)
// A Decoder reads and decodes index files from an input stream.
type Decoder struct {
buf *bufio.Reader
r io.Reader
hash hash.Hash
lastEntry *Entry
skipHash bool
extReader *bufio.Reader
}
// NewDecoder returns a new decoder that reads from r.
func NewDecoder(r io.Reader, h hash.Hash, opts ...Option) *Decoder {
var cfg options
for _, o := range opts {
o(&cfg)
}
buf := bufio.NewReader(r)
d := &Decoder{
buf: buf,
hash: h,
skipHash: cfg.skipHash,
extReader: bufio.NewReader(nil),
}
if d.skipHash {
d.r = buf
} else {
h.Reset()
d.r = io.TeeReader(buf, h)
}
return d
}
// Decode reads the whole index object from its input and stores it in the
// value pointed to by idx.
func (d *Decoder) Decode(idx *Index) error {
var err error
idx.Version, err = validateHeader(d.r)
if err != nil {
return err
}
trace.Internal.Printf("index: decode version %d", idx.Version)
entryCount, err := binary.ReadUint32(d.r)
if err != nil {
return err
}
trace.Internal.Printf("index: decode entry count %d", entryCount)
if err := d.readEntries(idx, int(entryCount)); err != nil {
return err
}
return d.readExtensions(idx)
}
func (d *Decoder) readEntries(idx *Index, count int) error {
for range count {
e, err := d.readEntry(idx)
if err != nil {
return err
}
d.lastEntry = e
idx.Entries = append(idx.Entries, e)
}
return nil
}
func (d *Decoder) readEntry(idx *Index) (*Entry, error) {
e := &Entry{}
var msec, mnsec, sec, nsec uint32
var flags uint16
flow := []any{
&sec, &nsec,
&msec, &mnsec,
&e.Dev,
&e.Inode,
&e.Mode,
&e.UID,
&e.GID,
&e.Size,
}
if err := binary.Read(d.r, flow...); err != nil {
return nil, err
}
e.Hash.ResetBySize(d.hash.Size())
if _, err := e.Hash.ReadFrom(d.r); err != nil {
return nil, err
}
if err := binary.Read(d.r, &flags); err != nil {
return nil, err
}
read := entryHeaderLength + d.hash.Size()
if sec != 0 || nsec != 0 {
e.CreatedAt = time.Unix(int64(sec), int64(nsec))
}
if msec != 0 || mnsec != 0 {
e.ModifiedAt = time.Unix(int64(msec), int64(mnsec))
}
e.Stage = Stage(flags>>12) & 0x3
if flags&entryExtended != 0 {
extended, err := binary.ReadUint16(d.r)
if err != nil {
return nil, err
}
read += 2
e.IntentToAdd = extended&intentToAddMask != 0
e.SkipWorktree = extended&skipWorkTreeMask != 0
}
nameConsumed, err := d.readEntryName(idx, e, flags)
if err != nil {
return nil, err
}
return e, d.padEntry(idx, e, read, nameConsumed)
}
// readEntryName reads the entry path and sets e.Name. It returns the
// number of bytes consumed from the stream for the name portion.
func (d *Decoder) readEntryName(idx *Index, e *Entry, flags uint16) (int, error) {
switch idx.Version {
case 2, 3:
nameLen := flags & nameMask
name, consumed, err := d.doReadEntryName(nameLen)
if err != nil {
return 0, err
}
e.Name = name
return consumed, nil
case 4:
name, err := d.doReadEntryNameV4()
if err != nil {
return 0, err
}
e.Name = name
return 0, nil // V4 has no padding; consumed count unused
default:
return 0, ErrUnsupportedVersion
}
}
// doReadEntryName reads the entry path for V2/V3 indexes. It returns the
// name, the number of bytes consumed from the stream, and any error.
// When nameLen equals nameMask (0xFFF), the name was too long to fit in
// the 12-bit field and the real length is found by scanning for the NUL
// terminator — matching C Git's strlen(name) fallback in create_from_disk.
func (d *Decoder) doReadEntryName(nameLen uint16) (string, int, error) {
if nameLen == nameMask {
name, err := binary.ReadUntil(d.r, '\x00')
if err != nil {
return "", 0, err
}
return string(name), len(name) + 1, nil // +1 for the consumed NUL delimiter
}
name := make([]byte, nameLen)
_, err := io.ReadFull(d.r, name)
return string(name), int(nameLen), err
}
func (d *Decoder) doReadEntryNameV4() (string, error) {
l, err := binary.ReadVariableWidthInt(d.r)
if err != nil {
return "", err
}
var base string
if d.lastEntry != nil {
if l < 0 || int(l) > len(d.lastEntry.Name) {
return "", fmt.Errorf("%w: invalid V4 entry name strip length %d (previous name length: %d)",
ErrMalformedIndexFile, l, len(d.lastEntry.Name))
}
base = d.lastEntry.Name[:len(d.lastEntry.Name)-int(l)]
} else if l > 0 {
return "", fmt.Errorf("%w: non-zero strip length %d on first V4 entry",
ErrMalformedIndexFile, l)
}
name, err := binary.ReadUntil(d.r, '\x00')
if err != nil {
return "", err
}
return base + string(name), nil
}
// padEntry discards NUL padding bytes that follow each V2/V3 entry on
// disk. nameConsumed is the number of stream bytes consumed while reading
// the entry name (which may exceed len(e.Name) when a NUL terminator was
// consumed for long names where the 12-bit length field overflowed).
func (d *Decoder) padEntry(idx *Index, e *Entry, read, nameConsumed int) error {
if idx.Version == 4 {
return nil
}
entrySize := read + len(e.Name)
padLen := 8 - entrySize%8
padLen -= nameConsumed - len(e.Name)
if padLen > 0 {
_, err := io.CopyN(io.Discard, d.r, int64(padLen))
return err
}
return nil
}
func (d *Decoder) readExtensions(idx *Index) error {
// TODO: support 'Split index' and 'Untracked cache' extensions, take in
// count that they are not supported by jgit or libgit
var expected []byte
var peeked []byte
var err error
// we should always be able to peek for 4 bytes (header) + 4 bytes (extlen) + final hash
// if this fails, we know that we're at the end of the index
peekLen := 4 + 4 + d.hash.Size()
for {
if !d.skipHash {
expected = d.hash.Sum(nil)
}
peeked, err = d.buf.Peek(peekLen)
if len(peeked) < peekLen {
trace.Internal.Printf("index: decode peeked %d bytes, less than minimum %d; done reading extensions", len(peeked), peekLen)
// there can't be an extension at this point, so let's bail out
break
}
if err != nil {
return err
}
err = d.readExtension(idx)
if err != nil {
return err
}
}
if !d.skipHash {
trace.Internal.Printf("index: verifying checksum, expected %x", expected)
}
return d.readChecksum(expected)
}
func (d *Decoder) readExtension(idx *Index) error {
var header [4]byte
if _, err := io.ReadFull(d.r, header[:]); err != nil {
return err
}
trace.Internal.Printf("index: decode extension header %s", string(header[:]))
r, err := d.getExtensionReader()
if err != nil {
return err
}
switch {
case bytes.Equal(header[:], treeExtSignature):
trace.Internal.Printf("index: decoding tree extension")
idx.Cache = &Tree{}
extDec := &treeExtensionDecoder{r, d.hash}
if err := extDec.Decode(idx.Cache); err != nil {
return err
}
trace.Internal.Printf("index: tree extension decoded, %d entries", len(idx.Cache.Entries))
case bytes.Equal(header[:], resolveUndoExtSignature):
trace.Internal.Printf("index: decoding resolve-undo extension")
idx.ResolveUndo = &ResolveUndo{}
extDec := &resolveUndoDecoder{r, d.hash}
if err := extDec.Decode(idx.ResolveUndo); err != nil {
return err
}
trace.Internal.Printf("index: resolve-undo extension decoded, %d entries", len(idx.ResolveUndo.Entries))
case bytes.Equal(header[:], endOfIndexEntryExtSignature):
trace.Internal.Printf("index: decoding end-of-index-entry extension")
idx.EndOfIndexEntry = &EndOfIndexEntry{}
extDec := &endOfIndexEntryDecoder{r, d.hash}
if err := extDec.Decode(idx.EndOfIndexEntry); err != nil {
return err
}
trace.Internal.Printf("index: end-of-index-entry extension decoded, offset %d hash %s", idx.EndOfIndexEntry.Offset, idx.EndOfIndexEntry.Hash)
default:
// See https://git-scm.com/docs/index-format, which says:
// If the first byte is 'A'..'Z' the extension is optional and can be ignored.
if header[0] < 'A' || header[0] > 'Z' {
trace.Internal.Printf("index: unknown mandatory extension %s", string(header[:]))
return ErrUnknownExtension
}
trace.Internal.Printf("index: skipping optional unknown extension %s", string(header[:]))
extDec := &unknownExtensionDecoder{r}
if err := extDec.Decode(); err != nil {
return err
}
}
return nil
}
func (d *Decoder) getExtensionReader() (*bufio.Reader, error) {
extLen, err := binary.ReadUint32(d.r)
if err != nil {
return nil, err
}
d.extReader.Reset(&io.LimitedReader{R: d.r, N: int64(extLen)})
return d.extReader, nil
}
func (d *Decoder) readChecksum(expected []byte) error {
var h plumbing.Hash
h.ResetBySize(d.hash.Size())
if _, err := h.ReadFrom(d.r); err != nil {
trace.Internal.Printf("index: checksum read error: %v", err)
return err
}
// A null (all-zero) trailing hash means the checksum was skipped when
// the index was written (git's index.skipHash, 2.40+). Upstream git
// disables verification in this case, so match that even when the
// caller did not opt in via WithSkipHash.
if h.IsZero() {
trace.Internal.Printf("index: null trailing checksum, skipping verification")
return nil
}
if d.skipHash {
trace.Internal.Printf("index: skipping checksum verification (skipHash)")
return nil
}
if h.Compare(expected) != 0 {
trace.Internal.Printf("index: checksum mismatch, expected %x got %s", expected, h)
return ErrInvalidChecksum
}
trace.Internal.Printf("index: checksum ok %s", h)
return nil
}
func validateHeader(r io.Reader) (version uint32, err error) {
s := make([]byte, 4)
if _, err := io.ReadFull(r, s); err != nil {
return 0, err
}
if !bytes.Equal(s, indexSignature) {
return 0, ErrMalformedSignature
}
version, err = binary.ReadUint32(r)
if err != nil {
return 0, err
}
if version < DecodeVersionSupported.Min || version > DecodeVersionSupported.Max {
return 0, ErrUnsupportedVersion
}
return version, err
}
type treeExtensionDecoder struct {
r *bufio.Reader
h hash.Hash
}
func (d *treeExtensionDecoder) Decode(t *Tree) error {
for {
e, err := d.readEntry()
if err != nil {
if err == io.EOF {
return nil
}
return err
}
if e == nil {
continue
}
t.Entries = append(t.Entries, *e)
}
}
func (d *treeExtensionDecoder) readEntry() (*TreeEntry, error) {
e := &TreeEntry{}
path, err := binary.ReadUntil(d.r, '\x00')
if err != nil {
return nil, err
}
e.Path = string(path)
count, err := binary.ReadUntil(d.r, ' ')
if err != nil {
return nil, err
}
i, err := strconv.Atoi(string(count))
if err != nil {
return nil, err
}
e.Entries = i
trees, err := binary.ReadUntil(d.r, '\n')
if err != nil {
return nil, err
}
subtrees, err := strconv.Atoi(string(trees))
if err != nil {
return nil, err
}
e.Trees = subtrees
// An entry can be in an invalidated state and is represented by having a
// negative number in the entry_count field. In this case, there is no
// object name and the next entry starts immediately after the newline.
if i < 0 {
trace.Internal.Printf("index: tree extension entry %q invalidated (entry count %d)", e.Path, i)
return nil, nil
}
e.Hash.ResetBySize(d.h.Size())
_, err = e.Hash.ReadFrom(d.r)
if err != nil {
return nil, err
}
return e, nil
}
type resolveUndoDecoder struct {
r *bufio.Reader
h hash.Hash
}
func (d *resolveUndoDecoder) Decode(ru *ResolveUndo) error {
for {
e, err := d.readEntry()
if err != nil {
if err == io.EOF {
return nil
}
return err
}
ru.Entries = append(ru.Entries, *e)
}
}
func (d *resolveUndoDecoder) readEntry() (*ResolveUndoEntry, error) {
e := &ResolveUndoEntry{
Stages: make(map[Stage]plumbing.Hash),
}
path, err := binary.ReadUntil(d.r, '\x00')
if err != nil {
return nil, err
}
e.Path = string(path)
for i := range 3 {
if err := d.readStage(e, Stage(i+1)); err != nil {
return nil, err
}
}
for s := range e.Stages {
var h plumbing.Hash
h.ResetBySize(d.h.Size())
if _, err := h.ReadFrom(d.r); err != nil {
return nil, err
}
e.Stages[s] = h
}
trace.Internal.Printf("index: resolve-undo entry %q, %d stages", e.Path, len(e.Stages))
return e, nil
}
func (d *resolveUndoDecoder) readStage(e *ResolveUndoEntry, s Stage) error {
ascii, err := binary.ReadUntil(d.r, '\x00')
if err != nil {
return err
}
stage, err := strconv.ParseInt(string(ascii), 8, 64)
if err != nil {
return err
}
if stage != 0 {
e.Stages[s] = plumbing.ZeroHash
}
return nil
}
type endOfIndexEntryDecoder struct {
r *bufio.Reader
h hash.Hash
}
func (d *endOfIndexEntryDecoder) Decode(e *EndOfIndexEntry) error {
var err error
e.Offset, err = binary.ReadUint32(d.r)
if err != nil {
return err
}
e.Hash.ResetBySize(d.h.Size())
_, err = e.Hash.ReadFrom(d.r)
return err
}
type unknownExtensionDecoder struct {
r *bufio.Reader
}
func (d *unknownExtensionDecoder) Decode() error {
_, err := io.Copy(io.Discard, d.r)
return err
}
package index
import (
"bytes"
"errors"
"fmt"
"io"
"sort"
"time"
"github.com/go-git/go-git/v6/plumbing/hash"
"github.com/go-git/go-git/v6/utils/binary"
)
var (
// EncodeVersionSupported is the range of supported index versions
EncodeVersionSupported uint32 = 4
// ErrInvalidTimestamp is returned by Encode if a Index with a Entry with
// negative timestamp values
ErrInvalidTimestamp = errors.New("negative timestamps are not allowed")
)
// An Encoder writes an Index to an output stream.
type Encoder struct {
w io.Writer
hash hash.Hash
lastEntry *Entry
skipHash bool
}
// NewEncoder returns a new encoder that writes to w.
func NewEncoder(w io.Writer, h hash.Hash, opts ...Option) *Encoder {
var cfg options
for _, o := range opts {
o(&cfg)
}
e := &Encoder{
hash: h,
skipHash: cfg.skipHash,
}
if e.skipHash {
e.w = w
} else {
h.Reset()
e.w = io.MultiWriter(w, h)
}
return e
}
// Encode writes the Index to the stream of the encoder.
func (e *Encoder) Encode(idx *Index) error {
return e.encode(idx, true)
}
func (e *Encoder) encode(idx *Index, footer bool) error {
// TODO: support extensions
if idx.Version > EncodeVersionSupported {
return ErrUnsupportedVersion
}
if err := e.encodeHeader(idx); err != nil {
return err
}
if err := e.encodeEntries(idx); err != nil {
return err
}
if footer {
return e.encodeFooter()
}
return nil
}
func (e *Encoder) encodeHeader(idx *Index) error {
return binary.Write(e.w,
indexSignature,
idx.Version,
uint32(len(idx.Entries)),
)
}
func (e *Encoder) encodeEntries(idx *Index) error {
sort.Sort(byNameAndStage(idx.Entries))
for _, entry := range idx.Entries {
if err := e.encodeEntry(idx, entry); err != nil {
return err
}
entryLength := entryHeaderLength + e.hash.Size()
if entry.IntentToAdd || entry.SkipWorktree {
entryLength += 2
}
wrote := entryLength + len(entry.Name)
if err := e.padEntry(idx, wrote); err != nil {
return err
}
}
return nil
}
func (e *Encoder) encodeEntry(idx *Index, entry *Entry) error {
sec, nsec, err := e.timeToUint32(&entry.CreatedAt)
if err != nil {
return err
}
msec, mnsec, err := e.timeToUint32(&entry.ModifiedAt)
if err != nil {
return err
}
flags := uint16(entry.Stage&0x3) << 12
if l := len(entry.Name); l < nameMask {
flags |= uint16(l)
} else {
flags |= nameMask
}
flagsFlow := []any{flags}
flow := make([]any, 0, 11+len(flagsFlow))
flow = append(flow,
sec, nsec,
msec, mnsec,
entry.Dev,
entry.Inode,
entry.Mode,
entry.UID,
entry.GID,
entry.Size,
entry.Hash.Bytes(),
)
if entry.IntentToAdd || entry.SkipWorktree {
var extendedFlags uint16
if entry.IntentToAdd {
extendedFlags |= intentToAddMask
}
if entry.SkipWorktree {
extendedFlags |= skipWorkTreeMask
}
flagsFlow = []any{flags | entryExtended, extendedFlags}
}
flow = append(flow, flagsFlow...)
if err := binary.Write(e.w, flow...); err != nil {
return err
}
switch idx.Version {
case 2, 3:
err = e.encodeEntryName(entry)
case 4:
err = e.encodeEntryNameV4(entry)
default:
err = ErrUnsupportedVersion
}
return err
}
func (e *Encoder) encodeEntryName(entry *Entry) error {
return binary.Write(e.w, []byte(entry.Name))
}
func (e *Encoder) encodeEntryNameV4(entry *Entry) error {
// V4 prefix compression: find the longest common prefix between the
// previous entry's name and the current one. The strip length tells
// the decoder how many bytes to remove from the end of the previous
// name, and the suffix is the remainder of the current name.
prefix := 0
if e.lastEntry != nil {
prefix = commonPrefixLen(e.lastEntry.Name, entry.Name)
}
stripLen := 0
if e.lastEntry != nil {
stripLen = len(e.lastEntry.Name) - prefix
}
e.lastEntry = entry
if err := binary.WriteVariableWidthInt(e.w, int64(stripLen)); err != nil {
return err
}
suffix := entry.Name[prefix:]
return binary.Write(e.w, append([]byte(suffix), '\x00'))
}
// commonPrefixLen returns the length of the longest common byte prefix
// between a and b.
func commonPrefixLen(a, b string) int {
n := min(len(b), len(a))
for i := range n {
if a[i] != b[i] {
return i
}
}
return n
}
func (e *Encoder) encodeRawExtension(signature string, data []byte) error {
if len(signature) != 4 {
return fmt.Errorf("invalid signature length")
}
_, err := e.w.Write([]byte(signature))
if err != nil {
return err
}
err = binary.WriteUint32(e.w, uint32(len(data)))
if err != nil {
return err
}
_, err = e.w.Write(data)
if err != nil {
return err
}
return nil
}
func (e *Encoder) timeToUint32(t *time.Time) (uint32, uint32, error) {
if t.IsZero() {
return 0, 0, nil
}
if t.Unix() < 0 || t.UnixNano() < 0 {
return 0, 0, ErrInvalidTimestamp
}
return uint32(t.Unix()), uint32(t.Nanosecond()), nil
}
func (e *Encoder) padEntry(idx *Index, wrote int) error {
if idx.Version == 4 {
return nil
}
padLen := 8 - wrote%8
_, err := e.w.Write(bytes.Repeat([]byte{'\x00'}, padLen))
return err
}
func (e *Encoder) encodeFooter() error {
if e.skipHash {
_, err := e.w.Write(make([]byte, e.hash.Size()))
return err
}
return binary.Write(e.w, e.hash.Sum(nil))
}
type byNameAndStage []*Entry
func (l byNameAndStage) Len() int { return len(l) }
func (l byNameAndStage) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
func (l byNameAndStage) Less(i, j int) bool {
if l[i].Name == l[j].Name {
return l[i].Stage < l[j].Stage
}
return l[i].Name < l[j].Name
}
package index
import (
"bytes"
"errors"
"fmt"
"path/filepath"
"strings"
"time"
"github.com/go-git/go-git/v6/internal/pathutil"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/filemode"
)
var (
// ErrUnsupportedVersion is returned by Decode when the index file version
// is not supported.
ErrUnsupportedVersion = errors.New("unsupported version")
// ErrEntryNotFound is returned by Index.Entry, if an entry is not found.
ErrEntryNotFound = errors.New("entry not found")
indexSignature = []byte{'D', 'I', 'R', 'C'}
treeExtSignature = []byte{'T', 'R', 'E', 'E'}
resolveUndoExtSignature = []byte{'R', 'E', 'U', 'C'}
endOfIndexEntryExtSignature = []byte{'E', 'O', 'I', 'E'}
)
// Stage during merge
type Stage int
const (
// Merged is the default stage, fully merged
Merged Stage = 1
// AncestorMode is the base revision
AncestorMode Stage = 1
// OurMode is the first tree revision, ours
OurMode Stage = 2
// TheirMode is the second tree revision, theirs
TheirMode Stage = 3
)
// Index contains the information about which objects are currently checked out
// in the worktree, having information about the working files. Changes in
// worktree are detected using this Index. The Index is also used during merges
type Index struct {
// Version is index version
Version uint32
// Entries collection of entries represented by this Index. The order of
// this collection is not guaranteed
Entries []*Entry
// Cache represents the 'Cached tree' extension
Cache *Tree
// ResolveUndo represents the 'Resolve undo' extension
ResolveUndo *ResolveUndo
// EndOfIndexEntry represents the 'End of Index Entry' extension
EndOfIndexEntry *EndOfIndexEntry
// ModTime is the modification time of the index file
ModTime time.Time
}
// Add creates a new Entry and returns it. The caller should first check
// that another entry with the same path does not exist.
//
// The path is validated against pathutil.ValidTreePath: the index feeds
// future trees, so a name that the tree-side gates would reject must
// not enter the index in the first place. Mirrors the FindEntry /
// TreeWalker / TreeEntryFile chokepoints on the read side.
func (i *Index) Add(path string) (*Entry, error) {
if err := pathutil.ValidTreePath(path); err != nil {
return nil, err
}
e := &Entry{
Name: filepath.ToSlash(path),
}
i.Entries = append(i.Entries, e)
return e, nil
}
// Entry returns the entry that match the given path, if any.
func (i *Index) Entry(path string) (*Entry, error) {
path = filepath.ToSlash(path)
for _, e := range i.Entries {
if e.Name == path {
return e, nil
}
}
return nil, ErrEntryNotFound
}
// Remove remove the entry that match the give path and returns deleted entry.
func (i *Index) Remove(path string) (*Entry, error) {
path = filepath.ToSlash(path)
for index, e := range i.Entries {
if e.Name == path {
i.Entries = append(i.Entries[:index], i.Entries[index+1:]...)
return e, nil
}
}
return nil, ErrEntryNotFound
}
// Glob returns the all entries matching pattern or nil if there is no matching
// entry. The syntax of patterns is the same as in filepath.Glob.
func (i *Index) Glob(pattern string) (matches []*Entry, err error) {
pattern = filepath.ToSlash(pattern)
for _, e := range i.Entries {
m, err := match(pattern, e.Name)
if err != nil {
return nil, err
}
if m {
matches = append(matches, e)
}
}
return matches, err
}
// String is equivalent to `git ls-files --stage --debug`
func (i *Index) String() string {
buf := bytes.NewBuffer(nil)
for _, e := range i.Entries {
buf.WriteString(e.String())
}
return buf.String()
}
// Entry represents a single file (or stage of a file) in the cache. An entry
// represents exactly one stage of a file. If a file path is unmerged then
// multiple Entry instances may appear for the same path name.
type Entry struct {
// Hash is the SHA1 of the represented file
Hash plumbing.Hash
// Name is the Entry path name relative to top level directory
Name string
// CreatedAt time when the tracked path was created
CreatedAt time.Time
// ModifiedAt time when the tracked path was changed
ModifiedAt time.Time
// Dev and Inode of the tracked path
Dev, Inode uint32
// Mode of the path
Mode filemode.FileMode
// UID and GID, userid and group id of the owner
UID, GID uint32
// Size is the length in bytes for regular files
Size uint32
// Stage on a merge is defines what stage is representing this entry
// https://git-scm.com/book/en/v2/Git-Tools-Advanced-Merging
Stage Stage
// SkipWorktree used in sparse checkouts
// https://git-scm.com/docs/git-read-tree#_sparse_checkout
SkipWorktree bool
// IntentToAdd record only the fact that the path will be added later
// https://git-scm.com/docs/git-add ("git add -N")
IntentToAdd bool
}
func (e Entry) String() string {
buf := bytes.NewBuffer(nil)
fmt.Fprintf(buf, "%06o %s %d\t%s\n", e.Mode, e.Hash, e.Stage, e.Name)
fmt.Fprintf(buf, " ctime: %d:%d\n", e.CreatedAt.Unix(), e.CreatedAt.Nanosecond())
fmt.Fprintf(buf, " mtime: %d:%d\n", e.ModifiedAt.Unix(), e.ModifiedAt.Nanosecond())
fmt.Fprintf(buf, " dev: %d\tino: %d\n", e.Dev, e.Inode)
fmt.Fprintf(buf, " uid: %d\tgid: %d\n", e.UID, e.GID)
fmt.Fprintf(buf, " size: %d\tflags: %x\n", e.Size, 0)
return buf.String()
}
// Tree contains pre-computed hashes for trees that can be derived from the
// index. It helps speed up tree object generation from index for a new commit.
type Tree struct {
Entries []TreeEntry
}
// TreeEntry entry of a cached Tree
type TreeEntry struct {
// Path component (relative to its parent directory)
Path string
// Entries is the number of entries in the index that is covered by the tree
// this entry represents.
Entries int
// Trees is the number that represents the number of subtrees this tree has
Trees int
// Hash object name for the object that would result from writing this span
// of index as a tree.
Hash plumbing.Hash
}
// ResolveUndo is used when a conflict is resolved (e.g. with "git add path"),
// these higher stage entries are removed and a stage-0 entry with proper
// resolution is added. When these higher stage entries are removed, they are
// saved in the resolve undo extension.
type ResolveUndo struct {
Entries []ResolveUndoEntry
}
// ResolveUndoEntry contains the information about a conflict when is resolved
type ResolveUndoEntry struct {
Path string
Stages map[Stage]plumbing.Hash
}
// EndOfIndexEntry is the End of Index Entry (EOIE) is used to locate the end of
// the variable length index entries and the beginning of the extensions. Code
// can take advantage of this to quickly locate the index extensions without
// having to parse through all of the index entries.
//
// Because it must be able to be loaded before the variable length cache
// entries and other index extensions, this extension must be written last.
type EndOfIndexEntry struct {
// Offset to the end of the index entries
Offset uint32
// Hash is a SHA-1 over the extension types and their sizes (but not
// their contents).
Hash plumbing.Hash
}
// SkipUnless applies patterns in the form of A, A/B, A/B/C
// to the index to prevent the files from being checked out.
// Files whose names match one of the patterns have SkipWorktree cleared;
// all other files have it set. This handles sparse-checkout dir switching
// correctly: files moving into the active set are un-skipped.
func (i *Index) SkipUnless(patterns []string) {
for _, e := range i.Entries {
var include bool
for _, pattern := range patterns {
if strings.HasPrefix(e.Name, pattern) {
include = true
break
}
}
if include {
e.SkipWorktree = false
} else {
e.SkipWorktree = true
}
}
}
package index
import (
"path/filepath"
"runtime"
"unicode/utf8"
)
// match is filepath.Match with support to match fullpath and not only filenames
// code from:
// https://github.com/golang/go/blob/39852bf4cce6927e01d0136c7843f65a801738cb/src/path/filepath/match.go#L44-L224
func match(pattern, name string) (matched bool, err error) {
Pattern:
for len(pattern) > 0 {
var star bool
var chunk string
star, chunk, pattern = scanChunk(pattern)
// Look for match at current position.
t, ok, err := matchChunk(chunk, name)
// if we're the last chunk, make sure we've exhausted the name
// otherwise we'll give a false result even if we could still match
// using the star
if ok && (len(t) == 0 || len(pattern) > 0) {
name = t
continue
}
if err != nil {
return false, err
}
if star {
// Look for match skipping i+1 bytes.
// Cannot skip /.
for i := 0; i < len(name); i++ {
t, ok, err := matchChunk(chunk, name[i+1:])
if ok {
// if we're the last chunk, make sure we exhausted the name
if len(pattern) == 0 && len(t) > 0 {
continue
}
name = t
continue Pattern
}
if err != nil {
return false, err
}
}
}
return false, nil
}
return len(name) == 0, nil
}
// scanChunk gets the next segment of pattern, which is a non-star string
// possibly preceded by a star.
func scanChunk(pattern string) (star bool, chunk, rest string) {
for len(pattern) > 0 && pattern[0] == '*' {
pattern = pattern[1:]
star = true
}
inrange := false
var i int
Scan:
for i = 0; i < len(pattern); i++ {
switch pattern[i] {
case '\\':
if runtime.GOOS != "windows" {
// error check handled in matchChunk: bad pattern.
if i+1 < len(pattern) {
i++
}
}
case '[':
inrange = true
case ']':
inrange = false
case '*':
if !inrange {
break Scan
}
}
}
return star, pattern[0:i], pattern[i:]
}
// matchChunk checks whether chunk matches the beginning of s.
// If so, it returns the remainder of s (after the match).
// Chunk is all single-character operators: literals, char classes, and ?.
func matchChunk(chunk, s string) (rest string, ok bool, err error) {
for len(chunk) > 0 {
if len(s) == 0 {
return rest, ok, err
}
switch chunk[0] {
case '[':
// character class
r, n := utf8.DecodeRuneInString(s)
s = s[n:]
chunk = chunk[1:]
// We can't end right after '[', we're expecting at least
// a closing bracket and possibly a caret.
if len(chunk) == 0 {
err = filepath.ErrBadPattern
return rest, ok, err
}
// possibly negated
negated := chunk[0] == '^'
if negated {
chunk = chunk[1:]
}
// parse all ranges
match := false
nrange := 0
for {
if len(chunk) > 0 && chunk[0] == ']' && nrange > 0 {
chunk = chunk[1:]
break
}
var lo, hi rune
if lo, chunk, err = getEsc(chunk); err != nil {
return rest, ok, err
}
hi = lo
if chunk[0] == '-' {
if hi, chunk, err = getEsc(chunk[1:]); err != nil {
return rest, ok, err
}
}
if lo <= r && r <= hi {
match = true
}
nrange++
}
if match == negated {
return rest, ok, err
}
case '?':
_, n := utf8.DecodeRuneInString(s)
s = s[n:]
chunk = chunk[1:]
case '\\':
if runtime.GOOS != "windows" {
chunk = chunk[1:]
if len(chunk) == 0 {
err = filepath.ErrBadPattern
return rest, ok, err
}
}
fallthrough
default:
if chunk[0] != s[0] {
return rest, ok, err
}
s = s[1:]
chunk = chunk[1:]
}
}
return s, true, nil
}
// getEsc gets a possibly-escaped character from chunk, for a character class.
func getEsc(chunk string) (r rune, nchunk string, err error) {
if len(chunk) == 0 || chunk[0] == '-' || chunk[0] == ']' {
err = filepath.ErrBadPattern
return r, nchunk, err
}
if chunk[0] == '\\' && runtime.GOOS != "windows" {
chunk = chunk[1:]
if len(chunk) == 0 {
err = filepath.ErrBadPattern
return r, nchunk, err
}
}
r, n := utf8.DecodeRuneInString(chunk)
if r == utf8.RuneError && n == 1 {
err = filepath.ErrBadPattern
}
nchunk = chunk[n:]
if len(nchunk) == 0 {
err = filepath.ErrBadPattern
}
return r, nchunk, err
}
package index
// Option configures an Encoder or Decoder.
type Option func(*options)
type options struct {
skipHash bool
}
// WithSkipHash disables checksum computation when encoding and checksum
// verification when decoding. This corresponds to git's index.skipHash
// configuration (git 2.40+), where git writes an all-zero checksum for
// performance on large repositories and skips verification on read.
func WithSkipHash() Option {
return func(o *options) {
o.skipHash = true
}
}
package objfile
import (
"errors"
"io"
"strconv"
"github.com/go-git/go-git/v6/plumbing"
format "github.com/go-git/go-git/v6/plumbing/format/config"
"github.com/go-git/go-git/v6/plumbing/format/packfile"
"github.com/go-git/go-git/v6/utils/sync"
)
// Errors returned by the objfile package.
var (
ErrClosed = errors.New("objfile: already closed")
ErrHeader = errors.New("objfile: invalid header")
ErrHeaderTooLong = errors.New("objfile: header exceeds maximum length")
ErrHeaderNotRead = errors.New("objfile: Header must be called before Read")
ErrNegativeSize = errors.New("objfile: negative object size")
)
// maxHeaderLen mirrors canonical Git's MAX_HEADER_LEN [1]. The type,
// delimiter, size, and trailing NUL of a loose-object header must fit
// within this many inflated bytes.
//
// [1]: https://github.com/git/git/blob/v2.54.0/object-file.c#L34
const maxHeaderLen = 32
// Reader reads and decodes compressed objfile data from a provided io.Reader.
// Reader implements io.ReadCloser. Close should be called when finished with
// the Reader. Close will not close the underlying io.Reader.
type Reader struct {
multi io.Reader
zlib *sync.ZLibReader
hasher plumbing.Hasher
objectFormat format.ObjectFormat
closed bool
}
// NewReader returns a new Reader reading from r and hashing objects with the
// given object format.
func NewReader(r io.Reader, objectFormat format.ObjectFormat) (*Reader, error) {
zlib, err := sync.GetZlibReader(r)
if err != nil {
return nil, packfile.ErrZLib.AddDetails("%s", err.Error())
}
return &Reader{
zlib: zlib,
objectFormat: objectFormat,
}, nil
}
// Header reads the type and the size of object, and prepares the reader for read
func (r *Reader) Header() (t plumbing.ObjectType, size int64, err error) {
budget := maxHeaderLen
var raw []byte
raw, budget, err = r.readUntil(' ', budget)
if err != nil {
return t, size, err
}
t, err = plumbing.ParseObjectType(string(raw))
if err != nil {
return t, size, err
}
raw, _, err = r.readUntil(0, budget)
if err != nil {
return t, size, err
}
size, err = strconv.ParseInt(string(raw), 10, 64)
if err != nil {
err = ErrHeader
return t, size, err
}
defer r.prepareForRead(t, size)
return t, size, err
}
// readUntil reads one inflated byte at a time from r.zlib until it encounters
// delim, the budget is exhausted, or an error. budget caps the total number
// of bytes consumed from r.zlib, including delim; it mirrors canonical Git's
// MAX_HEADER_LEN bound applied across the full loose-object header.
func (r *Reader) readUntil(delim byte, budget int) ([]byte, int, error) {
var buf [1]byte
value := make([]byte, 0, 16)
for {
if budget <= 0 {
return nil, 0, ErrHeaderTooLong
}
if n, err := r.zlib.Read(buf[:]); err != nil && (err != io.EOF || n == 0) {
if err == io.EOF {
return nil, 0, ErrHeader
}
return nil, 0, err
}
budget--
if buf[0] == delim {
return value, budget, nil
}
value = append(value, buf[0])
}
}
func (r *Reader) prepareForRead(t plumbing.ObjectType, size int64) {
r.hasher = plumbing.NewHasher(r.objectFormat, t, size)
r.multi = io.TeeReader(r.zlib, r.hasher)
}
// Read reads len(p) bytes into p from the object data stream. It returns
// the number of bytes read (0 <= n <= len(p)) and any error encountered. Even
// if Read returns n < len(p), it may use all of p as scratch space during the
// call.
//
// If Read encounters the end of the data stream it will return err == io.EOF,
// either in the current call if n > 0 or in a subsequent call.
//
// Read returns ErrHeaderNotRead if Header has not been called successfully.
func (r *Reader) Read(p []byte) (n int, err error) {
if r.multi == nil {
return 0, ErrHeaderNotRead
}
return r.multi.Read(p)
}
// Hash returns the hash of the object data stream that has been read so far.
// It returns a zero plumbing.Hash carrying the Reader's configured object
// format if Header has not been called successfully — the format matters
// because [plumbing.Hash] encodes it internally and the result feeds
// serialisers that emit a format-sized byte slice.
func (r *Reader) Hash() plumbing.Hash {
if r.multi == nil {
var h plumbing.Hash
h.ResetBySize(r.objectFormat.Size())
return h
}
return r.hasher.Sum()
}
// Close releases any resources consumed by the Reader. Calling Close does not
// close the wrapped io.Reader originally passed to NewReader.
func (r *Reader) Close() error {
if r.closed {
return nil
}
r.closed = true
defer sync.PutZlibReader(r.zlib)
return r.zlib.Close()
}
package objfile
import (
"errors"
"io"
"strconv"
"github.com/go-git/go-git/v6/plumbing"
format "github.com/go-git/go-git/v6/plumbing/format/config"
"github.com/go-git/go-git/v6/utils/sync"
)
// ErrOverflow is returned when the declared data length is exceeded.
var ErrOverflow = errors.New("objfile: declared data length exceeded (overflow)")
// Writer writes and encodes data in compressed objfile format to a provided
// io.Writer. Close should be called when finished with the Writer. Close will
// not close the underlying io.Writer.
type Writer struct {
raw io.Writer
hasher plumbing.Hasher
multi io.Writer
zlib sync.ZlibWriter
objectFormat format.ObjectFormat
closed bool
pending int64 // number of unwritten bytes
closeErr error
}
// NewWriter returns a new Writer writing to w and hashing objects with the
// given object format.
//
// The returned Writer implements io.WriteCloser. Close should be called when
// finished with the Writer. Close will not close the underlying io.Writer.
func NewWriter(w io.Writer, objectFormat format.ObjectFormat) *Writer {
zlib := sync.GetZlibWriter(w)
return &Writer{
raw: w,
zlib: zlib,
objectFormat: objectFormat,
}
}
// WriteHeader writes the type and the size and prepares to accept the
// object's contents. If an invalid t is provided, plumbing.ErrInvalidType
// is returned. If a negative size is provided, ErrNegativeSize is
// returned. If the encoded header exceeds maxHeaderLen,
// ErrHeaderTooLong is returned, mirroring the reader's bound.
func (w *Writer) WriteHeader(t plumbing.ObjectType, size int64) error {
if !t.Valid() {
return plumbing.ErrInvalidType
}
return w.writeHeader(t, t.Bytes(), size)
}
func (w *Writer) writeHeader(t plumbing.ObjectType, typeBytes []byte, size int64) error {
if size < 0 {
return ErrNegativeSize
}
b := make([]byte, 0, maxHeaderLen)
b = append(b, typeBytes...)
b = append(b, ' ')
b = strconv.AppendInt(b, size, 10)
b = append(b, 0)
if len(b) > maxHeaderLen {
return ErrHeaderTooLong
}
defer w.prepareForWrite(t, size)
_, err := w.zlib.Write(b)
return err
}
func (w *Writer) prepareForWrite(t plumbing.ObjectType, size int64) {
w.pending = size
w.hasher = plumbing.NewHasher(w.objectFormat, t, size)
w.multi = io.MultiWriter(w.zlib, w.hasher)
}
// Write writes the object's contents. Write returns the error ErrOverflow if
// more than size bytes are written after WriteHeader.
func (w *Writer) Write(p []byte) (n int, err error) {
if w.closed {
return 0, ErrClosed
}
overwrite := false
if int64(len(p)) > w.pending {
p = p[0:w.pending]
overwrite = true
}
n, err = w.multi.Write(p)
w.pending -= int64(n)
if err == nil && overwrite {
err = ErrOverflow
return n, err
}
return n, err
}
// Hash returns the hash of the object data stream that has been written so far.
// It can be called before or after Close.
func (w *Writer) Hash() plumbing.Hash {
return w.hasher.Sum() // Not yet closed, return hash of data written so far
}
// Close releases any resources consumed by the Writer.
//
// Calling Close does not close the wrapped io.Writer originally passed to
// NewWriter.
//
// It returns an error, if any. Close will return the same error if called
// multiple times.
func (w *Writer) Close() error {
if w.closed {
return w.closeErr
}
defer sync.PutZlibWriter(w.zlib)
if err := w.zlib.Close(); err != nil {
w.closeErr = err
return err
}
w.closed = true
return nil
}
package packfile
import (
"errors"
"fmt"
"io"
"time"
"github.com/go-git/go-git/v6/config"
formatcfg "github.com/go-git/go-git/v6/plumbing/format/config"
"github.com/go-git/go-git/v6/plumbing/storer"
"github.com/go-git/go-git/v6/utils/ioutil"
"github.com/go-git/go-git/v6/utils/trace"
)
var signature = []byte{'P', 'A', 'C', 'K'}
const (
// VersionSupported is the packfile version supported by this package
VersionSupported uint32 = 2
firstLengthBits = uint8(4) // the first byte into object header has 4 bits to store the length
lengthBits = uint8(7) // subsequent bytes has 7 bits to store the length
maskFirstLength = 15 // 0000 1111
maskContinue = 0x80 // 1000 0000
maskLength = uint8(127) // 0111 1111
)
// UpdateObjectStorage updates the storer with the objects in the given
// packfile.
func UpdateObjectStorage(s storer.Storer, packfile io.Reader) error {
if trace.Performance.Enabled() {
start := time.Now()
defer func() {
trace.Performance.Printf("performance: %.9f s: update_obj_storage", time.Since(start).Seconds())
}()
}
if pw, ok := s.(storer.PackfileWriter); ok {
return WritePackfileToObjectStorage(pw, packfile)
}
of := formatcfg.DefaultObjectFormat
if c, ok := s.(config.ConfigStorer); ok {
cfg, err := c.Config()
if err == nil {
of = cfg.Extensions.ObjectFormat
}
}
p := NewParser(packfile, WithStorage(s), WithObjectFormat(of))
_, err := p.Parse()
return err
}
// ErrPromisorPacksUnsupported is returned when a packfile from a promisor
// remote would be stored by a storer that writes packfiles but cannot record
// them as promisor packs. Writing it unmarked would leave a repository whose
// fsck reports broken links and whose gc fails, so the write is refused.
var ErrPromisorPacksUnsupported = errors.New("storage writes packfiles but cannot record them as promisor packs")
// SupportsPromisorPacks reports whether a packfile from a promisor remote can be
// stored without losing the fact that it came from one.
//
// Storage that records promisor packs qualifies. So does storage that does not
// write packfiles at all: it stores objects individually, so there is no pack to
// mark and nothing to lose — in-memory storage works this way. What does not
// qualify is storage that writes a packfile but cannot mark it, which is exactly
// how an unmarked pack of deliberately absent objects reaches disk.
func SupportsPromisorPacks(s storer.Storer) bool {
if _, ok := s.(storer.PromisorPackfileWriter); ok {
return true
}
_, writesPacks := s.(storer.PackfileWriter)
return !writesPacks
}
// UpdatePromisorObjectStorage is UpdateObjectStorage for a packfile received
// from a promisor remote, as a filtered (partial clone) fetch returns. The pack
// is recorded as a promisor pack so that the objects the filter excluded are
// understood to be promised by that remote rather than missing.
//
// Storage that writes packfiles without being able to mark them is refused with
// ErrPromisorPacksUnsupported rather than silently producing the corruption this
// marking exists to prevent. Storage that writes no packfiles at all stores the
// objects individually, where there is no marking to lose.
func UpdatePromisorObjectStorage(s storer.Storer, packfile io.Reader, marker string) error {
if trace.Performance.Enabled() {
start := time.Now()
defer func() {
trace.Performance.Printf("performance: %.9f s: update_promisor_obj_storage", time.Since(start).Seconds())
}()
}
pw, ok := s.(storer.PromisorPackfileWriter)
if !ok {
if !SupportsPromisorPacks(s) {
return ErrPromisorPacksUnsupported
}
return UpdateObjectStorage(s, packfile)
}
w, err := pw.PromisorPackfileWriter(marker)
if err != nil {
return err
}
return copyPackfile(w, packfile)
}
// WritePackfileToObjectStorage writes all the packfile objects into the given
// object storage.
func WritePackfileToObjectStorage(
sw storer.PackfileWriter,
packfile io.Reader,
) (err error) {
w, err := sw.PackfileWriter()
if err != nil {
return err
}
return copyPackfile(w, packfile)
}
func copyPackfile(w io.WriteCloser, packfile io.Reader) (err error) {
defer ioutil.CheckClose(w, &err)
n, err := ioutil.CopyBufferPool(w, packfile)
if err == nil && n == 0 {
return ErrEmptyPackfile
}
return err
}
// ValidateOFSDeltaBase enforces the canonical-Git invariant on an
// OFS-delta's encoded negative offset: the resolved base offset
// (deltaOffset - negativeOffset) must be strictly positive (past the
// 12-byte pack header) and strictly less than deltaOffset, since an
// OFS-delta can only reference an earlier entry in the same pack.
//
// Mirrors canonical Git's predicate in packfile.c[1]:
//
// base_offset = delta_obj_offset - base_offset;
// if (base_offset <= 0 || base_offset >= delta_obj_offset)
// return 0; /* out of bound */
//
// Returns a wrapped ErrMalformedPackfile when the bounds are violated;
// returns nil otherwise.
//
// [1]: https://github.com/git/git/blob/v2.54.0/packfile.c#L1289-L1290
func ValidateOFSDeltaBase(deltaOffset, negativeOffset int64) error {
if negativeOffset <= 0 || negativeOffset >= deltaOffset {
return fmt.Errorf("%w: invalid OFS delta offset", ErrMalformedPackfile)
}
return nil
}
package packfile
const (
blksz = 16
maxChainLength = 64
)
// deltaIndex is a modified version of JGit's DeltaIndex adapted to our current
// design.
type deltaIndex struct {
table []int
entries []int
mask int
}
func (idx *deltaIndex) init(buf []byte) {
scanner := newDeltaIndexScanner(buf, len(buf))
idx.mask = scanner.mask
idx.table = scanner.table
idx.entries = make([]int, countEntries(scanner)+1)
idx.copyEntries(scanner)
}
// findMatch returns the offset of src where the block starting at tgtOffset
// is and the length of the match. A length of 0 means there was no match. A
// length of -1 means the src length is lower than the blksz and whatever
// other positive length is the length of the match in bytes.
func (idx *deltaIndex) findMatch(src, tgt []byte, tgtOffset int) (srcOffset, l int) {
if len(tgt) < tgtOffset+s {
return 0, len(tgt) - tgtOffset
}
if len(src) < blksz {
return 0, -1
}
h := hashBlock(tgt, tgtOffset)
tIdx := h & idx.mask
eIdx := idx.table[tIdx]
if eIdx == 0 {
return srcOffset, l
}
srcOffset = idx.entries[eIdx]
l = matchLength(src, tgt, tgtOffset, srcOffset)
return srcOffset, l
}
func matchLength(src, tgt []byte, otgt, osrc int) (l int) {
lensrc := len(src)
lentgt := len(tgt)
for (osrc < lensrc && otgt < lentgt) && src[osrc] == tgt[otgt] {
l++
osrc++
otgt++
}
return l
}
func countEntries(scan *deltaIndexScanner) (cnt int) {
// Figure out exactly how many entries we need. As we do the
// enumeration truncate any delta chains longer than what we
// are willing to scan during encode. This keeps the encode
// logic linear in the size of the input rather than quadratic.
for i := 0; i < len(scan.table); i++ {
h := scan.table[i]
if h == 0 {
continue
}
size := 0
for {
size++
if size == maxChainLength {
scan.next[h] = 0
break
}
h = scan.next[h]
if h == 0 {
break
}
}
cnt += size
}
return cnt
}
func (idx *deltaIndex) copyEntries(scanner *deltaIndexScanner) {
// Rebuild the entries list from the scanner, positioning all
// blocks in the same hash chain next to each other. We can
// then later discard the next list, along with the scanner.
//
next := 1
for i := 0; i < len(idx.table); i++ {
h := idx.table[i]
if h == 0 {
continue
}
idx.table[i] = next
for {
idx.entries[next] = scanner.entries[h]
next++
h = scanner.next[h]
if h == 0 {
break
}
}
}
}
type deltaIndexScanner struct {
table []int
entries []int
next []int
mask int
count int
}
func newDeltaIndexScanner(buf []byte, size int) *deltaIndexScanner {
size -= size % blksz
worstCaseBlockCnt := size / blksz
if worstCaseBlockCnt < 1 {
return new(deltaIndexScanner)
}
tableSize := tableSize(worstCaseBlockCnt)
scanner := &deltaIndexScanner{
table: make([]int, tableSize),
mask: tableSize - 1,
entries: make([]int, worstCaseBlockCnt+1),
next: make([]int, worstCaseBlockCnt+1),
}
scanner.scan(buf, size)
return scanner
}
// slightly modified version of JGit's DeltaIndexScanner. We store the offset on the entries
// instead of the entries and the key, so we avoid operations to retrieve the offset later, as
// we don't use the key.
// See: https://github.com/eclipse/jgit/blob/005e5feb4ecd08c4e4d141a38b9e7942accb3212/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/DeltaIndexScanner.java
func (s *deltaIndexScanner) scan(buf []byte, end int) {
lastHash := 0
ptr := end - blksz
for {
key := hashBlock(buf, ptr)
tIdx := key & s.mask
head := s.table[tIdx]
if head != 0 && lastHash == key {
s.entries[head] = ptr
} else {
s.count++
eIdx := s.count
s.entries[eIdx] = ptr
s.next[eIdx] = head
s.table[tIdx] = eIdx
}
lastHash = key
ptr -= blksz
if 0 > ptr {
break
}
}
}
func tableSize(worstCaseBlockCnt int) int {
shift := 32 - leadingZeros(uint32(worstCaseBlockCnt))
sz := 1 << uint(shift-1)
if sz < worstCaseBlockCnt {
sz <<= 1
}
return sz
}
// use https://golang.org/pkg/math/bits/#LeadingZeros32 in the future
func leadingZeros(x uint32) (n int) {
if x >= 1<<16 {
x >>= 16
n = 16
}
if x >= 1<<8 {
x >>= 8
n += 8
}
n += int(len8tab[x])
return 32 - n
}
var len8tab = [256]uint8{
0x00, 0x01, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04,
0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05,
0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,
0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,
0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07,
0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07,
0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07,
0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07,
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
}
func hashBlock(raw []byte, ptr int) int {
// The first 4 steps collapse out into a 4 byte big-endian decode,
// with a larger right shift as we combined shift lefts together.
//
hash := ((uint32(raw[ptr]) & 0xff) << 24) |
((uint32(raw[ptr+1]) & 0xff) << 16) |
((uint32(raw[ptr+2]) & 0xff) << 8) |
(uint32(raw[ptr+3]) & 0xff)
hash ^= T[hash>>31]
hash = ((hash << 8) | (uint32(raw[ptr+4]) & 0xff)) ^ T[hash>>23]
hash = ((hash << 8) | (uint32(raw[ptr+5]) & 0xff)) ^ T[hash>>23]
hash = ((hash << 8) | (uint32(raw[ptr+6]) & 0xff)) ^ T[hash>>23]
hash = ((hash << 8) | (uint32(raw[ptr+7]) & 0xff)) ^ T[hash>>23]
hash = ((hash << 8) | (uint32(raw[ptr+8]) & 0xff)) ^ T[hash>>23]
hash = ((hash << 8) | (uint32(raw[ptr+9]) & 0xff)) ^ T[hash>>23]
hash = ((hash << 8) | (uint32(raw[ptr+10]) & 0xff)) ^ T[hash>>23]
hash = ((hash << 8) | (uint32(raw[ptr+11]) & 0xff)) ^ T[hash>>23]
hash = ((hash << 8) | (uint32(raw[ptr+12]) & 0xff)) ^ T[hash>>23]
hash = ((hash << 8) | (uint32(raw[ptr+13]) & 0xff)) ^ T[hash>>23]
hash = ((hash << 8) | (uint32(raw[ptr+14]) & 0xff)) ^ T[hash>>23]
hash = ((hash << 8) | (uint32(raw[ptr+15]) & 0xff)) ^ T[hash>>23]
return int(hash)
}
// T is the hash lookup table for delta index computation.
var T = []uint32{
0x00000000, 0xd4c6b32d, 0x7d4bd577,
0xa98d665a, 0x2e5119c3, 0xfa97aaee, 0x531accb4, 0x87dc7f99,
0x5ca23386, 0x886480ab, 0x21e9e6f1, 0xf52f55dc, 0x72f32a45,
0xa6359968, 0x0fb8ff32, 0xdb7e4c1f, 0x6d82d421, 0xb944670c,
0x10c90156, 0xc40fb27b, 0x43d3cde2, 0x97157ecf, 0x3e981895,
0xea5eabb8, 0x3120e7a7, 0xe5e6548a, 0x4c6b32d0, 0x98ad81fd,
0x1f71fe64, 0xcbb74d49, 0x623a2b13, 0xb6fc983e, 0x0fc31b6f,
0xdb05a842, 0x7288ce18, 0xa64e7d35, 0x219202ac, 0xf554b181,
0x5cd9d7db, 0x881f64f6, 0x536128e9, 0x87a79bc4, 0x2e2afd9e,
0xfaec4eb3, 0x7d30312a, 0xa9f68207, 0x007be45d, 0xd4bd5770,
0x6241cf4e, 0xb6877c63, 0x1f0a1a39, 0xcbcca914, 0x4c10d68d,
0x98d665a0, 0x315b03fa, 0xe59db0d7, 0x3ee3fcc8, 0xea254fe5,
0x43a829bf, 0x976e9a92, 0x10b2e50b, 0xc4745626, 0x6df9307c,
0xb93f8351, 0x1f8636de, 0xcb4085f3, 0x62cde3a9, 0xb60b5084,
0x31d72f1d, 0xe5119c30, 0x4c9cfa6a, 0x985a4947, 0x43240558,
0x97e2b675, 0x3e6fd02f, 0xeaa96302, 0x6d751c9b, 0xb9b3afb6,
0x103ec9ec, 0xc4f87ac1, 0x7204e2ff, 0xa6c251d2, 0x0f4f3788,
0xdb8984a5, 0x5c55fb3c, 0x88934811, 0x211e2e4b, 0xf5d89d66,
0x2ea6d179, 0xfa606254, 0x53ed040e, 0x872bb723, 0x00f7c8ba,
0xd4317b97, 0x7dbc1dcd, 0xa97aaee0, 0x10452db1, 0xc4839e9c,
0x6d0ef8c6, 0xb9c84beb, 0x3e143472, 0xead2875f, 0x435fe105,
0x97995228, 0x4ce71e37, 0x9821ad1a, 0x31accb40, 0xe56a786d,
0x62b607f4, 0xb670b4d9, 0x1ffdd283, 0xcb3b61ae, 0x7dc7f990,
0xa9014abd, 0x008c2ce7, 0xd44a9fca, 0x5396e053, 0x8750537e,
0x2edd3524, 0xfa1b8609, 0x2165ca16, 0xf5a3793b, 0x5c2e1f61,
0x88e8ac4c, 0x0f34d3d5, 0xdbf260f8, 0x727f06a2, 0xa6b9b58f,
0x3f0c6dbc, 0xebcade91, 0x4247b8cb, 0x96810be6, 0x115d747f,
0xc59bc752, 0x6c16a108, 0xb8d01225, 0x63ae5e3a, 0xb768ed17,
0x1ee58b4d, 0xca233860, 0x4dff47f9, 0x9939f4d4, 0x30b4928e,
0xe47221a3, 0x528eb99d, 0x86480ab0, 0x2fc56cea, 0xfb03dfc7,
0x7cdfa05e, 0xa8191373, 0x01947529, 0xd552c604, 0x0e2c8a1b,
0xdaea3936, 0x73675f6c, 0xa7a1ec41, 0x207d93d8, 0xf4bb20f5,
0x5d3646af, 0x89f0f582, 0x30cf76d3, 0xe409c5fe, 0x4d84a3a4,
0x99421089, 0x1e9e6f10, 0xca58dc3d, 0x63d5ba67, 0xb713094a,
0x6c6d4555, 0xb8abf678, 0x11269022, 0xc5e0230f, 0x423c5c96,
0x96faefbb, 0x3f7789e1, 0xebb13acc, 0x5d4da2f2, 0x898b11df,
0x20067785, 0xf4c0c4a8, 0x731cbb31, 0xa7da081c, 0x0e576e46,
0xda91dd6b, 0x01ef9174, 0xd5292259, 0x7ca44403, 0xa862f72e,
0x2fbe88b7, 0xfb783b9a, 0x52f55dc0, 0x8633eeed, 0x208a5b62,
0xf44ce84f, 0x5dc18e15, 0x89073d38, 0x0edb42a1, 0xda1df18c,
0x739097d6, 0xa75624fb, 0x7c2868e4, 0xa8eedbc9, 0x0163bd93,
0xd5a50ebe, 0x52797127, 0x86bfc20a, 0x2f32a450, 0xfbf4177d,
0x4d088f43, 0x99ce3c6e, 0x30435a34, 0xe485e919, 0x63599680,
0xb79f25ad, 0x1e1243f7, 0xcad4f0da, 0x11aabcc5, 0xc56c0fe8,
0x6ce169b2, 0xb827da9f, 0x3ffba506, 0xeb3d162b, 0x42b07071,
0x9676c35c, 0x2f49400d, 0xfb8ff320, 0x5202957a, 0x86c42657,
0x011859ce, 0xd5deeae3, 0x7c538cb9, 0xa8953f94, 0x73eb738b,
0xa72dc0a6, 0x0ea0a6fc, 0xda6615d1, 0x5dba6a48, 0x897cd965,
0x20f1bf3f, 0xf4370c12, 0x42cb942c, 0x960d2701, 0x3f80415b,
0xeb46f276, 0x6c9a8def, 0xb85c3ec2, 0x11d15898, 0xc517ebb5,
0x1e69a7aa, 0xcaaf1487, 0x632272dd, 0xb7e4c1f0, 0x3038be69,
0xe4fe0d44, 0x4d736b1e, 0x99b5d833,
}
package packfile
import (
"sort"
"sync"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/storer"
)
const (
// deltas based on deltas, how many steps we can do.
// 50 is the default value used in JGit
maxDepth = int64(50)
)
// applyDelta is the set of object types that we should apply deltas
var applyDelta = map[plumbing.ObjectType]bool{
plumbing.BlobObject: true,
plumbing.TreeObject: true,
}
// DeltaSelector decides which objects in a pack will be encoded as
// deltas and against which base, using a sliding window over the
// object set. It is the default object selector used by Encoder.
//
// Callers can also run a DeltaSelector ahead of time and feed the
// result back into an Encoder via WithObjectSelector + a passthrough
// ObjectSelector, so the pack-write phase can stream output without
// an internal delay during selection. This is useful when the
// encoder's writer is something like an HTTP request body where
// mid-stream stalls trip server timeouts.
type DeltaSelector struct {
storer storer.EncodedObjectStorer
}
// NewDeltaSelector returns a DeltaSelector backed by s.
func NewDeltaSelector(s storer.EncodedObjectStorer) *DeltaSelector {
return &DeltaSelector{s}
}
// ObjectsToPack creates a list of ObjectToPack from the hashes
// provided, creating deltas if it's suitable, using an specific
// internal logic. `packWindow` specifies the size of the sliding
// window used to compare objects for delta compression; 0 turns off
// delta compression entirely.
func (dw *DeltaSelector) ObjectsToPack(
hashes []plumbing.Hash,
packWindow uint,
) ([]*ObjectToPack, error) {
otp, err := dw.objectsToPack(hashes, packWindow)
if err != nil {
return nil, err
}
if packWindow == 0 {
return otp, nil
}
dw.sort(otp)
var objectGroups [][]*ObjectToPack
var prev *ObjectToPack
i := -1
for _, obj := range otp {
if prev == nil || prev.Type() != obj.Type() {
objectGroups = append(objectGroups, []*ObjectToPack{obj})
i++
prev = obj
} else {
objectGroups[i] = append(objectGroups[i], obj)
}
}
var wg sync.WaitGroup
var once sync.Once
for _, objs := range objectGroups {
wg.Go(func() {
if walkErr := dw.walk(objs, packWindow); walkErr != nil {
once.Do(func() {
err = walkErr
})
}
})
}
wg.Wait()
if err != nil {
return nil, err
}
return otp, nil
}
func (dw *DeltaSelector) objectsToPack(
hashes []plumbing.Hash,
packWindow uint,
) ([]*ObjectToPack, error) {
objectsToPack := make([]*ObjectToPack, 0, len(hashes))
for _, h := range hashes {
var o plumbing.EncodedObject
var err error
if packWindow == 0 {
o, err = dw.encodedObject(h)
} else {
o, err = dw.encodedDeltaObject(h)
}
if err != nil {
return nil, err
}
otp := newObjectToPack(o)
if _, ok := o.(plumbing.DeltaObject); ok {
otp.CleanOriginal()
}
objectsToPack = append(objectsToPack, otp)
}
if packWindow == 0 {
return objectsToPack, nil
}
if err := dw.fixAndBreakChains(objectsToPack); err != nil {
return nil, err
}
return objectsToPack, nil
}
func (dw *DeltaSelector) encodedDeltaObject(h plumbing.Hash) (plumbing.EncodedObject, error) {
edos, ok := dw.storer.(storer.DeltaObjectStorer)
if !ok {
return dw.encodedObject(h)
}
return edos.DeltaObject(plumbing.AnyObject, h)
}
func (dw *DeltaSelector) encodedObject(h plumbing.Hash) (plumbing.EncodedObject, error) {
return dw.storer.EncodedObject(plumbing.AnyObject, h)
}
func (dw *DeltaSelector) fixAndBreakChains(objectsToPack []*ObjectToPack) error {
m := make(map[plumbing.Hash]*ObjectToPack, len(objectsToPack))
for _, otp := range objectsToPack {
m[otp.Hash()] = otp
}
// visiting holds the objects on the current resolution path, so that a
// delta chain looping back on itself can be detected and broken.
visiting := make(map[plumbing.Hash]bool)
for _, otp := range objectsToPack {
if err := dw.fixAndBreakChainsOne(m, otp, visiting); err != nil {
return err
}
}
return nil
}
func (dw *DeltaSelector) fixAndBreakChainsOne(
objectsToPack map[plumbing.Hash]*ObjectToPack,
otp *ObjectToPack,
visiting map[plumbing.Hash]bool,
) error {
if !otp.Object.Type().IsDelta() {
return nil
}
// Initial ObjectToPack instances might have a delta assigned to Object
// but no actual base initially. Once Base is assigned to a delta, it means
// we already fixed it.
if otp.Base != nil {
return nil
}
do, ok := otp.Object.(plumbing.DeltaObject)
if !ok {
// if this is not a DeltaObject, then we cannot retrieve its base,
// so we have to break the delta chain here.
return dw.undeltify(otp)
}
base, ok := objectsToPack[do.BaseHash()]
if !ok {
// The base of the delta is not in our list of objects to pack, so
// we break the chain.
return dw.undeltify(otp)
}
// Mark this object as being resolved before looking at its base, so that
// a delta based on itself is caught by the check below.
h := otp.Hash()
visiting[h] = true
defer delete(visiting, h)
// A delta chain that loops back onto an object we are already resolving
// cannot be written: every delta needs its base written first. Break the
// chain here instead of following the cycle, which would recurse until
// the goroutine stack is exhausted.
if visiting[do.BaseHash()] {
return dw.undeltify(otp)
}
if err := dw.fixAndBreakChainsOne(objectsToPack, base, visiting); err != nil {
return err
}
otp.SetDelta(base, otp.Object)
return nil
}
func (dw *DeltaSelector) restoreOriginal(otp *ObjectToPack) error {
if otp.Original != nil {
return nil
}
if !otp.Object.Type().IsDelta() {
return nil
}
obj, err := dw.encodedObject(otp.Hash())
if err != nil {
return err
}
otp.SetOriginal(obj)
return nil
}
// undeltify undeltifies an *ObjectToPack by retrieving the original object from
// the storer and resetting it.
func (dw *DeltaSelector) undeltify(otp *ObjectToPack) error {
if err := dw.restoreOriginal(otp); err != nil {
return err
}
otp.Object = otp.Original
otp.Depth = 0
return nil
}
func (dw *DeltaSelector) sort(objectsToPack []*ObjectToPack) {
sort.Sort(byTypeAndSize(objectsToPack))
}
func (dw *DeltaSelector) walk(
objectsToPack []*ObjectToPack,
packWindow uint,
) error {
indexMap := make(map[plumbing.Hash]*deltaIndex)
for i := range len(objectsToPack) {
// Clean up the index map and reconstructed delta objects for anything
// outside our pack window, to save memory.
if i > int(packWindow) {
obj := objectsToPack[i-int(packWindow)]
delete(indexMap, obj.Hash())
if obj.IsDelta() {
obj.SaveOriginalMetadata()
obj.CleanOriginal()
}
}
target := objectsToPack[i]
// If we already have a delta, we don't try to find a new one for this
// object. This happens when a delta is set to be reused from an existing
// packfile.
if target.IsDelta() {
continue
}
// We only want to create deltas from specific types.
if !applyDelta[target.Type()] {
continue
}
for j := i - 1; j >= 0 && i-j < int(packWindow); j-- {
base := objectsToPack[j]
// Objects must use only the same type as their delta base.
// Since objectsToPack is sorted by type and size, once we find
// a different type, we know we won't find more of them.
if base.Type() != target.Type() {
break
}
if err := dw.tryToDeltify(indexMap, base, target); err != nil {
return err
}
}
}
return nil
}
func (dw *DeltaSelector) tryToDeltify(indexMap map[plumbing.Hash]*deltaIndex, base, target *ObjectToPack) error {
// Original object might not be present if we're reusing a delta, so we
// ensure it is restored.
if err := dw.restoreOriginal(target); err != nil {
return err
}
if err := dw.restoreOriginal(base); err != nil {
return err
}
// If the sizes are radically different, this is a bad pairing.
if target.Size() < base.Size()>>4 {
return nil
}
msz := dw.deltaSizeLimit(
target.Object.Size(),
base.Depth,
target.Depth,
target.IsDelta(),
)
// Nearly impossible to fit useful delta.
if msz <= 8 {
return nil
}
// If we have to insert a lot to make this work, find another.
if base.Size()-target.Size() > msz {
return nil
}
if _, ok := indexMap[base.Hash()]; !ok {
indexMap[base.Hash()] = new(deltaIndex)
}
// Now we can generate the delta using originals
delta, err := getDelta(indexMap[base.Hash()], base.Original, target.Original)
if err != nil {
return err
}
// if delta better than target
if delta.Size() < msz {
target.SetDelta(base, delta)
}
return nil
}
func (dw *DeltaSelector) deltaSizeLimit(targetSize int64, baseDepth int,
targetDepth int, targetDelta bool,
) int64 {
if !targetDelta {
// Any delta should be no more than 50% of the original size
// (for text files deflate of whole form should shrink 50%).
n := targetSize >> 1
// Evenly distribute delta size limits over allowed depth.
// If src is non-delta (depth = 0), delta <= 50% of original.
// If src is almost at limit (9/10), delta <= 10% of original.
return n * (maxDepth - int64(baseDepth)) / maxDepth
}
// With a delta base chosen any new delta must be "better".
// Retain the distribution described above.
d := int64(targetDepth)
n := targetSize
// If target depth is bigger than maxDepth, this delta is not suitable to be used.
if d >= maxDepth {
return 0
}
// If src is whole (depth=0) and base is near limit (depth=9/10)
// any delta using src can be 10x larger and still be better.
//
// If src is near limit (depth=9/10) and base is whole (depth=0)
// a new delta dependent on src must be 1/10th the size.
return n * (maxDepth - int64(baseDepth)) / (maxDepth - d)
}
type byTypeAndSize []*ObjectToPack
func (a byTypeAndSize) Len() int { return len(a) }
func (a byTypeAndSize) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a byTypeAndSize) Less(i, j int) bool {
if a[i].Type() < a[j].Type() {
return false
}
if a[i].Type() > a[j].Type() {
return true
}
return a[i].Size() > a[j].Size()
}
package packfile
import (
"bytes"
"github.com/go-git/go-git/v6/plumbing"
packutil "github.com/go-git/go-git/v6/plumbing/format/packfile/util"
"github.com/go-git/go-git/v6/utils/ioutil"
"github.com/go-git/go-git/v6/utils/sync"
)
// See https://github.com/jelmer/dulwich/blob/master/dulwich/pack.py and
// https://github.com/tarruda/node-git-core/blob/master/src/js/delta.js
// for more info
const (
// Standard chunk size used to generate fingerprints
s = 16
// https://github.com/git/git/blob/f7466e94375b3be27f229c78873f0acf8301c0a5/diff-delta.c#L428
// Max size of a copy operation (64KB).
maxCopySize = 64 * 1024
)
// GetDelta returns an EncodedObject of type OFSDeltaObject. Base and Target object,
// will be loaded into memory to be able to create the delta object.
// To generate target again, you will need the obtained object and "base" one.
// Error will be returned if base or target object cannot be read.
func GetDelta(base, target plumbing.EncodedObject) (plumbing.EncodedObject, error) {
return getDelta(new(deltaIndex), base, target)
}
func getDelta(index *deltaIndex, base, target plumbing.EncodedObject) (o plumbing.EncodedObject, err error) {
br, err := base.Reader()
if err != nil {
return nil, err
}
defer ioutil.CheckClose(br, &err)
tr, err := target.Reader()
if err != nil {
return nil, err
}
defer ioutil.CheckClose(tr, &err)
bb := sync.GetBytesBuffer()
defer sync.PutBytesBuffer(bb)
_, err = bb.ReadFrom(br)
if err != nil {
return nil, err
}
tb := sync.GetBytesBuffer()
defer sync.PutBytesBuffer(tb)
_, err = tb.ReadFrom(tr)
if err != nil {
return nil, err
}
db := diffDelta(index, bb.Bytes(), tb.Bytes())
delta := &plumbing.MemoryObject{}
_, err = delta.Write(db)
if err != nil {
return nil, err
}
delta.SetSize(int64(len(db)))
delta.SetType(plumbing.OFSDeltaObject)
return delta, nil
}
// DiffDelta returns the delta that transforms src into tgt.
func DiffDelta(src, tgt []byte) []byte {
return diffDelta(new(deltaIndex), src, tgt)
}
func diffDelta(index *deltaIndex, src, tgt []byte) []byte {
buf := sync.GetBytesBuffer()
defer sync.PutBytesBuffer(buf)
buf.Write(packutil.EncodeLEB128(uint(len(src))))
buf.Write(packutil.EncodeLEB128(uint(len(tgt))))
if len(index.entries) == 0 {
index.init(src)
}
ibuf := sync.GetBytesBuffer()
defer sync.PutBytesBuffer(ibuf)
for i := 0; i < len(tgt); i++ {
offset, l := index.findMatch(src, tgt, i)
switch {
case l == 0:
// couldn't find a match, just write the current byte and continue
ibuf.WriteByte(tgt[i])
case l < 0:
// src is less than blksz, copy the rest of the target to avoid
// calls to findMatch
for ; i < len(tgt); i++ {
ibuf.WriteByte(tgt[i])
}
case l < s:
// remaining target is less than blksz, copy what's left of it
// and avoid calls to findMatch
for j := i; j < i+l; j++ {
ibuf.WriteByte(tgt[j])
}
i += l - 1
default:
encodeInsertOperation(ibuf, buf)
rl := l
aOffset := offset
for rl > 0 {
if rl < maxCopySize {
buf.Write(encodeCopyOperation(aOffset, rl))
break
}
buf.Write(encodeCopyOperation(aOffset, maxCopySize))
rl -= maxCopySize
aOffset += maxCopySize
}
i += l - 1
}
}
encodeInsertOperation(ibuf, buf)
// buf.Bytes() is only valid until the next modifying operation on the buffer. Copy it.
return append([]byte{}, buf.Bytes()...)
}
func encodeInsertOperation(ibuf, buf *bytes.Buffer) {
if ibuf.Len() == 0 {
return
}
b := ibuf.Bytes()
s := ibuf.Len()
o := 0
for s > 127 {
buf.WriteByte(byte(127))
buf.Write(b[o : o+127])
s -= 127
o += 127
}
buf.WriteByte(byte(s))
buf.Write(b[o : o+s])
ibuf.Reset()
}
func encodeCopyOperation(offset, length int) []byte {
code := 0x80
var opcodes []byte
var i uint
for i = 0; i < 4; i++ {
f := 0xff << (i * 8)
if offset&f != 0 {
opcodes = append(opcodes, byte(offset&f>>(i*8)))
code |= 0x01 << i
}
}
for i = range 3 {
f := 0xff << (i * 8)
if length&f != 0 {
opcodes = append(opcodes, byte(length&f>>(i*8)))
code |= 0x10 << i
}
}
return append([]byte{byte(code)}, opcodes...)
}
package packfile
import (
"crypto"
"errors"
"fmt"
"io"
"github.com/go-git/go-git/v6/config"
"github.com/go-git/go-git/v6/plumbing"
cfgformat "github.com/go-git/go-git/v6/plumbing/format/config"
"github.com/go-git/go-git/v6/plumbing/hash"
"github.com/go-git/go-git/v6/plumbing/storer"
"github.com/go-git/go-git/v6/utils/binary"
"github.com/go-git/go-git/v6/utils/ioutil"
"github.com/go-git/go-git/v6/utils/sync"
)
// ObjectSelector decides which objects go into a pack and in what
// order, including any delta relationships. The default selector is
// *DeltaSelector.
type ObjectSelector interface {
ObjectsToPack(hashes []plumbing.Hash, packWindow uint) ([]*ObjectToPack, error)
}
// Encoder gets the data from the storage and write it into the writer in PACK
// format.
//
// The encoder has two selector fields: deltaSelector is the
// encoder's own *DeltaSelector, used internally for write-phase
// recovery (e.g. restoreOriginal on cyclic chains). objectSelector is
// what Encode calls to obtain the object list — by default the same
// *DeltaSelector, but a caller can override it via WithObjectSelector.
type Encoder struct {
deltaSelector *DeltaSelector
objectSelector ObjectSelector
w *offsetWriter
zw sync.ZlibWriter
hasher hash.Hash
useRefDeltas bool
}
// EncoderOption configures an Encoder at construction time.
type EncoderOption func(*Encoder)
// WithObjectSelector overrides the ObjectSelector used by Encode to
// produce the object list. The default is the encoder's own
// *DeltaSelector, which runs delta selection synchronously when
// Encode is called.
//
// Supplying a selector that returns a precomputed []*ObjectToPack
// (typically the result of a prior DeltaSelector.ObjectsToPack call)
// lets Encode skip the selection step and start writing pack bytes
// immediately. This is useful when the encoder's writer is something
// like an HTTP request body where a multi-second mid-stream stall
// trips server timeouts. The encoder still uses its own internal
// *DeltaSelector for recovery operations during the write phase
// (e.g. when a concurrent repack invalidates a chosen delta base),
// so the storer passed to NewEncoder must remain valid.
func WithObjectSelector(s ObjectSelector) EncoderOption {
return func(e *Encoder) {
if s != nil {
e.objectSelector = s
}
}
}
// NewEncoder creates a new packfile encoder using a specific Writer and
// EncodedObjectStorer. By default deltas used to generate the packfile will be
// OFSDeltaObject. To use Reference deltas, set useRefDeltas to true.
//
// Optional EncoderOptions configure encoder behavior; see
// WithObjectSelector for the main use case (precomputed selection for
// streaming output).
func NewEncoder(w io.Writer, s storer.EncodedObjectStorer, useRefDeltas bool, opts ...EncoderOption) *Encoder {
var of cfgformat.ObjectFormat
if c, ok := s.(config.ConfigStorer); ok {
cfg, err := c.Config()
if err == nil {
of = cfg.Extensions.ObjectFormat
}
}
var h hash.Hash
if of == cfgformat.SHA256 {
h = hash.New(crypto.SHA256)
} else {
h = hash.New(crypto.SHA1)
}
mw := io.MultiWriter(w, h)
ow := newOffsetWriter(mw)
zw := sync.GetZlibWriter(mw)
sel := NewDeltaSelector(s)
e := &Encoder{
deltaSelector: sel,
objectSelector: sel,
w: ow,
zw: zw,
hasher: h,
useRefDeltas: useRefDeltas,
}
for _, opt := range opts {
opt(e)
}
return e
}
// Encode creates a packfile containing all the objects referenced in
// hashes and writes it to the writer in the Encoder. `packWindow`
// specifies the size of the sliding window used to compare objects
// for delta compression; 0 turns off delta compression entirely.
//
// The object set is produced by the configured ObjectSelector (see
// WithObjectSelector). The encoder's internal *DeltaSelector is still
// used for recovery operations during the write phase regardless of
// the configured selector.
func (e *Encoder) Encode(
hashes []plumbing.Hash,
packWindow uint,
) (plumbing.Hash, error) {
objects, err := e.objectSelector.ObjectsToPack(hashes, packWindow)
if err != nil {
return plumbing.ZeroHash, err
}
return e.encode(objects)
}
func (e *Encoder) encode(objects []*ObjectToPack) (plumbing.Hash, error) {
if err := e.head(len(objects)); err != nil {
return plumbing.ZeroHash, err
}
for _, o := range objects {
if err := e.entry(o); err != nil {
return plumbing.ZeroHash, err
}
}
return e.footer()
}
func (e *Encoder) head(numEntries int) error {
return binary.Write(
e.w,
signature,
int32(VersionSupported),
int32(numEntries),
)
}
func (e *Encoder) entry(o *ObjectToPack) (err error) {
if o.WantWrite() {
// A cycle exists in this delta chain. This should only occur if a
// selected object representation disappeared during writing
// (for example due to a concurrent repack) and a different base
// was chosen, forcing a cycle. Select something other than a
// delta, and write this object.
if err := e.deltaSelector.restoreOriginal(o); err != nil {
return err
}
o.BackToOriginal()
}
if o.IsWritten() {
return nil
}
o.MarkWantWrite()
if err := e.writeBaseIfDelta(o); err != nil {
return err
}
// We need to check if we already write that object due a cyclic delta chain
if o.IsWritten() {
return nil
}
o.Offset = e.w.Offset()
if o.IsDelta() {
if err := e.writeDeltaHeader(o); err != nil {
return err
}
} else {
if err := e.entryHead(o.Type(), o.Size()); err != nil {
return err
}
}
e.zw.Reset(e.w)
defer ioutil.CheckClose(e.zw, &err)
or, err := o.Object.Reader()
if err != nil {
return err
}
defer ioutil.CheckClose(or, &err)
_, err = ioutil.CopyBufferPool(e.zw, or)
return err
}
func (e *Encoder) writeBaseIfDelta(o *ObjectToPack) error {
if o.IsDelta() && !o.Base.IsWritten() {
// We must write base first
return e.entry(o.Base)
}
return nil
}
func (e *Encoder) writeDeltaHeader(o *ObjectToPack) error {
// Every delta in an encoded pack uses the same kind — all OFS_DELTA
// by default, or all REF_DELTA when useRefDeltas is set. The parser
// (see Parser.resolveDeltas) accepts packs that mix OFS_DELTA and
// REF_DELTA in a single chain, because mixed-kind packs occur in
// the wild (repacks across servers with differing
// --delta-base-offset settings, thin-pack splices, third-party
// tooling); the encoder deliberately doesn't introduce that
// complexity on the write side.
t := plumbing.OFSDeltaObject
if e.useRefDeltas {
t = plumbing.REFDeltaObject
}
if err := e.entryHead(t, o.Object.Size()); err != nil {
return err
}
if e.useRefDeltas {
return e.writeRefDeltaHeader(o.Base.Hash())
}
return e.writeOfsDeltaHeader(o)
}
func (e *Encoder) writeRefDeltaHeader(base plumbing.Hash) error {
_, err := base.WriteTo(e.w)
return err
}
func (e *Encoder) writeOfsDeltaHeader(o *ObjectToPack) error {
// for OFS_DELTA, offset of the base is interpreted as negative offset
// relative to the type-byte of the header of the ofs-delta entry.
relativeOffset := o.Offset - o.Base.Offset
if relativeOffset <= 0 {
return fmt.Errorf("bad offset for OFS_DELTA entry: %d", relativeOffset)
}
return binary.WriteVariableWidthInt(e.w, relativeOffset)
}
func (e *Encoder) entryHead(typeNum plumbing.ObjectType, size int64) error {
t := int64(typeNum)
header := []byte{}
c := (t << firstLengthBits) | (size & maskFirstLength)
size >>= firstLengthBits
for size != 0 {
header = append(header, byte(c|maskContinue))
c = size & int64(maskLength)
size >>= lengthBits
}
header = append(header, byte(c))
_, err := e.w.Write(header)
return err
}
func (e *Encoder) footer() (plumbing.Hash, error) {
h, ok := plumbing.FromBytes(e.hasher.Sum(nil))
if !ok {
return plumbing.ZeroHash, errors.New("packfile encoder yielded invalid hash")
}
_, err := h.WriteTo(e.w)
return h, err
}
type offsetWriter struct {
w io.Writer
offset int64
}
func newOffsetWriter(w io.Writer) *offsetWriter {
return &offsetWriter{w: w}
}
func (ow *offsetWriter) Write(p []byte) (n int, err error) {
n, err = ow.w.Write(p)
ow.offset += int64(n)
return n, err
}
func (ow *offsetWriter) Offset() int64 {
return ow.offset
}
package packfile
import "fmt"
// Error specifies errors returned during packfile parsing.
type Error struct {
reason, details string
}
// NewError returns a new error.
func NewError(reason string) *Error {
return &Error{reason: reason}
}
// Error returns a text representation of the error.
func (e *Error) Error() string {
if e.details == "" {
return e.reason
}
return fmt.Sprintf("%s: %s", e.reason, e.details)
}
// AddDetails adds details to an error, with additional text.
func (e *Error) AddDetails(format string, args ...any) *Error {
return &Error{
reason: e.reason,
details: fmt.Sprintf(format, args...),
}
}
package packfile
import (
"bufio"
"errors"
"io"
"math"
"os"
stdsync "sync"
billy "github.com/go-git/go-billy/v6"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/cache"
"github.com/go-git/go-git/v6/plumbing/format/idxfile"
"github.com/go-git/go-git/v6/utils/ioutil"
"github.com/go-git/go-git/v6/utils/sync"
)
// probeSize is the byte count for the closed-FD probe. A one-byte
// ReadAt distinguishes a live descriptor from a closed one without
// mutating the file's seek cursor; a zero-length read is unusable
// because some implementations (e.g. [os.File]) return (0, nil) on
// a closed file.
const probeSize = 1
// probeBufPool returns the per-call backing array for [probePack].
// Pooling keeps the read path allocation-free on what is a very hot
// code path.
var probeBufPool = stdsync.Pool{
New: func() any {
var buf [probeSize]byte
return &buf
},
}
// probePack tests whether pack is still readable at offset by
// issuing a [probeSize]-byte [io.ReaderAt.ReadAt]. The error is
// returned verbatim so any wrapping context (path, syscall) is
// preserved for the caller; classification is left to
// [errors.Is]:
//
// - nil means the descriptor is live; the caller may keep using
// pack.
// - an error matching [os.ErrClosed] means the descriptor has
// been closed and the caller should reopen the file.
// - any other error is propagated, matching the canonical Git
// behaviour in `packfile.c:use_pack`, which does not retry on
// transient I/O errors.
//
// [io.EOF] indicates the offset is at or past end-of-file, which
// implies a truncated pack — propagate rather than masking.
func probePack(pack io.ReaderAt, offset int64) error {
buf := probeBufPool.Get().(*[probeSize]byte)
defer probeBufPool.Put(buf)
_, err := pack.ReadAt(buf[:], offset)
return err
}
// FSObject is an object from the packfile on the filesystem.
type FSObject struct {
hash plumbing.Hash
offset int64
size int64
typ plumbing.ObjectType
index idxfile.Index
fs billy.Filesystem
pack billy.File
packPath string
cache cache.Object
// acquireRandom, when set, supersedes pack/packPath/fs in
// [FSObject.Reader]: each call yields a fresh cursor that
// Close releases.
acquireRandom func() (RandomReader, error)
}
// NewFSObject creates a new filesystem object.
func NewFSObject(
hash plumbing.Hash,
finalType plumbing.ObjectType,
offset int64,
contentSize int64,
index idxfile.Index,
fs billy.Filesystem,
pack billy.File,
packPath string,
cache cache.Object,
) *FSObject {
return &FSObject{
hash: hash,
offset: offset,
size: contentSize,
typ: finalType,
index: index,
fs: fs,
pack: pack,
packPath: packPath,
cache: cache,
}
}
// Reader implements the plumbing.EncodedObject interface.
//
// Reader is safe for concurrent use: it uses ReadAt (which does
// not modify the file's seek cursor) instead of Seek+Read, so
// multiple goroutines can call Reader on FSObjects that share the
// same underlying packfile handle.
func (o *FSObject) Reader() (io.ReadCloser, error) {
obj, ok := o.cache.Get(o.hash)
if ok && obj != o {
reader, err := obj.Reader()
if err != nil {
return nil, err
}
return reader, nil
}
var (
pack io.ReaderAt
file io.Closer
)
if o.acquireRandom != nil {
cur, err := o.acquireRandom()
if err != nil {
return nil, err
}
pack = cur
file = cur
} else {
pack = o.pack
switch err := probePack(pack, o.offset); {
case err == nil:
// FD is live; keep using pack.
case errors.Is(err, os.ErrClosed):
reopened, oerr := o.fs.Open(o.packPath)
if oerr != nil {
return nil, oerr
}
pack = reopened
file = reopened
default:
return nil, err
}
}
// SectionReader provides a standalone io.Reader backed by ReadAt. Each
// SectionReader maintains its own read position, so concurrent calls
// to Reader do not interfere with each other or with the packfile's
// Scanner. The upper bound is set to math.MaxInt64 because zlib
// streams are self-terminating — the decompressor stops at the DEFLATE
// end marker regardless of how many bytes remain available.
sr := io.NewSectionReader(pack, o.offset, math.MaxInt64-o.offset)
br := sync.GetBufioReader(sr)
zr, err := sync.GetZlibReader(br)
if err != nil {
sync.PutBufioReader(br)
if file != nil {
_ = file.Close()
}
return nil, err
}
return NewBoundedReadCloser(&zlibReadCloser{r: zr, f: file, rbuf: br}, o.size), nil
}
type zlibReadCloser struct {
r *sync.ZLibReader
f io.Closer
rbuf *bufio.Reader
closed bool
}
// Read reads up to len(p) bytes into p from the data.
func (r *zlibReadCloser) Read(p []byte) (int, error) {
return r.r.Read(p)
}
func (r *zlibReadCloser) Close() (err error) {
if r.closed {
return nil
}
r.closed = true
if r.f != nil {
defer ioutil.CheckClose(r.f, &err)
}
defer sync.PutBufioReader(r.rbuf)
defer sync.PutZlibReader(r.r)
return r.r.Close()
}
// SetSize implements the plumbing.EncodedObject interface. This method
// is a noop.
func (o *FSObject) SetSize(int64) {}
// SetType implements the plumbing.EncodedObject interface. This method is
// a noop.
func (o *FSObject) SetType(plumbing.ObjectType) {}
// Hash implements the plumbing.EncodedObject interface.
func (o *FSObject) Hash() plumbing.Hash { return o.hash }
// Size implements the plumbing.EncodedObject interface.
func (o *FSObject) Size() int64 { return o.size }
// Type implements the plumbing.EncodedObject interface.
func (o *FSObject) Type() plumbing.ObjectType {
return o.typ
}
// Writer implements the plumbing.EncodedObject interface. This method always
// returns a nil writer.
func (o *FSObject) Writer() (io.WriteCloser, error) {
return nil, nil
}
package packfile
import (
"github.com/go-git/go-git/v6/plumbing"
)
// ObjectToPack is a representation of an object that is going to be into a
// pack file.
type ObjectToPack struct {
// The main object to pack, it could be any object, including deltas.
Object plumbing.EncodedObject
// Base is the object that a delta is based on, which could also be another delta.
// Nil when the main object is not a delta.
Base *ObjectToPack
// Original is the object that we can generate applying the delta to
// Base, or the same object as Object in the case of a non-delta
// object.
Original plumbing.EncodedObject
// Depth is the amount of deltas needed to resolve to obtain Original
// (delta based on delta based on ...)
Depth int
// offset in pack when object has been already written, or 0 if it
// has not been written yet
Offset int64
// Information from the original object
resolvedOriginal bool
originalType plumbing.ObjectType
originalSize int64
originalHash plumbing.Hash
}
// newObjectToPack creates a correct ObjectToPack based on a non-delta object
func newObjectToPack(o plumbing.EncodedObject) *ObjectToPack {
return &ObjectToPack{
Object: o,
Original: o,
}
}
// newDeltaObjectToPack creates a correct ObjectToPack for a delta object, based on
// his base (could be another delta), the delta target (in this case called original),
// and the delta Object itself
func newDeltaObjectToPack(base *ObjectToPack, original, delta plumbing.EncodedObject) *ObjectToPack {
return &ObjectToPack{
Object: delta,
Base: base,
Original: original,
Depth: base.Depth + 1,
}
}
// BackToOriginal converts that ObjectToPack to a non-deltified object if it was one
func (o *ObjectToPack) BackToOriginal() {
if o.IsDelta() && o.Original != nil {
o.Object = o.Original
o.Base = nil
o.Depth = 0
}
}
// IsWritten returns if that ObjectToPack was
// already written into the packfile or not
func (o *ObjectToPack) IsWritten() bool {
return o.Offset > 1
}
// MarkWantWrite marks this ObjectToPack as WantWrite
// to avoid delta chain loops
func (o *ObjectToPack) MarkWantWrite() {
o.Offset = 1
}
// WantWrite checks if this ObjectToPack was marked as WantWrite before
func (o *ObjectToPack) WantWrite() bool {
return o.Offset == 1
}
// SetOriginal sets both Original and saves size, type and hash. If object
// is nil Original is set but previous resolved values are kept
func (o *ObjectToPack) SetOriginal(obj plumbing.EncodedObject) {
o.Original = obj
o.SaveOriginalMetadata()
}
// SaveOriginalMetadata saves size, type and hash of Original object
func (o *ObjectToPack) SaveOriginalMetadata() {
if o.Original != nil {
o.originalSize = o.Original.Size()
o.originalType = o.Original.Type()
o.originalHash = o.Original.Hash()
o.resolvedOriginal = true
}
}
// CleanOriginal sets Original to nil
func (o *ObjectToPack) CleanOriginal() {
o.Original = nil
}
// Type returns the object type.
func (o *ObjectToPack) Type() plumbing.ObjectType {
if o.Original != nil {
return o.Original.Type()
}
if o.resolvedOriginal {
return o.originalType
}
if o.Base != nil {
return o.Base.Type()
}
if o.Object != nil {
return o.Object.Type()
}
panic("cannot get type")
}
// Hash returns the object hash.
func (o *ObjectToPack) Hash() plumbing.Hash {
if o.Original != nil {
return o.Original.Hash()
}
if o.resolvedOriginal {
return o.originalHash
}
do, ok := o.Object.(plumbing.DeltaObject)
if ok {
return do.ActualHash()
}
panic("cannot get hash")
}
// Size returns the object size.
func (o *ObjectToPack) Size() int64 {
if o.Original != nil {
return o.Original.Size()
}
if o.resolvedOriginal {
return o.originalSize
}
do, ok := o.Object.(plumbing.DeltaObject)
if ok {
return do.ActualSize()
}
panic("cannot get ObjectToPack size")
}
// IsDelta returns true if the object is a delta.
func (o *ObjectToPack) IsDelta() bool {
return o.Base != nil
}
// SetDelta sets the object's base and delta.
func (o *ObjectToPack) SetDelta(base *ObjectToPack, delta plumbing.EncodedObject) {
o.Object = delta
o.Base = base
o.Depth = base.Depth + 1
}
package packfile
import (
"io"
"github.com/go-git/go-git/v6/plumbing"
)
// PackHandle is the handle [NewPackfile] consumes when
// [WithPackHandle] is supplied.
type PackHandle interface {
// OpenPackReader returns a fresh sequential cursor over the
// .pack file. The cursor is closed by the caller.
OpenPackReader() (io.ReadSeekCloser, error)
// OpenRandomReader returns a fresh random-access cursor over
// the .pack file. The cursor is closed by the caller.
OpenRandomReader() (RandomReader, error)
// PackHash returns the .pack file's trailing checksum, which
// by canonical-Git construction equals the pack's identity
// hash (the hex in pack-<hash>.pack).
PackHash() (plumbing.Hash, error)
}
// RandomReader is the per-read random-access cursor returned by
// [PackHandle.OpenRandomReader]. ReadAt is safe to call
// concurrently with itself; Close releases the cursor's hold on
// the underlying pack file descriptor.
type RandomReader interface {
io.ReaderAt
io.Closer
}
// PackHandleResolver returns the current [PackHandle] for one
// .pack file. It is invoked on scanner init (once per [Packfile])
// and on every [FSObject.Reader] call. See [DotGit.PackHandle]
// for the reference implementation.
//
// Contract:
//
// - Every handle returned for the lifetime of a given [Packfile]
// MUST address the same .pack file on disk (same PackHash).
// The handle value MAY change across calls. [Packfile] does
// NOT re-validate identity on re-resolution.
// - Errors propagate to the caller as object-read errors. The
// resolver SHOULD NOT retry internally.
// - The handle returned MUST remain valid until at least one
// cursor obtained from it has been closed by the caller.
type PackHandleResolver func() (PackHandle, error)
// WithPackHandle injects an externally-owned [PackHandle] resolver.
// The resolved handle is not closed by [Packfile.Close]; its
// lifetime is owned by the resolver. See [PackHandleResolver] for
// the resolver contract.
func WithPackHandle(get PackHandleResolver) PackfileOption {
return func(p *Packfile) {
p.resolveHandle = get
}
}
// openRandomReader re-resolves the pack handle via resolveHandle
// and returns a fresh random-access cursor.
func (p *Packfile) openRandomReader() (RandomReader, error) {
h, err := p.resolveHandle()
if err != nil {
return nil, err
}
return h.OpenRandomReader()
}
package packfile
import (
"bufio"
"crypto"
"fmt"
"io"
"io/fs"
"sync"
"sync/atomic"
billy "github.com/go-git/go-billy/v6"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/cache"
format "github.com/go-git/go-git/v6/plumbing/format/config"
"github.com/go-git/go-git/v6/plumbing/format/idxfile"
"github.com/go-git/go-git/v6/plumbing/storer"
"github.com/go-git/go-git/v6/utils/ioutil"
gogitsync "github.com/go-git/go-git/v6/utils/sync"
)
var (
// ErrInvalidObject is returned by Decode when an invalid object is
// found in the packfile.
ErrInvalidObject = NewError("invalid git object")
// ErrZLib is returned by Decode when there was an error unzipping
// the packfile contents.
ErrZLib = NewError("zlib reading error")
)
// Packfile allows retrieving information from inside a packfile.
type Packfile struct {
idxfile.Index
fs billy.Filesystem
file billy.File
// handle is the resolved PackHandle once init has run; nil
// in legacy mode. See NewPackfile for the modes.
handle PackHandle
resolveHandle PackHandleResolver
scanReader io.ReadSeekCloser
scanner *Scanner
cache cache.Object
rbuf *bufio.Reader
id plumbing.Hash
m sync.Mutex
objectIDSize int
once sync.Once
onceErr error
closed atomic.Bool
}
// NewPackfile returns a packfile representation for the given .pack
// file and idx. If [WithFs] is set the packfile returns [FSObject]s;
// otherwise it returns [plumbing.MemoryObject]s.
//
// When [WithPackHandle] is supplied, the resolver owns the pack
// file descriptor and the file argument is redundant; the
// constructor closes it and [Packfile.Close] does not close the
// resolver-owned handle. Otherwise the file argument is used as-is
// and is closed by [Packfile.Close].
func NewPackfile(
file billy.File,
opts ...PackfileOption,
) *Packfile {
p := &Packfile{
file: file,
objectIDSize: crypto.SHA1.Size(),
}
for _, opt := range opts {
opt(p)
}
if p.resolveHandle != nil && file != nil {
_ = file.Close()
p.file = nil
}
return p
}
// Get retrieves the encoded object in the packfile with the given hash.
func (p *Packfile) Get(h plumbing.Hash) (plumbing.EncodedObject, error) {
if p.closed.Load() {
return nil, fs.ErrClosed
}
if err := p.init(); err != nil {
return nil, err
}
p.m.Lock()
defer p.m.Unlock()
// Re-check after Lock: Close may have flipped closed and torn
// down the scanner between the early Load and the Lock.
if p.closed.Load() {
return nil, fs.ErrClosed
}
return p.get(h)
}
// GetByOffset retrieves the encoded object from the packfile at the given
// offset.
func (p *Packfile) GetByOffset(offset int64) (plumbing.EncodedObject, error) {
if p.closed.Load() {
return nil, fs.ErrClosed
}
if err := p.init(); err != nil {
return nil, err
}
p.m.Lock()
defer p.m.Unlock()
// Re-check after Lock: Close may have flipped closed and torn
// down the scanner between the early Load and the Lock.
if p.closed.Load() {
return nil, fs.ErrClosed
}
return p.getByOffset(offset)
}
// GetSizeByOffset retrieves the size of the encoded object from the
// packfile with the given offset.
func (p *Packfile) GetSizeByOffset(offset int64) (size int64, err error) {
if p.closed.Load() {
return 0, fs.ErrClosed
}
if err := p.init(); err != nil {
return 0, err
}
d, err := p.GetByOffset(offset)
if err != nil {
return 0, err
}
return d.Size(), nil
}
// GetAll returns an iterator with all encoded objects in the packfile.
// The iterator returned is not thread-safe, it should be used in the same
// thread as the Packfile instance.
func (p *Packfile) GetAll() (storer.EncodedObjectIter, error) {
return p.GetByType(plumbing.AnyObject)
}
// GetByType returns all the objects of the given type.
func (p *Packfile) GetByType(typ plumbing.ObjectType) (storer.EncodedObjectIter, error) {
if p.closed.Load() {
return nil, fs.ErrClosed
}
if err := p.init(); err != nil {
return nil, err
}
switch typ {
case plumbing.AnyObject,
plumbing.BlobObject,
plumbing.TreeObject,
plumbing.CommitObject,
plumbing.TagObject:
entries, err := p.EntriesByOffset()
if err != nil {
return nil, err
}
return &objectIter{
p: p,
iter: entries,
typ: typ,
}, nil
default:
return nil, plumbing.ErrInvalidType
}
}
// Scanner returns the Packfile's inner scanner.
//
// Deprecated: this will be removed in future versions of the packfile package
// to avoid exposing the package internals and to improve its thread-safety.
// TODO: Remove Scanner method
func (p *Packfile) Scanner() (*Scanner, error) {
if p.closed.Load() {
return nil, fs.ErrClosed
}
if err := p.init(); err != nil {
return nil, err
}
return p.scanner, nil
}
// ID returns the ID of the packfile, which is the checksum at the end of it.
func (p *Packfile) ID() (plumbing.Hash, error) {
if err := p.init(); err != nil {
return plumbing.ZeroHash, err
}
return p.id, nil
}
// get is not threat-safe, and should only be called within packfile.go.
func (p *Packfile) get(h plumbing.Hash) (plumbing.EncodedObject, error) {
if obj, ok := p.cache.Get(h); ok {
return obj, nil
}
offset, err := p.FindOffset(h)
if err != nil {
return nil, err
}
oh, err := p.headerFromOffset(offset)
if err != nil {
return nil, err
}
return p.objectFromHeader(oh)
}
// getByOffset is not threat-safe, and should only be called within packfile.go.
func (p *Packfile) getByOffset(offset int64) (plumbing.EncodedObject, error) {
h, err := p.FindHash(offset)
if err != nil {
return nil, err
}
if obj, ok := p.cache.Get(h); ok {
return obj, nil
}
oh, err := p.headerFromOffset(offset)
if err != nil {
return nil, err
}
return p.objectFromHeader(oh)
}
func (p *Packfile) init() error {
p.once.Do(func() {
if p.handle == nil && p.resolveHandle != nil {
h, err := p.resolveHandle()
if err != nil {
p.onceErr = fmt.Errorf("packfile: resolve pack handle: %w", err)
return
}
p.handle = h
}
if p.handle == nil && p.file == nil {
p.onceErr = fmt.Errorf("file is not set")
return
}
if p.Index == nil {
p.onceErr = fmt.Errorf("index is not set")
return
}
p.rbuf = gogitsync.GetBufioReader(nil)
opts := []ScannerOption{WithBufioReader(p.rbuf)}
if p.objectIDSize == format.SHA256Size {
opts = append(opts, WithSHA256())
}
var scanSrc io.Reader
if p.handle != nil {
r, err := p.handle.OpenPackReader()
if err != nil {
p.onceErr = fmt.Errorf("packfile: open pack reader: %w", err)
return
}
p.scanReader = r
scanSrc = r
} else {
scanSrc = p.file
}
p.scanner = NewScanner(scanSrc, opts...)
// Validate packfile signature.
if !p.scanner.Scan() {
p.onceErr = p.scanner.Error()
return
}
if p.handle != nil {
id, err := p.handle.PackHash()
if err != nil {
p.onceErr = fmt.Errorf("packfile: read pack hash: %w", err)
return
}
p.id = id
} else {
_, err := p.scanner.Seek(-int64(p.objectIDSize), io.SeekEnd)
if err != nil {
p.onceErr = err
return
}
p.id.ResetBySize(p.objectIDSize)
_, err = p.id.ReadFrom(p.scanner)
if err != nil {
p.onceErr = err
}
}
if p.cache == nil {
p.cache = cache.NewObjectLRUDefault()
}
})
return p.onceErr
}
func (p *Packfile) headerFromOffset(offset int64) (*ObjectHeader, error) {
err := p.scanner.SeekFromStart(offset)
if err != nil {
return nil, err
}
if !p.scanner.Scan() {
if err := p.scanner.Error(); err != nil {
return nil, err
}
return nil, plumbing.ErrObjectNotFound
}
oh := p.scanner.Data().Value().(ObjectHeader)
return &oh, nil
}
// Close the packfile and its resources. Subsequent calls to [Packfile.Get],
// [Packfile.GetByOffset], and the other entry points return [fs.ErrClosed].
// Close is idempotent.
func (p *Packfile) Close() error {
if !p.closed.CompareAndSwap(false, true) {
return nil
}
p.m.Lock()
defer p.m.Unlock()
gogitsync.PutBufioReader(p.rbuf)
if p.handle != nil {
// The resolver owns the handle; close only the scanner cursor.
if p.scanReader != nil {
err := p.scanReader.Close()
p.scanReader = nil
return err
}
return nil
}
closer, ok := p.file.(io.Closer)
if !ok {
return nil
}
return closer.Close()
}
func (p *Packfile) objectFromHeader(oh *ObjectHeader) (plumbing.EncodedObject, error) {
if oh == nil {
return nil, plumbing.ErrObjectNotFound
}
// If we have filesystem, and the object is not a delta type, return a FSObject.
// This avoids having to inflate the object more than once.
if !oh.Type.IsDelta() && p.fs != nil {
var fsObj *FSObject
if p.handle != nil {
fsObj = &FSObject{
hash: oh.ID(),
offset: oh.ContentOffset,
size: oh.Size,
typ: oh.Type,
index: p.Index,
fs: p.fs,
cache: p.cache,
acquireRandom: p.openRandomReader,
}
} else {
fsObj = NewFSObject(
oh.ID(),
oh.Type,
oh.ContentOffset,
oh.Size,
p.Index,
p.fs,
p.file,
p.file.Name(),
p.cache,
)
}
p.cache.Put(fsObj)
return fsObj, nil
}
return p.getMemoryObject(oh)
}
func (p *Packfile) getMemoryObject(oh *ObjectHeader) (plumbing.EncodedObject, error) {
of := format.SHA1
if p.objectIDSize == format.SHA256.Size() {
of = format.SHA256
}
h := plumbing.FromObjectFormat(of)
obj := plumbing.NewMemoryObject(h)
obj.SetSize(oh.Size)
obj.SetType(oh.Type)
w, err := obj.Writer()
if err != nil {
return nil, err
}
defer ioutil.CheckClose(w, &err)
switch oh.Type {
case plumbing.CommitObject, plumbing.TreeObject, plumbing.BlobObject, plumbing.TagObject:
err = p.scanner.inflateContent(oh.ContentOffset, w, oh.Size)
case plumbing.REFDeltaObject, plumbing.OFSDeltaObject:
var parent plumbing.EncodedObject
switch oh.Type {
case plumbing.REFDeltaObject:
var ok bool
parent, ok = p.cache.Get(oh.Reference)
if !ok {
parent, err = p.get(oh.Reference)
}
case plumbing.OFSDeltaObject:
parent, err = p.getByOffset(oh.OffsetReference)
}
if err != nil {
return nil, fmt.Errorf("cannot find base object: %w", err)
}
// The scanner pre-populates oh.content for delta objects when
// running outside low-memory mode; only inflate when we don't
// already hold the bytes, otherwise this would append a
// duplicate copy of the delta payload.
if oh.content == nil {
oh.content = gogitsync.GetBytesBuffer()
err = p.scanner.inflateContent(oh.ContentOffset, oh.content, oh.Size)
if err != nil {
return nil, fmt.Errorf("cannot inflate content: %w", err)
}
}
obj.SetType(parent.Type())
err = ApplyDelta(obj, parent, oh.content)
default:
err = ErrInvalidObject.AddDetails("type %q", oh.Type)
}
if err != nil {
return nil, err
}
p.cache.Put(obj)
return obj, nil
}
package packfile
import (
"io"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/format/idxfile"
)
type objectIter struct {
p *Packfile
typ plumbing.ObjectType
iter idxfile.EntryIter
}
func (i *objectIter) Next() (plumbing.EncodedObject, error) {
if err := i.p.init(); err != nil {
return nil, err
}
i.p.m.Lock()
defer i.p.m.Unlock()
return i.next()
}
func (i *objectIter) next() (plumbing.EncodedObject, error) {
for {
e, err := i.iter.Next()
if err != nil {
return nil, err
}
oh, err := i.p.headerFromOffset(int64(e.Offset))
if err != nil {
return nil, err
}
if i.typ == plumbing.AnyObject {
return i.p.objectFromHeader(oh)
}
// Current object header type is a delta, get the actual object to
// assess the actual type.
if oh.Type.IsDelta() {
o, err := i.p.objectFromHeader(oh)
if err != nil {
return nil, err
}
if o.Type() == i.typ {
return o, nil
}
continue
}
if oh.Type == i.typ {
return i.p.objectFromHeader(oh)
}
continue
}
}
func (i *objectIter) ForEach(f func(plumbing.EncodedObject) error) error {
if err := i.p.init(); err != nil {
return err
}
i.p.m.Lock()
defer i.p.m.Unlock()
for {
o, err := i.next()
if err != nil {
if err == io.EOF {
return nil
}
return err
}
if err := f(o); err != nil {
return err
}
}
}
func (i *objectIter) Close() {
i.p.m.Lock()
defer i.p.m.Unlock()
_ = i.iter.Close()
}
package packfile
import (
billy "github.com/go-git/go-billy/v6"
"github.com/go-git/go-git/v6/plumbing/cache"
"github.com/go-git/go-git/v6/plumbing/format/idxfile"
)
// PackfileOption configures a Packfile.
type PackfileOption func(*Packfile) //nolint:revive // stutters but is a well-established name
// WithCache sets the cache to be used throughout Packfile operations.
// Use this to share existing caches with the Packfile. If not used, a
// new cache instance will be created.
func WithCache(cache cache.Object) PackfileOption {
return func(p *Packfile) {
p.cache = cache
}
}
// WithIdx sets the idxfile for the packfile.
func WithIdx(idx idxfile.Index) PackfileOption {
return func(p *Packfile) {
p.Index = idx
}
}
// WithFs sets the filesystem to be used.
func WithFs(fs billy.Filesystem) PackfileOption {
return func(p *Packfile) {
p.fs = fs
}
}
// WithObjectIDSize sets the size of the object IDs inside the packfile.
// Valid options are hash.SHA1Size and hash.SHA256Size.
//
// When no object ID size is set, hash.SHA1Size will be used.
func WithObjectIDSize(sz int) PackfileOption {
return func(p *Packfile) {
p.objectIDSize = sz
}
}
package packfile
import (
"bytes"
"errors"
"fmt"
"io"
stdsync "sync"
"github.com/go-git/go-git/v6/plumbing"
format "github.com/go-git/go-git/v6/plumbing/format/config"
"github.com/go-git/go-git/v6/plumbing/storer"
"github.com/go-git/go-git/v6/utils/ioutil"
"github.com/go-git/go-git/v6/utils/sync"
)
var (
// ErrReferenceDeltaNotFound is returned when the reference delta is not
// found.
ErrReferenceDeltaNotFound = errors.New("reference delta not found")
// ErrNotSeekableSource is returned when the source for the parser is not
// seekable and a storage was not provided, so it can't be parsed.
ErrNotSeekableSource = errors.New("parser source is not seekable and storage was not provided")
// ErrDeltaNotCached is returned when the delta could not be found in cache.
ErrDeltaNotCached = errors.New("delta could not be found in cache")
// ErrParserConsumed is returned by Parse when called against a Parser
// instance that has already been consumed by a prior Parse call,
// whether that call returned successfully or with an error. Parsers
// are single-shot; construct a new one per pack.
ErrParserConsumed = errors.New("parser already consumed")
)
// maxObjectPreallocBytes caps the up-front size hint passed to
// bytes.Buffer.Grow when staging an object's contents, so a malformed length
// cannot trigger a huge or out-of-range allocation. The buffer still grows
// dynamically as data is written; this is purely a hint cap.
const maxObjectPreallocBytes = 1 << 30 // 1 GiB
// Match upstream Git's pack depth ceiling: pack-objects.h OE_DEPTH_BITS,
// enforced in builtin/pack-objects.c as (1 << OE_DEPTH_BITS) - 1.
const maxDeltaChainDepth = 4095
// growHint returns a non-negative int64 size, clamped to a sane upper bound,
// suitable for passing to bytes.Buffer.Grow.
func growHint(n int64) int {
switch {
case n <= 0:
return 0
case n > maxObjectPreallocBytes:
return maxObjectPreallocBytes
default:
return int(n)
}
}
// Parser decodes a packfile and calls any observer associated to it. Is used
// to generate indexes.
//
// A Parser is single-shot: Parse may be called at most once per
// instance. The cache maps and the per-delta parent pointers built up
// during a Parse call are not reset on entry, so a second call would
// observe the prior call's state — successful or not — and produce
// undefined results; the second call therefore returns
// ErrParserConsumed without running. Construct a new Parser for each
// pack you intend to decode.
type Parser struct {
storage storer.EncodedObjectStorer
cache *parserCache
lowMemoryMode bool
scanner *Scanner
observers []Observer
hasher plumbing.Hasher
objectFormat format.ObjectFormat
checksum plumbing.Hash
m stdsync.Mutex
parsed bool
}
// LowMemoryCapable is implemented by storage types that are capable of
// operating in low-memory mode.
type LowMemoryCapable interface {
// LowMemoryMode defines whether the storage is able and willing for
// the parser to operate in low-memory mode.
LowMemoryMode() bool
}
// NewParser creates a new Parser.
// When a storage is set, the objects are written to storage as they
// are parsed.
func NewParser(data io.Reader, opts ...ParserOption) *Parser {
p := &Parser{
objectFormat: format.DefaultObjectFormat,
}
for _, opt := range opts {
if opt != nil {
opt(p)
}
}
p.hasher = plumbing.NewHasher(p.objectFormat, plumbing.AnyObject, 0)
var sopts []ScannerOption
if p.objectFormat == format.SHA256 {
sopts = append(sopts, WithSHA256())
}
p.scanner = NewScanner(data, sopts...)
if p.storage != nil {
p.scanner.storage = p.storage
lm, ok := p.storage.(LowMemoryCapable)
p.lowMemoryMode = ok && lm.LowMemoryMode()
}
if p.scanner.seeker == nil {
p.lowMemoryMode = false
}
p.scanner.lowMemoryMode = p.lowMemoryMode
p.cache = newParserCache()
return p
}
func (p *Parser) storeOrCache(oh *ObjectHeader) error {
// Only need to store deltas, as the scanner already stored non-delta
// objects.
if p.storage != nil && oh.diskType.IsDelta() {
w, err := p.storage.RawObjectWriter(oh.Type, oh.Size)
if err != nil {
return err
}
defer func() { _ = w.Close() }()
_, err = ioutil.CopyBufferPool(w, oh.content)
if err != nil {
return err
}
}
if p.cache != nil {
o := oh
for p.lowMemoryMode && o.content != nil {
sync.PutBytesBuffer(o.content)
o.content = nil
if o.parent == nil || o.parent.content == nil {
break
}
o = o.parent
}
p.cache.Add(oh)
}
if err := p.onInflatedObjectHeader(oh.Type, oh.Size, oh.Offset); err != nil {
return err
}
return p.onInflatedObjectContent(oh.Hash, oh.Offset, oh.Crc32, nil)
}
func (p *Parser) resetCache(qty int) {
if p.cache != nil {
p.cache.Reset(qty)
}
}
// Parse start decoding phase of the packfile.
func (p *Parser) Parse() (plumbing.Hash, error) {
p.m.Lock()
defer p.m.Unlock()
if p.parsed {
return plumbing.ZeroHash, ErrParserConsumed
}
p.parsed = true
var pendingDeltas []*ObjectHeader
var pendingDeltaREFs []*ObjectHeader
for p.scanner.Scan() {
data := p.scanner.Data()
switch data.Section {
case HeaderSection:
header := data.Value().(Header)
p.resetCache(int(header.ObjectsQty))
_ = p.onHeader(header.ObjectsQty)
case ObjectSection:
oh := data.Value().(ObjectHeader)
if oh.Type.IsDelta() {
oh.Hash.ResetBySize(p.scanner.objectIDSize)
switch oh.Type {
case plumbing.OFSDeltaObject:
pendingDeltas = append(pendingDeltas, &oh)
case plumbing.REFDeltaObject:
pendingDeltaREFs = append(pendingDeltaREFs, &oh)
}
continue
}
if p.lowMemoryMode && oh.content != nil {
sync.PutBytesBuffer(oh.content)
oh.content = nil
}
_ = p.storeOrCache(&oh)
case FooterSection:
p.checksum = data.Value().(plumbing.Hash)
}
}
err := p.scanner.Error()
if err != nil {
if errors.Is(err, io.EOF) && p.scanner.objects == 0 {
return plumbing.ZeroHash, ErrEmptyPackfile
}
return plumbing.ZeroHash, err
}
if err := p.resolveDeltas(pendingDeltas, pendingDeltaREFs); err != nil {
return plumbing.ZeroHash, err
}
// Return to pool all objects used.
go func() {
for _, oh := range p.cache.oi {
if oh.content != nil {
sync.PutBytesBuffer(oh.content)
oh.content = nil
}
}
}()
return p.checksum, p.onFooter(p.checksum)
}
func (p *Parser) ensureContent(oh *ObjectHeader) error {
// Skip if this object already has the correct content.
if oh.content != nil && oh.content.Len() == int(oh.Size) && !oh.Hash.IsZero() {
return nil
}
if oh.content == nil {
oh.content = sync.GetBytesBuffer()
}
var err error
switch {
case !p.lowMemoryMode && oh.content != nil && oh.content.Len() > 0:
source := oh.content
oh.content = sync.GetBytesBuffer()
defer sync.PutBytesBuffer(source)
err = p.applyPatchBaseHeader(oh, source, oh.content, nil)
case p.scanner.seeker != nil:
deltaData := sync.GetBytesBuffer()
defer sync.PutBytesBuffer(deltaData)
err = p.scanner.inflateContent(oh.ContentOffset, deltaData, oh.Size)
if err != nil {
return fmt.Errorf("inflating content at offset %v: %w", oh.ContentOffset, err)
}
err = p.applyPatchBaseHeader(oh, deltaData, oh.content, nil)
default:
return fmt.Errorf("can't ensure content: %w", plumbing.ErrObjectNotFound)
}
if err != nil {
return fmt.Errorf("apply delta patch: %w", err)
}
return nil
}
// resolveDeltas walks the pack's delta DAG depth-first from each
// non-delta base, processing OFS and REF delta children of every parent
// together. Mirrors canonical Git's threaded_second_pass in
// builtin/index-pack.c[1], which advances both kinds of children from
// each in-progress parent in a single walk.
//
// Splitting REF and OFS resolution into separate passes (REF first, OFS
// second) is incorrect: a REF-delta whose base is an OFS-delta in the
// same pack would look up its base hash before the OFS-delta has been
// applied, since the OFS-delta's resolved hash is unknown at scan time.
// The lookup would then misclassify the in-pack base as a thin-pack
// external reference and the chain would fail to resolve.
//
// Any REF-delta not reached through the depth-first walk has a base
// outside this pack and is processed via the external-reference
// placeholder path. An OFS-delta whose recorded negative offset does
// not match any in-pack object header is rejected as malformed input.
//
// [1]: https://github.com/git/git/blob/v2.54.0/builtin/index-pack.c#L1103
func (p *Parser) resolveDeltas(ofsDeltas, refDeltas []*ObjectHeader) error {
// Map sizes correspond to the count of distinct parent offsets /
// hashes, not the count of delta entries. Real packs cluster many
// children under one parent (chains and wide trees), so a hint
// sized to len(deltas) consistently overshoots. Let the maps grow.
ofsChildren := map[int64][]*ObjectHeader{}
for _, d := range ofsDeltas {
ofsChildren[d.OffsetReference] = append(ofsChildren[d.OffsetReference], d)
}
refChildren := map[plumbing.Hash][]*ObjectHeader{}
for _, d := range refDeltas {
refChildren[d.Reference] = append(refChildren[d.Reference], d)
}
var visit func(*ObjectHeader) error
visit = func(parent *ObjectHeader) error {
for _, c := range refChildren[parent.Hash] {
// Two non-delta entries with identical content (or an
// OFS-delta that resolves to the same hash as a non-delta
// elsewhere in the pack) make this child reachable from
// more than one parent; only the first reach resolves it.
if c.parent != nil {
continue
}
if err := p.processDelta(c); err != nil {
return fmt.Errorf("processing ref-delta at offset %v: %w", c.Offset, err)
}
if err := visit(c); err != nil {
return err
}
}
for _, c := range ofsChildren[parent.Offset] {
if c.parent != nil {
continue
}
if err := p.processDelta(c); err != nil {
return fmt.Errorf("processing ofs-delta at offset %v: %w", c.Offset, err)
}
if err := visit(c); err != nil {
return err
}
}
return nil
}
// Snapshot the non-delta bases before walking, since processDelta
// appends resolved deltas to p.cache.oi via storeOrCache. The
// non-delta fraction of a real pack is small (typical 5-20%), so
// preallocating to len(p.cache.oi) would waste most of the slot.
var bases []*ObjectHeader
for _, oh := range p.cache.oi {
if !oh.Type.IsDelta() {
bases = append(bases, oh)
}
}
for _, base := range bases {
if err := visit(base); err != nil {
return err
}
}
for _, d := range refDeltas {
if d.parent != nil {
continue
}
if err := p.processDelta(d); err != nil {
return fmt.Errorf("processing ref-delta at offset %v: %w", d.Offset, err)
}
}
for _, d := range ofsDeltas {
if d.parent != nil {
continue
}
return fmt.Errorf("processing ofs-delta at offset %v: %w", d.Offset, plumbing.ErrObjectNotFound)
}
return nil
}
func (p *Parser) processDelta(oh *ObjectHeader) error {
switch oh.Type {
case plumbing.OFSDeltaObject:
pa, ok := p.cache.oiByOffset[oh.OffsetReference]
if !ok {
return plumbing.ErrObjectNotFound
}
oh.parent = pa
case plumbing.REFDeltaObject:
pa, ok := p.cache.oiByHash[oh.Reference]
if !ok {
// can't find referenced object in this pack file
// this must be a "thin" pack.
oh.parent = &ObjectHeader{ // Placeholder parent
Hash: oh.Reference,
externalRef: true, // mark as an external reference that must be resolved
Type: plumbing.AnyObject,
diskType: plumbing.AnyObject,
}
} else {
oh.parent = pa
}
// For a thin-pack external reference, store the placeholder so
// subsequent REF-deltas naming the same external hash chain
// through this entry. For an in-pack base the write is a no-op.
p.cache.oiByHash[oh.Reference] = oh.parent
default:
return fmt.Errorf("unsupported delta type: %v", oh.Type)
}
if err := checkDeltaChainDepth(oh); err != nil {
return err
}
if err := p.ensureContent(oh); err != nil {
return err
}
return p.storeOrCache(oh)
}
// checkDeltaChainDepth verifies that the delta chain rooted at oh
// stays within [maxDeltaChainDepth] links. The result is cached on
// [ObjectHeader.chainDepth] so a subsequent walk that crosses the
// same parent reuses the work — every entry on the chain ends up
// with its depth set once, which keeps the verification linear in
// the number of distinct objects rather than quadratic in the
// chain length. This mirrors the cached `oe->depth` field that
// upstream Git carries on the object entry in
// `builtin/pack-objects.c`.
func checkDeltaChainDepth(oh *ObjectHeader) error {
if oh.chainDepth > 0 {
return nil
}
var depth int
for current := oh; current != nil && current.isDeltaOnDisk(); current = current.parent {
if current.chainDepth > 0 {
depth += current.chainDepth
if depth > maxDeltaChainDepth {
return fmt.Errorf("%w: delta chain depth exceeds %d", ErrMalformedPackfile, maxDeltaChainDepth)
}
break
}
depth++
if depth > maxDeltaChainDepth {
return fmt.Errorf("%w: delta chain depth exceeds %d", ErrMalformedPackfile, maxDeltaChainDepth)
}
}
oh.chainDepth = depth
return nil
}
func (oh *ObjectHeader) isDeltaOnDisk() bool {
return oh.Type.IsDelta() || oh.diskType.IsDelta()
}
// parentReader returns a [io.ReaderAt] for the decompressed contents
// of the parent.
func (p *Parser) parentReader(parent *ObjectHeader) (io.ReaderAt, error) {
if parent.content != nil && parent.content.Len() > 0 {
return bytes.NewReader(parent.content.Bytes()), nil
}
// If parent is a Delta object, the inflated object must come
// from either cache or storage, else we would need to inflate
// it to then inflate the current object, which could go on
// indefinitely.
if p.storage != nil && !parent.Hash.IsZero() {
obj, err := p.storage.EncodedObject(parent.Type, parent.Hash)
if err == nil {
// Ensure that external references have the correct type and size.
parent.Type = obj.Type()
parent.Size = obj.Size()
r, err := obj.Reader()
if err == nil {
defer func() { _ = r.Close() }()
if parent.content == nil {
parent.content = sync.GetBytesBuffer()
}
parent.content.Grow(growHint(parent.Size))
_, err = ioutil.CopyBufferPool(parent.content, r)
if err == nil {
return bytes.NewReader(parent.content.Bytes()), nil
}
}
}
}
// If the parent is not an external ref and we don't have the
// content offset, we won't be able to inflate via seeking through
// the packfile.
if !parent.externalRef && parent.ContentOffset == 0 {
return nil, plumbing.ErrObjectNotFound
}
// Not a seeker data source, so avoid seeking the content.
if p.scanner.seeker == nil {
return nil, plumbing.ErrObjectNotFound
}
if parent.content == nil {
parent.content = sync.GetBytesBuffer()
}
parent.content.Grow(growHint(parent.Size))
err := p.scanner.inflateContent(parent.ContentOffset, parent.content, parent.Size)
if err != nil {
return nil, ErrReferenceDeltaNotFound
}
return bytes.NewReader(parent.content.Bytes()), nil
}
func (p *Parser) applyPatchBaseHeader(ota *ObjectHeader, delta io.Reader, target io.Writer, wh objectHeaderWriter) error {
if target == nil {
return fmt.Errorf("cannot apply patch against nil target")
}
parentContents, err := p.parentReader(ota.parent)
if err != nil {
return err
}
typ := ota.Type
if ota.Hash.IsZero() {
typ = ota.parent.Type
}
deltaBuf := sync.GetBufioReader(delta)
defer sync.PutBufioReader(deltaBuf)
sz, h, err := patchDeltaWriter(target, parentContents, deltaBuf, typ, wh, p.objectFormat)
if err != nil {
return err
}
if ota.Hash.IsZero() {
ota.Type = typ
ota.Size = int64(sz)
ota.Hash = h
}
return nil
}
func (p *Parser) forEachObserver(f func(o Observer) error) error {
for _, o := range p.observers {
if err := f(o); err != nil {
return err
}
}
return nil
}
func (p *Parser) onHeader(count uint32) error {
return p.forEachObserver(func(o Observer) error {
return o.OnHeader(count)
})
}
func (p *Parser) onInflatedObjectHeader(
t plumbing.ObjectType,
objSize int64,
pos int64,
) error {
return p.forEachObserver(func(o Observer) error {
return o.OnInflatedObjectHeader(t, objSize, pos)
})
}
func (p *Parser) onInflatedObjectContent(
h plumbing.Hash,
pos int64,
crc uint32,
content []byte,
) error {
return p.forEachObserver(func(o Observer) error {
return o.OnInflatedObjectContent(h, pos, crc, content)
})
}
func (p *Parser) onFooter(h plumbing.Hash) error {
return p.forEachObserver(func(o Observer) error {
return o.OnFooter(h)
})
}
package packfile
import (
"slices"
"github.com/go-git/go-git/v6/plumbing"
)
// maxObjectsPrealloc caps the up-front capacity reserved from the pack's
// declared object count, so a header advertising an absurd quantity cannot
// trigger a multi-gigabyte allocation. The slice and maps still grow
// organically beyond this hint.
const maxObjectsPrealloc = 1 << 16 // 64 Ki entries
func newParserCache() *parserCache {
c := &parserCache{}
return c
}
// parserCache defines the cache used within the parser.
// This is not thread safe by itself, and relies on the parser to
// enforce thread-safety.
type parserCache struct {
oi []*ObjectHeader
oiByHash map[plumbing.Hash]*ObjectHeader
oiByOffset map[int64]*ObjectHeader
}
func (c *parserCache) Add(oh *ObjectHeader) {
c.oiByHash[oh.Hash] = oh
c.oiByOffset[oh.Offset] = oh
c.oi = append(c.oi, oh)
}
func (c *parserCache) Reset(n int) {
hint := min(max(n, 0), maxObjectsPrealloc)
if c.oi == nil {
c.oi = make([]*ObjectHeader, 0, hint)
c.oiByHash = make(map[plumbing.Hash]*ObjectHeader, hint)
c.oiByOffset = make(map[int64]*ObjectHeader, hint)
} else {
c.oi = c.oi[:0]
c.oi = slices.Grow(c.oi, hint)
clear(c.oiByHash)
clear(c.oiByOffset)
}
}
package packfile
import (
"github.com/go-git/go-git/v6/plumbing/format/config"
"github.com/go-git/go-git/v6/plumbing/storer"
)
// ParserOption configures a Parser.
type ParserOption func(*Parser)
// WithStorage sets the storage to be used while parsing a pack file.
func WithStorage(storage storer.EncodedObjectStorer) ParserOption {
return func(p *Parser) {
p.storage = storage
}
}
// WithScannerObservers sets the observers to be notified during the
// scanning or parsing of a pack file. The scanner is responsible for
// notifying observers around general pack file information, such as
// header and footer. The scanner also notifies object headers for
// non-delta objects.
//
// Delta objects are notified as part of the parser logic.
func WithScannerObservers(ob ...Observer) ParserOption {
return func(p *Parser) {
p.observers = ob
}
}
// WithObjectFormat sets the object format for the parser.
func WithObjectFormat(of config.ObjectFormat) ParserOption {
return func(p *Parser) {
if of == config.UnsetObjectFormat {
of = config.DefaultObjectFormat
}
p.objectFormat = of
}
}
// WithHighMemoryMode optimises the parser for speed rather than
// for memory consumption, making the Parser faster from an execution
// time perspective, but yielding much more allocations, which in the
// long run could make the application slower due to GC pressure.
//
// When the parser is being used without a storage, this is enabled
// automatically, as it can't operate without it. Some storage types
// may no support low memory mode (i.e. memory storage), for storage
// types that do support it, this becomes an opt-in feature.
//
// When enabled the inflated content of all delta objects (ofs and ref)
// will be loaded into cache, making it faster to navigate through them.
// If the reader provided to the parser does not implement io.Seeker,
// full objects may also be loaded into memory.
func WithHighMemoryMode() ParserOption {
return func(p *Parser) {
p.lowMemoryMode = false
}
}
package packfile
import (
"bufio"
"bytes"
"errors"
"fmt"
"io"
"math"
"github.com/go-git/go-git/v6/plumbing"
format "github.com/go-git/go-git/v6/plumbing/format/config"
packutil "github.com/go-git/go-git/v6/plumbing/format/packfile/util"
"github.com/go-git/go-git/v6/utils/ioutil"
"github.com/go-git/go-git/v6/utils/sync"
)
// See https://github.com/git/git/blob/49fa3dc76179e04b0833542fa52d0f287a4955ac/delta.h
// https://github.com/git/git/blob/c2c5f6b1e479f2c38e0e01345350620944e3527f/patch-delta.c,
// and https://github.com/tarruda/node-git-core/blob/master/src/js/delta.js
// for details about the delta format.
// Delta errors.
var (
ErrInvalidDelta = errors.New("invalid delta")
ErrDeltaCmd = errors.New("wrong delta command")
)
const (
// maxPatchPreemptionSize defines what is the max size of bytes to be
// preemptively made available for a patch operation.
maxPatchPreemptionSize uint = 65536
// minDeltaSize is the smallest valid delta: a 1-byte srcSz LEB128
// header followed by a 1-byte targetSz LEB128 header (the
// shortest case being targetSz=0 with no operations).
minDeltaSize = 2
)
type offset struct {
mask byte
shift uint
}
var offsets = []offset{
{mask: 0x01, shift: 0},
{mask: 0x02, shift: 8},
{mask: 0x04, shift: 16},
{mask: 0x08, shift: 24},
}
var sizes = []offset{
{mask: 0x10, shift: 0},
{mask: 0x20, shift: 8},
{mask: 0x40, shift: 16},
}
// ApplyDelta writes to target the result of applying the modification deltas in delta to base.
func ApplyDelta(target, base plumbing.EncodedObject, delta *bytes.Buffer) (err error) {
r, err := base.Reader()
if err != nil {
return err
}
defer ioutil.CheckClose(r, &err)
w, err := target.Writer()
if err != nil {
return err
}
defer ioutil.CheckClose(w, &err)
buf := sync.GetBytesBuffer()
defer sync.PutBytesBuffer(buf)
_, err = buf.ReadFrom(r)
if err != nil {
return err
}
src := buf.Bytes()
dst := sync.GetBytesBuffer()
defer sync.PutBytesBuffer(dst)
err = patchDelta(dst, src, delta.Bytes())
if err != nil {
return err
}
target.SetSize(int64(dst.Len()))
_, err = ioutil.CopyBufferPool(w, dst)
return err
}
// PatchDelta returns the result of applying the modification deltas in delta to src.
// An error will be returned if delta is corrupted (ErrInvalidDelta) or an action command
// is not copy from source or copy from delta (ErrDeltaCmd).
func PatchDelta(src, delta []byte) ([]byte, error) {
if len(src) == 0 || len(delta) < minDeltaSize {
return nil, ErrInvalidDelta
}
b := &bytes.Buffer{}
if err := patchDelta(b, src, delta); err != nil {
return nil, err
}
return b.Bytes(), nil
}
// ReaderFromDelta returns a reader that applies a delta to a base object.
func ReaderFromDelta(base plumbing.EncodedObject, deltaRC io.Reader) (io.ReadCloser, error) {
deltaBuf := bufio.NewReaderSize(deltaRC, 1024)
srcSz, err := packutil.DecodeLEB128FromReader(deltaBuf)
if err != nil {
if err == io.EOF {
return nil, ErrInvalidDelta
}
return nil, err
}
if srcSz != uint(base.Size()) {
return nil, ErrInvalidDelta
}
targetSz, err := packutil.DecodeLEB128FromReader(deltaBuf)
if err != nil {
if err == io.EOF {
return nil, ErrInvalidDelta
}
return nil, err
}
remainingTargetSz := targetSz
dstRd, dstWr := io.Pipe()
go func() {
baseRd, err := base.Reader()
if err != nil {
_ = dstWr.CloseWithError(ErrInvalidDelta)
return
}
defer func() { _ = baseRd.Close() }()
baseBuf := bufio.NewReader(baseRd)
basePos := uint(0)
for remainingTargetSz > 0 {
cmd, err := deltaBuf.ReadByte()
if err == io.EOF {
_ = dstWr.CloseWithError(ErrInvalidDelta)
return
}
if err != nil {
_ = dstWr.CloseWithError(err)
return
}
switch {
case isCopyFromSrc(cmd):
offset, err := decodeOffsetByteReader(cmd, deltaBuf)
if err != nil {
_ = dstWr.CloseWithError(err)
return
}
sz, err := decodeSizeByteReader(cmd, deltaBuf)
if err != nil {
_ = dstWr.CloseWithError(err)
return
}
if invalidSize(sz, remainingTargetSz) ||
invalidOffsetSize(offset, sz, srcSz) {
_ = dstWr.CloseWithError(ErrInvalidDelta)
return
}
discard := offset - basePos
if basePos > offset {
_ = baseRd.Close()
baseRd, err = base.Reader()
if err != nil {
_ = dstWr.CloseWithError(ErrInvalidDelta)
return
}
baseBuf.Reset(baseRd)
discard = offset
}
for discard > math.MaxInt32 {
n, err := baseBuf.Discard(math.MaxInt32)
if err != nil {
_ = dstWr.CloseWithError(err)
return
}
basePos += uint(n)
discard -= uint(n)
}
for discard > 0 {
n, err := baseBuf.Discard(int(discard))
if err != nil {
_ = dstWr.CloseWithError(err)
return
}
basePos += uint(n)
discard -= uint(n)
}
if _, err := ioutil.CopyBufferPool(dstWr, io.LimitReader(baseBuf, int64(sz))); err != nil {
_ = dstWr.CloseWithError(err)
return
}
remainingTargetSz -= sz
basePos += sz
case isCopyFromDelta(cmd):
sz := uint(cmd) // cmd is the size itself
if invalidSize(sz, remainingTargetSz) {
_ = dstWr.CloseWithError(ErrInvalidDelta)
return
}
if _, err := ioutil.CopyBufferPool(dstWr, io.LimitReader(deltaBuf, int64(sz))); err != nil {
_ = dstWr.CloseWithError(err)
return
}
remainingTargetSz -= sz
default:
_ = dstWr.CloseWithError(ErrDeltaCmd)
return
}
}
// Mirror upstream's `data != top` post-loop check: every byte
// of the delta payload must be consumed.
if _, err := deltaBuf.ReadByte(); err == nil {
_ = dstWr.CloseWithError(ErrInvalidDelta)
return
} else if err != io.EOF {
_ = dstWr.CloseWithError(err)
return
}
_ = dstWr.Close()
}()
return dstRd, nil
}
func patchDelta(dst *bytes.Buffer, src, delta []byte) error {
srcSz, delta, err := packutil.DecodeLEB128(delta)
if err != nil {
return fmt.Errorf("%w: %w", ErrInvalidDelta, err)
}
if srcSz != uint(len(src)) {
return ErrInvalidDelta
}
targetSz, delta, err := packutil.DecodeLEB128(delta)
if err != nil {
return fmt.Errorf("%w: %w", ErrInvalidDelta, err)
}
remainingTargetSz := targetSz
growSz := min(targetSz, maxPatchPreemptionSize)
dst.Grow(int(growSz))
for remainingTargetSz > 0 {
if len(delta) == 0 {
return ErrInvalidDelta
}
cmd := delta[0]
delta = delta[1:]
switch {
case isCopyFromSrc(cmd):
var offset, sz uint
var err error
offset, delta, err = decodeOffset(cmd, delta)
if err != nil {
return err
}
sz, delta, err = decodeSize(cmd, delta)
if err != nil {
return err
}
if invalidSize(sz, remainingTargetSz) ||
invalidOffsetSize(offset, sz, srcSz) {
return ErrInvalidDelta
}
dst.Write(src[offset : offset+sz])
remainingTargetSz -= sz
case isCopyFromDelta(cmd):
sz := uint(cmd) // cmd is the size itself
if invalidSize(sz, remainingTargetSz) {
return ErrInvalidDelta
}
if uint(len(delta)) < sz {
return ErrInvalidDelta
}
dst.Write(delta[0:sz])
remainingTargetSz -= sz
delta = delta[sz:]
default:
return ErrDeltaCmd
}
}
// Mirror upstream's `data != top` post-loop check: every byte of
// the delta payload must be consumed.
if len(delta) != 0 {
return ErrInvalidDelta
}
return nil
}
func patchDeltaWriter(dst io.Writer, base io.ReaderAt, deltaBuf *bufio.Reader,
typ plumbing.ObjectType, writeHeader objectHeaderWriter, of format.ObjectFormat,
) (uint, plumbing.Hash, error) {
srcSz, err := packutil.DecodeLEB128FromReader(deltaBuf)
if err != nil {
if err == io.EOF {
return 0, plumbing.ZeroHash, ErrInvalidDelta
}
return 0, plumbing.ZeroHash, err
}
if r, ok := base.(*bytes.Reader); ok && srcSz != uint(r.Size()) {
return 0, plumbing.ZeroHash, ErrInvalidDelta
}
targetSz, err := packutil.DecodeLEB128FromReader(deltaBuf)
if err != nil {
if err == io.EOF {
return 0, plumbing.ZeroHash, ErrInvalidDelta
}
return 0, plumbing.ZeroHash, err
}
// Avoid several interactions expanding the buffer, which can be quite
// inefficient on large deltas. The preemptive growth is capped at
// maxPatchPreemptionSize so that the header-derived targetSz cannot
// drive a large allocation; this mirrors patchDelta.
if b, ok := dst.(*bytes.Buffer); ok {
// The hint is clamped because targetSz is decoded from untrusted
// input and Grow takes a non-negative int.
growSz := min(targetSz, maxPatchPreemptionSize)
b.Grow(int(growSz))
}
// If header still needs to be written, caller will provide
// a LazyObjectWriterHeader. This seems to be the case when
// dealing with thin-packs.
if writeHeader != nil {
err = writeHeader(typ, int64(targetSz))
if err != nil {
return 0, plumbing.ZeroHash, fmt.Errorf("could not lazy write header: %w", err)
}
}
remainingTargetSz := targetSz
hasher := plumbing.NewHasher(of, typ, int64(targetSz))
mw := io.MultiWriter(dst, hasher)
bufp := sync.GetByteSlice()
defer sync.PutByteSlice(bufp)
sr := io.NewSectionReader(base, int64(0), int64(srcSz))
// Keep both the io.LimitedReader types, so we can reset N.
baselr := io.LimitReader(sr, 0).(*io.LimitedReader)
deltalr := io.LimitReader(deltaBuf, 0).(*io.LimitedReader)
for remainingTargetSz > 0 {
buf := *bufp
cmd, err := deltaBuf.ReadByte()
if err == io.EOF {
return 0, plumbing.ZeroHash, ErrInvalidDelta
}
if err != nil {
return 0, plumbing.ZeroHash, err
}
switch {
case isCopyFromSrc(cmd):
offset, err := decodeOffsetByteReader(cmd, deltaBuf)
if err != nil {
return 0, plumbing.ZeroHash, err
}
sz, err := decodeSizeByteReader(cmd, deltaBuf)
if err != nil {
return 0, plumbing.ZeroHash, err
}
if invalidSize(sz, remainingTargetSz) ||
invalidOffsetSize(offset, sz, srcSz) {
return 0, plumbing.ZeroHash, ErrInvalidDelta
}
if _, err := sr.Seek(int64(offset), io.SeekStart); err != nil {
return 0, plumbing.ZeroHash, err
}
baselr.N = int64(sz)
if _, err := io.CopyBuffer(mw, baselr, buf); err != nil {
return 0, plumbing.ZeroHash, err
}
remainingTargetSz -= sz
case isCopyFromDelta(cmd):
sz := uint(cmd) // cmd is the size itself
if invalidSize(sz, remainingTargetSz) {
return 0, plumbing.ZeroHash, ErrInvalidDelta
}
deltalr.N = int64(sz)
if _, err := io.CopyBuffer(mw, deltalr, buf); err != nil {
return 0, plumbing.ZeroHash, err
}
remainingTargetSz -= sz
default:
return 0, plumbing.ZeroHash, ErrDeltaCmd
}
}
// Mirror upstream's `data != top` post-loop check: every byte of
// the delta payload must be consumed.
if _, err := deltaBuf.ReadByte(); err == nil {
return 0, plumbing.ZeroHash, ErrInvalidDelta
} else if err != io.EOF {
return 0, plumbing.ZeroHash, err
}
return targetSz, hasher.Sum(), nil
}
func isCopyFromSrc(cmd byte) bool {
return (cmd & maskContinue) != 0
}
func isCopyFromDelta(cmd byte) bool {
return (cmd&maskContinue) == 0 && cmd != 0
}
func decodeOffsetByteReader(cmd byte, delta io.ByteReader) (uint, error) {
var offset uint
for _, o := range offsets {
if (cmd & o.mask) != 0 {
next, err := delta.ReadByte()
if err != nil {
return 0, err
}
offset |= uint(next) << o.shift
}
}
return offset, nil
}
func decodeOffset(cmd byte, delta []byte) (uint, []byte, error) {
var offset uint
for _, o := range offsets {
if (cmd & o.mask) != 0 {
if len(delta) == 0 {
return 0, nil, ErrInvalidDelta
}
offset |= uint(delta[0]) << o.shift
delta = delta[1:]
}
}
return offset, delta, nil
}
func decodeSizeByteReader(cmd byte, delta io.ByteReader) (uint, error) {
var sz uint
for _, s := range sizes {
if (cmd & s.mask) != 0 {
next, err := delta.ReadByte()
if err != nil {
return 0, err
}
sz |= uint(next) << s.shift
}
}
if sz == 0 {
sz = maxCopySize
}
return sz, nil
}
func decodeSize(cmd byte, delta []byte) (uint, []byte, error) {
var sz uint
for _, s := range sizes {
if (cmd & s.mask) != 0 {
if len(delta) == 0 {
return 0, nil, ErrInvalidDelta
}
sz |= uint(delta[0]) << s.shift
delta = delta[1:]
}
}
if sz == 0 {
sz = maxCopySize
}
return sz, delta, nil
}
// invalidSize reports whether sz exceeds the remaining target size.
func invalidSize(sz, remaining uint) bool {
return sz > remaining
}
func invalidOffsetSize(offset, sz, srcSz uint) bool {
return sumOverflows(offset, sz) ||
offset+sz > srcSz
}
func sumOverflows(a, b uint) bool {
return a+b < a
}
package packfile
import (
"bufio"
"bytes"
"crypto"
"encoding/hex"
"errors"
"fmt"
"hash"
"hash/crc32"
"io"
"sync"
"github.com/go-git/go-git/v6/plumbing"
format "github.com/go-git/go-git/v6/plumbing/format/config"
packutil "github.com/go-git/go-git/v6/plumbing/format/packfile/util"
gogithash "github.com/go-git/go-git/v6/plumbing/hash"
"github.com/go-git/go-git/v6/plumbing/storer"
"github.com/go-git/go-git/v6/utils/binary"
"github.com/go-git/go-git/v6/utils/ioutil"
gogitsync "github.com/go-git/go-git/v6/utils/sync"
)
var (
// ErrEmptyPackfile is returned by ReadHeader when no data is found in the packfile.
ErrEmptyPackfile = NewError("empty packfile")
// ErrBadSignature is returned by ReadHeader when the signature in the packfile is incorrect.
ErrBadSignature = NewError("bad signature")
// ErrMalformedPackfile is returned when the packfile format is incorrect.
ErrMalformedPackfile = NewError("malformed pack file")
// ErrUnsupportedVersion is returned by ReadHeader when the packfile version is
// different than VersionSupported.
ErrUnsupportedVersion = NewError("unsupported packfile version")
// ErrSeekNotSupported returned if seek is not support.
ErrSeekNotSupported = NewError("not seek support")
// ErrInflatedSizeMismatch is returned when a packfile object inflates to
// more bytes than the size declared in its object header. A well-formed
// packfile never produces more data than the declared size; exceeding it
// indicates a structurally invalid entry.
ErrInflatedSizeMismatch = errors.New("packfile: inflated object exceeds declared size")
)
// boundedWriter passes writes through to w up to limit bytes total, then
// returns ErrInflatedSizeMismatch. It is used to enforce that a packfile
// object's inflated length does not exceed the size declared in its header.
type boundedWriter struct {
w io.Writer
limit int64
n int64
}
// Write forwards p to the underlying writer while keeping the running total
// at or below limit. On overrun it forwards the legal prefix and reports
// the number of bytes actually consumed alongside ErrInflatedSizeMismatch,
// matching the contract in io.Writer. A write error from the underlying
// writer during overrun-handling is joined with ErrInflatedSizeMismatch so
// it is not silently dropped.
func (b *boundedWriter) Write(p []byte) (int, error) {
if b.n+int64(len(p)) > b.limit {
remain := int(b.limit - b.n)
err := error(ErrInflatedSizeMismatch)
if remain > 0 {
n, werr := b.w.Write(p[:remain])
b.n += int64(n)
if werr != nil {
err = errors.Join(ErrInflatedSizeMismatch, werr)
}
return n, err
}
return 0, err
}
n, err := b.w.Write(p)
b.n += int64(n)
return n, err
}
// BoundedReadCloser wraps a ReadCloser and reports ErrInflatedSizeMismatch
// once more than limit bytes have been read. It is used by the on-demand
// object readers to enforce the same bound that the scanner applies during
// a forward scan, so a lazy Read of a packfile object cannot stream past
// its declared inflated size.
//
// The implementation builds on io.LimitedReader with the standard
// overrun-detection trick: request limit+1 bytes from the underlying so
// that the moment the sentinel byte materializes (LimitedReader.N drops
// to zero) we know the source produced more than limit bytes.
type BoundedReadCloser struct {
lr io.LimitedReader
closer io.Closer
overrun bool
}
// NewBoundedReadCloser wraps rc so that the cumulative bytes returned from
// Read never exceed limit. The first call that would have returned a byte
// past limit instead returns ErrInflatedSizeMismatch; subsequent calls
// keep returning the same error. A negative limit is treated as zero, so
// the first byte produced by rc surfaces ErrInflatedSizeMismatch.
func NewBoundedReadCloser(rc io.ReadCloser, limit int64) *BoundedReadCloser {
if limit < 0 {
limit = 0
}
return &BoundedReadCloser{
lr: io.LimitedReader{R: rc, N: limit + 1},
closer: rc,
}
}
// Read forwards Read up to the configured byte limit. When the underlying
// stream produces the limit+1 sentinel byte, the legal prefix is returned
// alongside ErrInflatedSizeMismatch; on subsequent calls only the error
// is returned.
func (b *BoundedReadCloser) Read(p []byte) (int, error) {
if b.overrun {
return 0, ErrInflatedSizeMismatch
}
n, err := b.lr.Read(p)
if b.lr.N == 0 {
b.overrun = true
return n - 1, ErrInflatedSizeMismatch
}
return n, err
}
// Close closes the underlying ReadCloser.
func (b *BoundedReadCloser) Close() error { return b.closer.Close() }
// Scanner provides sequential access to the data stored in a Git packfile.
//
// A Git packfile is a compressed binary format that stores multiple Git objects,
// such as commits, trees, delta objects and blobs. These packfiles are used to
// reduce the size of data when transferring or storing Git repositories.
//
// A Git packfile is structured as follows:
//
// +----------------------------------------------------+
// | PACK File Header |
// +----------------------------------------------------+
// | "PACK" | Version Number | Number of Objects |
// | (4 bytes) | (4 bytes) | (4 bytes) |
// +----------------------------------------------------+
// | Object Entry #1 |
// +----------------------------------------------------+
// | Object Header | Compressed Object Data / Delta |
// | (type + size) | (var-length, zlib compressed) |
// +----------------------------------------------------+
// | ... |
// +----------------------------------------------------+
// | PACK File Footer |
// +----------------------------------------------------+
// | SHA-1 Checksum (20 bytes) |
// +----------------------------------------------------+
//
// For upstream docs, refer to https://git-scm.com/docs/gitformat-pack.
type Scanner struct {
// version holds the packfile version.
version Version
// objects holds the quantity of objects within the packfile.
objects uint32
// objIndex is the current index when going through the packfile objects.
objIndex int
// hasher is used to hash non-delta objects.
hasher plumbing.Hasher
// crc is used to generate the CRC-32 checksum of each object's content.
crc hash.Hash32
// packhash hashes the pack contents so that at the end it is able to
// validate the packfile's footer checksum against the calculated hash.
packhash gogithash.Hash
// objectIdSize holds the object ID size.
objectIDSize int
// next holds what state function should be executed on the next
// call to Scan().
nextFn stateFn
// packData holds the data for the last successful call to Scan().
packData PackData
// err holds the first error that occurred.
err error
m sync.Mutex
// storage is optional, and when set is used to store full objects found.
// Note that delta objects are not stored.
storage storer.EncodedObjectStorer
*scannerReader
rbuf *bufio.Reader
lowMemoryMode bool
}
// NewScanner creates a new instance of Scanner.
func NewScanner(rs io.Reader, opts ...ScannerOption) *Scanner {
crc := crc32.NewIEEE()
packhash := gogithash.New(crypto.SHA1)
r := &Scanner{
objIndex: -1,
hasher: plumbing.NewHasher(format.SHA1, plumbing.AnyObject, 0),
crc: crc,
packhash: packhash,
nextFn: packHeaderSignature,
// Set the default size, which can be overridden by opts.
objectIDSize: packhash.Size(),
}
for _, opt := range opts {
opt(r)
}
r.scannerReader = newScannerReader(rs, io.MultiWriter(crc, r.packhash), r.rbuf)
return r
}
// Scan scans a Packfile sequently. Each call will navigate from a section
// to the next, until the entire file is read.
//
// The section data can be accessed via calls to Data(). Example:
//
// for scanner.Scan() {
// v := scanner.Data().Value()
//
// switch scanner.Data().Section {
// case HeaderSection:
// header := v.(Header)
// fmt.Println("[Header] Objects Qty:", header.ObjectsQty)
// case ObjectSection:
// oh := v.(ObjectHeader)
// fmt.Println("[Object] Object Type:", oh.Type)
// case FooterSection:
// checksum := v.(plumbing.Hash)
// fmt.Println("[Footer] Checksum:", checksum)
// }
// }
func (r *Scanner) Scan() bool {
r.m.Lock()
defer r.m.Unlock()
if r.err != nil || r.nextFn == nil {
return false
}
if err := scan(r); err != nil {
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
err = fmt.Errorf("%w: %w", ErrMalformedPackfile, err)
}
r.err = err
return false
}
return true
}
// Reset resets the current scanner, enabling it to be used to scan the
// same Packfile again.
func (r *Scanner) Reset() error {
if err := r.Flush(); err != nil {
return err
}
if _, err := r.Seek(0, io.SeekStart); err != nil {
return err
}
r.packhash.Reset()
r.objIndex = -1
r.version = 0
r.objects = 0
r.packData = PackData{}
r.err = nil
r.nextFn = packHeaderSignature
return nil
}
// Data returns the pack data based on the last call to Scan().
func (r *Scanner) Data() PackData {
return r.packData
}
// Data returns the first error that occurred on the last call to Scan().
// Once an error occurs, calls to Scan() becomes a no-op.
func (r *Scanner) Error() error {
return r.err
}
// SeekFromStart seeks to the given offset from the start of the packfile.
func (r *Scanner) SeekFromStart(offset int64) error {
if err := r.Reset(); err != nil {
return err
}
if !r.Scan() {
return fmt.Errorf("failed to reset and read header")
}
_, err := r.Seek(offset, io.SeekStart)
return err
}
// WriteObject writes the content of the given ObjectHeader to the provided writer.
func (r *Scanner) WriteObject(oh *ObjectHeader, writer io.Writer) error {
if oh.content != nil && oh.content.Len() > 0 {
_, err := ioutil.CopyBufferPool(writer, oh.content)
return err
}
// If the oh is not an external ref and we don't have the
// content offset, we won't be able to inflate via seeking through
// the packfile.
if oh.externalRef && oh.ContentOffset == 0 {
return plumbing.ErrObjectNotFound
}
// Not a seeker data source.
if r.seeker == nil {
return plumbing.ErrObjectNotFound
}
err := r.inflateContent(oh.ContentOffset, writer, oh.Size)
if err != nil {
return ErrReferenceDeltaNotFound
}
return nil
}
func (r *Scanner) inflateContent(contentOffset int64, writer io.Writer, declaredSize int64) error {
bounded := &boundedWriter{w: writer, limit: declaredSize}
_, err := r.Seek(contentOffset, io.SeekStart)
if err != nil {
return err
}
zr, err := gogitsync.GetZlibReader(r.scannerReader)
if err != nil {
return fmt.Errorf("zlib reset error: %w", err)
}
defer gogitsync.PutZlibReader(zr)
_, err = ioutil.CopyBufferPool(bounded, zr)
return err
}
// scan goes through the next stateFn.
//
// State functions are chained by returning a non-nil value for stateFn.
// In such cases, the returned stateFn will be called immediately after
// the current func.
func scan(r *Scanner) error {
var err error
for state := r.nextFn; state != nil; {
state, err = state(r)
if err != nil {
return err
}
}
return nil
}
// stateFn defines each individual state within the state machine that
// represents a packfile.
type stateFn func(*Scanner) (stateFn, error)
// packHeaderSignature validates the packfile's header signature and
// returns [ErrBadSignature] if the value provided is invalid.
//
// This is always the first state of a packfile and starts the chain
// that handles the entire packfile header.
func packHeaderSignature(r *Scanner) (stateFn, error) {
start := make([]byte, 4)
n, err := r.Read(start)
if err != nil {
if n == 0 && err == io.EOF {
return nil, ErrEmptyPackfile
}
return nil, fmt.Errorf("read signature: %w", err)
}
if bytes.Equal(start, signature) {
return packVersion, nil
}
return nil, fmt.Errorf("%w: %w", ErrMalformedPackfile, ErrBadSignature)
}
// packVersion parses the packfile version. It returns [ErrMalformedPackfile]
// when the version cannot be parsed. If a valid version is parsed, but it is
// not currently supported, it returns [ErrUnsupportedVersion] instead.
func packVersion(r *Scanner) (stateFn, error) {
version, err := binary.ReadUint32(r.scannerReader)
if err != nil {
return nil, fmt.Errorf("read version: %w", err)
}
v := Version(version)
if !v.Supported() {
return nil, ErrUnsupportedVersion
}
r.version = v
return packObjectsQty, nil
}
// packObjectsQty parses the quantity of objects that the packfile contains.
// If the value cannot be parsed, [ErrMalformedPackfile] is returned.
//
// This state ends the packfile header chain.
func packObjectsQty(r *Scanner) (stateFn, error) {
qty, err := binary.ReadUint32(r.scannerReader)
if err != nil {
return nil, fmt.Errorf("read number of objects: %w", err)
}
if qty == 0 {
return packFooter, nil
}
r.objects = qty
r.packData = PackData{
Section: HeaderSection,
header: Header{Version: r.version, ObjectsQty: r.objects},
}
r.nextFn = objectEntry
return nil, nil
}
// objectEntry handles the object entries within a packfile. This is generally
// split between object headers and their contents.
//
// The object header contains the object type and size. If the type cannot be parsed,
// [ErrMalformedPackfile] is returned.
//
// When SHA256 is enabled, the scanner will also calculate the SHA256 for each object.
func objectEntry(r *Scanner) (stateFn, error) {
if r.objIndex+1 >= int(r.objects) {
return packFooter, nil
}
r.objIndex++
offset := r.offset
if err := r.Flush(); err != nil {
return nil, err
}
r.crc.Reset()
b := []byte{0}
_, err := r.Read(b)
if err != nil {
return nil, err
}
typ := packutil.ObjectType(b[0])
if !typ.Valid() {
return nil, fmt.Errorf("%w: invalid object type: %v", ErrMalformedPackfile, b[0])
}
size, err := packutil.VariableLengthSize(b[0], r)
if err != nil {
if errors.Is(err, packutil.ErrLengthOverflow) {
return nil, fmt.Errorf("%w: %w", ErrMalformedPackfile, err)
}
return nil, err
}
oh := ObjectHeader{
Offset: offset,
Type: typ,
diskType: typ,
Size: int64(size),
}
switch oh.Type {
case plumbing.OFSDeltaObject, plumbing.REFDeltaObject:
oh.Hash.ResetBySize(r.objectIDSize)
// For delta objects, we need to skip the base reference
if oh.Type == plumbing.OFSDeltaObject {
no, err := binary.ReadVariableWidthInt(r.scannerReader)
if err != nil {
return nil, err
}
if err := ValidateOFSDeltaBase(oh.Offset, no); err != nil {
return nil, err
}
oh.OffsetReference = oh.Offset - no
} else {
oh.Reference.ResetBySize(r.objectIDSize)
_, err := oh.Reference.ReadFrom(r.scannerReader)
if err != nil {
return nil, err
}
}
}
oh.ContentOffset = r.offset
zr, err := gogitsync.GetZlibReader(r.scannerReader)
if err != nil {
return nil, fmt.Errorf("zlib reset error: %w", err)
}
defer gogitsync.PutZlibReader(zr)
mw := io.Discard
if !oh.Type.IsDelta() {
r.hasher.Reset(oh.Type, oh.Size)
mw = r.hasher
if r.storage != nil {
w, err := r.storage.RawObjectWriter(oh.Type, oh.Size)
if err != nil {
return nil, err
}
defer func() { _ = w.Close() }()
mw = io.MultiWriter(r.hasher, w)
}
}
// If low memory mode isn't supported, and either the reader
// isn't seekable or this is a delta object, keep the contents
// of the objects in memory.
if !r.lowMemoryMode && (oh.Type.IsDelta() || r.seeker == nil) {
oh.content = gogitsync.GetBytesBuffer()
mw = io.MultiWriter(mw, oh.content)
}
// Bind the inflated stream by the size declared in the object header.
// A well-formed packfile never produces more inflated bytes than that
// value, so any overrun signals a malformed entry. For delta entries
// the declared size is the size of the delta instruction stream, not
// the resolved object.
mw = &boundedWriter{w: mw, limit: oh.Size}
_, err = ioutil.CopyBufferPool(mw, zr)
if err != nil {
return nil, err
}
if err := r.Flush(); err != nil {
return nil, err
}
oh.Crc32 = r.crc.Sum32()
if !oh.Type.IsDelta() {
oh.Hash = r.hasher.Sum()
}
r.packData.Section = ObjectSection
r.packData.objectHeader = oh
return nil, nil
}
// packFooter parses the packfile checksum.
// If the checksum cannot be parsed, or it does not match the checksum
// calculated during the scanning process, an [ErrMalformedPackfile] is
// returned.
func packFooter(r *Scanner) (stateFn, error) {
if err := r.Flush(); err != nil {
return nil, err
}
actual := r.packhash.Sum(nil)
var checksum plumbing.Hash
checksum.ResetBySize(r.objectIDSize)
_, err := checksum.ReadFrom(r.scannerReader)
if err != nil {
return nil, fmt.Errorf("read pack checksum: %w", err)
}
if checksum.Compare(actual) != 0 {
return nil, fmt.Errorf("%w: checksum mismatch: %q instead of %q",
ErrMalformedPackfile, hex.EncodeToString(actual), checksum)
}
r.packData.Section = FooterSection
r.packData.checksum = checksum
r.nextFn = nil
return nil, nil
}
package packfile
import (
"bufio"
"crypto"
"github.com/go-git/go-git/v6/plumbing"
format "github.com/go-git/go-git/v6/plumbing/format/config"
"github.com/go-git/go-git/v6/plumbing/hash"
)
// ScannerOption configures a Scanner.
type ScannerOption func(*Scanner)
// WithSHA256 enables the SHA256 hashing while scanning a pack file.
func WithSHA256() ScannerOption {
return func(s *Scanner) {
h := plumbing.NewHasher(format.SHA256, plumbing.AnyObject, 0)
s.objectIDSize = format.SHA256Size
s.hasher = h
s.packhash = hash.New(crypto.SHA256)
}
}
// WithBufioReader passes a bufio.Reader for scanner to use.
// It is used for reusing the buffer across multiple scanner instances.
func WithBufioReader(buf *bufio.Reader) ScannerOption {
return func(s *Scanner) {
s.rbuf = buf
}
}
package packfile
import (
"bufio"
"io"
)
// scannerReader has the following characteristics:
// - Provides an io.SeekReader impl for bufio.Reader, when the underlying
// reader supports it.
// - Keeps track of the current read position, for when the underlying reader
// isn't an io.SeekReader, but we still want to know the current offset.
// - Writes to the hash writer what it reads, with the aid of a smaller buffer.
// The buffer helps avoid a performance penalty for performing small writes
// to the crc32 hash writer.
//
// Note that this is passed on to zlib, and it mmust support io.BytesReader, else
// it won't be able to just read the content of the current object, but rather it
// will read the entire packfile.
//
// scannerReader is not thread-safe.
type scannerReader struct {
reader io.Reader
crc io.Writer
rbuf *bufio.Reader
wbuf *bufio.Writer
offset int64
seeker io.Seeker
}
func newScannerReader(r io.Reader, h io.Writer, rbuf *bufio.Reader) *scannerReader {
if rbuf == nil {
rbuf = bufio.NewReader(nil)
}
sr := &scannerReader{
rbuf: rbuf,
wbuf: bufio.NewWriterSize(nil, 64),
crc: h,
}
sr.Reset(r)
return sr
}
func (r *scannerReader) Reset(reader io.Reader) {
r.reader = reader
r.rbuf.Reset(r.reader)
r.wbuf.Reset(r.crc)
r.offset = 0
seeker, ok := r.reader.(io.ReadSeeker)
r.seeker = seeker
if ok {
r.offset, _ = seeker.Seek(0, io.SeekStart)
}
}
func (r *scannerReader) Read(p []byte) (n int, err error) {
n, err = r.rbuf.Read(p)
r.offset += int64(n)
if _, err := r.wbuf.Write(p[:n]); err != nil {
return n, err
}
return n, err
}
func (r *scannerReader) ReadByte() (b byte, err error) {
b, err = r.rbuf.ReadByte()
if err == nil {
r.offset++
return b, r.wbuf.WriteByte(b)
}
return b, err
}
func (r *scannerReader) Flush() error {
return r.wbuf.Flush()
}
// Seek seeks to a location. If the underlying reader is not an io.ReadSeeker,
// then only whence=io.SeekCurrent is supported, any other operation fails.
func (r *scannerReader) Seek(offset int64, whence int) (int64, error) {
var err error
if r.seeker == nil {
if whence != io.SeekCurrent || offset != 0 {
return -1, ErrSeekNotSupported
}
}
if whence == io.SeekCurrent && offset == 0 {
return r.offset, nil
}
r.offset, err = r.seeker.Seek(offset, whence)
r.rbuf.Reset(r.reader)
return r.offset, err
}
package packfile
import (
"bytes"
"github.com/go-git/go-git/v6/plumbing"
)
// Version represents the packfile version.
type Version uint32
// Packfile versions.
const (
V2 Version = 2
)
// Supported returns true if the version is supported.
func (v Version) Supported() bool {
switch v {
case V2:
return true
default:
return false
}
}
// ObjectHeader contains the information related to the object, this information
// is collected from the previous bytes to the content of the object.
type ObjectHeader struct {
Type plumbing.ObjectType
Offset int64
ContentOffset int64
Size int64
Reference plumbing.Hash
OffsetReference int64
Crc32 uint32
Hash plumbing.Hash
content *bytes.Buffer
parent *ObjectHeader
diskType plumbing.ObjectType
externalRef bool
// chainDepth caches the result of [checkDeltaChainDepth] for
// this header. A positive value is the number of delta links
// from this object down to (but not including) the first
// non-delta base. Zero means either "not yet computed" or
// "this header is not a delta"; both cases collapse to a
// constant-time re-check, so the dual meaning is harmless.
chainDepth int
}
// ID returns the object ID.
func (oh *ObjectHeader) ID() plumbing.Hash {
return oh.Hash
}
// SectionType represents the type of section in a packfile.
type SectionType int
// Section types.
const (
HeaderSection SectionType = iota
ObjectSection
FooterSection
)
// Header represents the packfile header.
type Header struct {
Version Version
ObjectsQty uint32
}
// PackData represents the data returned by the scanner.
type PackData struct {
Section SectionType
header Header
objectHeader ObjectHeader
checksum plumbing.Hash
}
// Value returns the value of the PackData based on its section type.
func (p PackData) Value() any {
switch p.Section {
case HeaderSection:
return p.header
case ObjectSection:
return p.objectHeader
case FooterSection:
return p.checksum
default:
return nil
}
}
// Package util provides low-level helpers for packfile encoding and decoding.
package util
import (
"errors"
"io"
"github.com/go-git/go-git/v6/plumbing"
)
const (
firstLengthBits = uint8(4) // the first byte into object header has 4 bits to store the length
maskPayload = 0x7f // 0111 1111
maskContinue = 0x80 // 1000 0000
maskType = uint8(112) // 0111 0000
)
// ErrLengthOverflow is returned when a variable-length integer would not fit
// into a uint64 because the input declares more continuation bytes than the
// type can hold.
var ErrLengthOverflow = errors.New("variable-length integer overflow")
// VariableLengthSize reads a variable length size from first, and uses reader
// to continue on reading until the full size is determined.
func VariableLengthSize(first byte, reader io.ByteReader) (uint64, error) {
// Extract the first part of the size (last 3 bits of the first byte).
size := uint64(first & 0x0F)
// | 001xxxx | xxxxxxxx | xxxxxxxx | ...
//
// ^^^ ^^^^^^^^ ^^^^^^^^
// Type Size Part 1 Size Part 2
//
// Check if more bytes are needed to fully determine the size.
if first&maskContinue != 0 {
shift := uint(4)
if reader == nil {
return 0, errors.New("reader is nil")
}
for {
// Mirrors unpack_object_header_buffer in canonical Git's
// packfile.c: a continuation byte at shift > 64-7 cannot
// contribute without overflowing a uint64.
if shift > 64-7 {
return 0, ErrLengthOverflow
}
b, err := reader.ReadByte()
if err != nil {
return 0, err
}
// Add the next 7 bits to the size.
size |= uint64(b&0x7F) << shift
// Check if the continuation bit is set.
if b&maskContinue == 0 {
break
}
// Prepare for the next byte.
shift += 7
}
}
return size, nil
}
// ObjectType returns the plumbing.ObjectType which is represented by b.
func ObjectType(b byte) plumbing.ObjectType {
return plumbing.ObjectType((b & maskType) >> firstLengthBits)
}
// EncodeLEB128 encodes num as an unsigned LEB128 byte sequence and
// returns it. Inverse of DecodeLEB128.
func EncodeLEB128(num uint) []byte {
var out []byte
for {
b := byte(num & maskPayload)
num >>= 7
if num == 0 {
return append(out, b)
}
out = append(out, b|maskContinue)
}
}
// EncodeLEB128ToWriter encodes num as an unsigned LEB128 byte sequence
// and writes it to writer. Inverse of DecodeLEB128FromReader.
func EncodeLEB128ToWriter(writer io.Writer, num uint) error {
_, err := writer.Write(EncodeLEB128(num))
return err
}
// DecodeLEB128 decodes a number encoded as an unsigned LEB128 at the
// start of some binary data and returns the decoded number, the rest
// of the bytes, and an error if the encoded value does not fit in a
// uint.
func DecodeLEB128(input []byte) (uint, []byte, error) {
if len(input) == 0 {
return 0, input, nil
}
var num, sz uint
var b byte
for {
// A continuation byte at shift > uintSize-7 cannot contribute
// without overflowing the accumulator.
if sz*7 > uintBits-7 {
return 0, input, ErrLengthOverflow
}
b = input[sz]
num |= (uint(b) & maskPayload) << (sz * 7) // concats 7 bits chunks
sz++
if uint(b)&maskContinue == 0 || sz == uint(len(input)) {
break
}
}
return num, input[sz:], nil
}
// DecodeLEB128FromReader decodes a number encoded as an unsigned LEB128
// from a byte reader and returns the decoded number.
func DecodeLEB128FromReader(input io.ByteReader) (uint, error) {
var num, sz uint
for {
if sz*7 > uintBits-7 {
return 0, ErrLengthOverflow
}
b, err := input.ReadByte()
if err != nil {
return 0, err
}
num |= (uint(b) & maskPayload) << (sz * 7) // concats 7 bits chunks
sz++
if uint(b)&maskContinue == 0 {
break
}
}
return num, nil
}
// uintBits is the bit width of uint on the current platform (32 or 64).
const uintBits = 32 << (^uint(0) >> 63)
package pktline
import (
"errors"
"io"
)
var (
// ErrInvalidErrorLine is returned by Decode when the packet line is not an
// error line.
ErrInvalidErrorLine = errors.New("expected an error-line")
// ErrNilWriter is returned when a nil writer is passed to a write function.
ErrNilWriter = errors.New("nil writer")
// ErrNilReader is returned when a nil reader is passed to a read function.
ErrNilReader = errors.New("nil reader")
// ErrNilError is returned when a nil error is passed to WriteError.
ErrNilError = errors.New("nil error")
errPrefix = []byte("ERR ")
)
const (
errPrefixSize = LenSize
)
// ErrorLine is a packet line that contains an error message.
// Once this packet is sent by client or server, the data transfer process is
// terminated.
// See https://git-scm.com/docs/pack-protocol#_pkt_line_format
type ErrorLine struct {
Text string
}
// Error implements the error interface.
func (e *ErrorLine) Error() string {
return e.Text
}
// Encode encodes the ErrorLine into a packet line.
func (e *ErrorLine) Encode(w io.Writer) error {
_, err := Writef(w, "%s%s\n", errPrefix, e.Text)
return err
}
// Decode decodes a packet line into an ErrorLine.
func (e *ErrorLine) Decode(r io.Reader) error {
_, _, err := ReadLine(r)
var el *ErrorLine
if !errors.As(err, &el) {
return ErrInvalidErrorLine
}
e.Text = el.Text
return nil
}
package pktline
import "fmt"
// ParseLength parses a four digit hexadecimal number from the given byte slice
// into its integer representation. If the byte slice contains non-hexadecimal,
// it will return an error.
func ParseLength(b []byte) (int, error) {
if b == nil {
return Err, fmt.Errorf("%w: missing pkt-line", ErrInvalidPktLen)
}
n, err := hexDecode(b)
if err != nil {
return Err, err
}
if n == 3 {
return Err, fmt.Errorf("%w: %04x", ErrInvalidPktLen, n)
}
// Limit the maximum size of a pkt-line to 65520 bytes.
// Fixes: b4177b89c08b (plumbing: format: pktline, Accept oversized pkt-lines up to 65524 bytes)
// See https://github.com/git/git/commit/7841c4801ce51f1f62d376d164372e8677c6bc94
if n > MaxSize {
return Err, fmt.Errorf("%w: %04x is too big", ErrInvalidPktLen, n)
}
return n, nil
}
// Turns the hexadecimal representation of a number in a byte slice into
// a number. This function substitute strconv.ParseUint(string(buf), 16,
// 16) and/or hex.Decode, to avoid generating new strings, thus helping the
// GC.
func hexDecode(buf []byte) (int, error) {
if len(buf) < 4 {
return 0, fmt.Errorf("%w: small pkt-line buffer", ErrInvalidPktLen)
}
var ret int
for i := range LenSize {
n, err := asciiHexToByte(buf[i])
if err != nil {
return 0, fmt.Errorf("%w: %w", ErrInvalidPktLen, err)
}
ret = 16*ret + int(n)
}
return ret, nil
}
// turns the hexadecimal ascii representation of a byte into its
// numerical value. Example: from 'b' to 11 (0xb).
func asciiHexToByte(b byte) (byte, error) {
switch {
case b >= '0' && b <= '9':
return b - '0', nil
case b >= 'a' && b <= 'f':
return b - 'a' + 10, nil
case b >= 'A' && b <= 'F':
return b - 'A' + 10, nil
default:
return 0, fmt.Errorf("not a hexadecimal byte %q", b)
}
}
// Returns the hexadecimal ascii representation of the 16 less
// significant bits of n. The length of the returned slice will always
// be 4. Example: if n is 1234 (0x4d2), the return value will be
// []byte{'0', '4', 'd', '2'}.
func asciiHex16(n int) []byte {
var ret [4]byte
ret[0] = byteToASCIIHex(byte(n & 0xf000 >> 12))
ret[1] = byteToASCIIHex(byte(n & 0x0f00 >> 8))
ret[2] = byteToASCIIHex(byte(n & 0x00f0 >> 4))
ret[3] = byteToASCIIHex(byte(n & 0x000f))
return ret[:]
}
// turns a byte into its hexadecimal ascii representation. Example:
// from 11 (0xb) to 'b'.
func byteToASCIIHex(n byte) byte {
if n < 10 {
return '0' + n
}
return 'a' - 10 + n
}
package pktline
import (
"bytes"
"errors"
"fmt"
"io"
"github.com/go-git/go-git/v6/utils/ioutil"
"github.com/go-git/go-git/v6/utils/trace"
)
// Write writes a pktline packet.
func Write(w io.Writer, p []byte) (n int, err error) {
if w == nil {
return 0, ErrNilWriter
}
defer func() {
if err == nil {
maskPackDataTrace(true, n, p)
}
}()
if len(p) == 0 {
return w.Write(emptyPkt)
}
if len(p) > MaxPayloadSize {
return 0, ErrPayloadTooLong
}
pktlen := len(p) + LenSize
n, err = w.Write(asciiHex16(pktlen))
if err != nil {
return n, err
}
n2, err := w.Write(p)
n += n2
return n, err
}
// Writef writes a pktline packet from a format string.
func Writef(w io.Writer, format string, a ...any) (n int, err error) {
if len(a) == 0 {
return Write(w, []byte(format))
}
return Write(w, fmt.Appendf(nil, format, a...))
}
// Writeln writes a pktline packet from a string and appends a newline.
func Writeln(w io.Writer, s string) (n int, err error) {
return Write(w, []byte(s+"\n"))
}
// WriteString writes a pktline packet from a string.
func WriteString(w io.Writer, s string) (n int, err error) {
return Write(w, []byte(s))
}
// WriteError writes an error packet.
func WriteError(w io.Writer, e error) (n int, err error) {
if w == nil {
return 0, ErrNilWriter
}
if e == nil {
return 0, ErrNilError
}
return Writef(w, "%s%s\n", errPrefix, e.Error())
}
// WriteFlush writes a flush packet.
// This always writes 4 bytes.
func WriteFlush(w io.Writer) (err error) {
if w == nil {
return ErrNilWriter
}
defer func() {
if err == nil {
trace.Packet.Printf("packet: > 0000")
}
}()
_, err = w.Write(flushPkt)
return err
}
// WriteDelim writes a delimiter packet.
// This always writes 4 bytes.
func WriteDelim(w io.Writer) (err error) {
if w == nil {
return ErrNilWriter
}
defer func() {
if err == nil {
trace.Packet.Printf("packet: > 0001")
}
}()
_, err = w.Write(delimPkt)
return err
}
// WriteResponseEnd writes a response-end packet.
// This always writes 4 bytes.
func WriteResponseEnd(w io.Writer) (err error) {
if w == nil {
return ErrNilWriter
}
defer func() {
if err == nil {
trace.Packet.Printf("packet: > 0002")
}
}()
_, err = w.Write(responseEndPkt)
return err
}
// Read reads a pktline packet payload into p and returns the packet full
// length.
//
// If p is less than 4 bytes, Read returns ErrInvalidPktLen. If p cannot hold
// the entire packet, Read discards the packet and returns io.ErrUnexpectedEOF;
// the stream is left positioned after the packet so subsequent pkt-line reads
// stay in sync.
// The error can be of type *ErrorLine if the packet is an error packet.
//
// Use packet length to determine the type of packet i.e. 0 is a flush packet,
// 1 is a delim packet, 2 is a response-end packet, and a length greater or
// equal to 4 is a data packet.
func Read(r io.Reader, p []byte) (l int, err error) {
if r == nil {
return Err, ErrNilReader
}
if len(p) < LenSize {
return Err, fmt.Errorf("%w: small pkt-line buffer", ErrInvalidPktLen)
}
_, err = io.ReadFull(r, p[:LenSize])
if err != nil {
if errors.Is(err, io.ErrUnexpectedEOF) {
return Err, fmt.Errorf("%w: short pkt-line %d", ErrInvalidPktLen, LenSize)
}
return Err, err
}
length, err := ParseLength(p)
if err != nil {
return Err, err
}
switch length {
case Flush, Delim, ResponseEnd:
trace.Packet.Printf("packet: < %04x", length)
return length, nil
case LenSize: // empty line
trace.Packet.Printf("packet: < %04x", length)
return length, nil
}
if length > len(p) {
// Drain the payload so subsequent pkt-line reads stay in sync.
_, _ = io.CopyN(io.Discard, r, int64(length-LenSize))
return Err, io.ErrUnexpectedEOF
}
_, err = io.ReadFull(r, p[LenSize:length])
if err != nil {
return Err, err
}
if bytes.HasPrefix(p[LenSize:length], errPrefix) {
err = &ErrorLine{
Text: string(bytes.TrimSpace(p[LenSize+errPrefixSize : length])),
}
}
maskPackDataTrace(false, length, p[LenSize:length])
return length, err
}
// ReadLine reads a packet line into a newly allocated buffer and
// returns the packet length and payload.
//
// Use packet length to determine the type of packet i.e. 0 is a flush packet,
// 1 is a delim packet, 2 is a response-end packet, and a length greater or
// equal to 4 is a data packet.
//
// The error can be of type *ErrorLine if the packet is an error packet.
func ReadLine(r io.Reader) (l int, p []byte, err error) {
buf := GetBuffer()
defer PutBuffer(buf)
l, err = Read(r, (*buf)[:])
if l < LenSize {
return l, nil, err
}
clone := bytes.Clone((*buf)[LenSize:l])
return l, clone, err
}
// PeekLine reads a packet line without consuming it.
//
// Use packet length to determine the type of packet i.e. 0 is a flush packet,
// 1 is a delim packet, 2 is a response-end packet, and a length greater or
// equal to 4 is a data packet.
//
// The error can be of type *ErrorLine if the packet is an error packet.
func PeekLine(r ioutil.ReadPeeker) (l int, p []byte, err error) {
if r == nil {
return Err, nil, ErrNilReader
}
n, err := r.Peek(LenSize)
if err != nil {
return Err, nil, err
}
length, err := ParseLength(n)
if err != nil {
return Err, nil, err
}
switch length {
case Flush, Delim, ResponseEnd:
trace.Packet.Printf("packet: < %04x", length)
return length, nil, nil
case LenSize: // empty line
trace.Packet.Printf("packet: < %04x", length)
return length, []byte{}, nil
}
data, err := r.Peek(length)
if err != nil {
return Err, nil, err
}
buf := data[LenSize:length]
if bytes.HasPrefix(buf, errPrefix) {
err = &ErrorLine{
Text: string(bytes.TrimSpace(buf[errPrefixSize:])),
}
}
maskPackDataTrace(false, length, buf)
return length, buf, err
}
func maskPackDataTrace(out bool, l int, data []byte) {
if !trace.Packet.Enabled() {
return
}
output := []byte("[ PACKDATA ]")
if l < 400 && len(data) > 0 && data[0] != 1 { // [sideband.PackData]
output = data
}
arrow := '<'
if out {
arrow = '>'
}
trace.Packet.Printf("packet: %c %04x %q", arrow, l, output)
}
package pktline
import (
"errors"
"io"
)
// Scanner provides a convenient interface for reading the payloads of a
// series of pkt-lines. It takes an io.Reader providing the source,
// which then can be tokenized through repeated calls to the Scan
// method.
//
// After each Scan call, the Bytes method returns the payload of the
// corresponding pkt-line as a slice into the Scanner's internal buffer.
// This buffer is overwritten on the next call to Scan, so callers must
// process or copy the data before the next Scan. For a string copy, use
// Text.
//
// Special pkt-lines ([Flush], [Delim], [ResponseEnd]) return a nil slice
// from Bytes; Len returns the pkt-line length, which equals the
// corresponding constant (Flush=0, Delim=1, ResponseEnd=2).
//
// Scanning stops at EOF or the first I/O error.
type Scanner struct {
r io.Reader // The reader provided by the client
err error // Sticky error
buf [MaxSize]byte // Buffer used to read the pktlines
n int // Number of bytes read in the last read
}
// NewScanner returns a new Scanner to read from r.
func NewScanner(r io.Reader) *Scanner {
return &Scanner{
r: r,
}
}
// Err returns the first error encountered by the Scanner.
func (s *Scanner) Err() error {
return s.err
}
// Scan advances the Scanner to the next pkt-line, whose payload will
// then be available through the Bytes method. Scanning stops at EOF
// or the first I/O error. After Scan returns false, the Err method
// will return any error that occurred during scanning, except that if
// it was io.EOF, Err will return nil.
func (s *Scanner) Scan() bool {
if s.r == nil {
return false
}
s.n, s.err = Read(s.r, s.buf[:])
if errors.Is(s.err, io.EOF) {
s.err = nil
return false
}
return s.err == nil
}
// Bytes returns the payload of the most recent pkt-line as a slice
// into the Scanner's internal buffer. The slice is valid only until
// the next call to Scan, which overwrites the buffer. Use [Text] or
// copy the data when the payload must outlive the next Scan.
//
// Bytes does no allocation. It returns nil for special pkt-lines
// ([Flush], [Delim], [ResponseEnd]); use [Len] to distinguish them.
func (s *Scanner) Bytes() []byte {
if s.n >= LenSize {
return s.buf[LenSize:s.n]
}
return nil
}
// Text returns the most recent packet generated by a call to Scan.
func (s *Scanner) Text() string {
return string(s.Bytes())
}
// Len returns the pkt-line length of the most recent pkt-line. For data
// lines this is the length of the entire pkt-line including the 4-byte
// length prefix. For special pkt-lines, Len returns the corresponding
// constant: [Flush] (0), [Delim] (1), or [ResponseEnd] (2).
func (s *Scanner) Len() int {
return s.n
}
package pktline
import "sync"
var pktBuffer = sync.Pool{
New: func() any {
var b [MaxSize]byte
return &b
},
}
// GetBuffer returns a *[MaxSize]byte that is managed by a sync.Pool. The
// initial slice length will be 65520 (65kb).
//
// After use, the *[MaxSize]byte should be put back into the sync.Pool by
// calling PutBuffer.
func GetBuffer() *[MaxSize]byte {
buf := pktBuffer.Get().(*[MaxSize]byte)
return buf
}
// PutBuffer puts buf back into its sync.Pool.
func PutBuffer(buf *[MaxSize]byte) {
if buf == nil {
return
}
pktBuffer.Put(buf)
}
package reflog
import (
"bufio"
"bytes"
"fmt"
"io"
"strconv"
"strings"
"time"
"github.com/go-git/go-git/v6/plumbing"
)
// Signature represents an author or committer identity with a timestamp.
// This mirrors object.Signature but is defined here to avoid an import cycle
// (reflog -> object -> storer -> reflog).
type Signature struct {
// Name represents a person name.
Name string
// Email is an email address.
Email string
// When is the timestamp of the signature.
When time.Time
}
// Entry represents a single reflog entry.
type Entry struct {
// OldHash is the hash the reference pointed to before the change.
OldHash plumbing.Hash
// NewHash is the hash the reference points to after the change.
NewHash plumbing.Hash
// Committer holds the signature for the entry, including name, email and when it was created.
Committer Signature
// Message describes the action that caused the change (e.g. "commit: Add feature").
Message string
}
// Decoder reads reflog entries from a reader one at a time.
type Decoder struct {
r *bufio.Reader
}
// NewDecoder creates a Decoder that reads reflog entries from r.
func NewDecoder(r io.Reader) *Decoder {
return &Decoder{r: bufio.NewReader(r)}
}
// Next returns the next reflog entry. It returns io.EOF when there are no more entries.
func (d *Decoder) Next() (*Entry, error) {
for {
line, err := d.r.ReadBytes('\n')
if err != nil && err != io.EOF {
return nil, err
}
line = bytes.TrimSuffix(line, []byte{'\n'})
if len(line) != 0 {
return decodeLine(line)
}
if err == io.EOF {
return nil, io.EOF
}
}
}
// Decode reads all reflog entries from the reader.
// Entries are returned in file order (oldest first).
func Decode(r io.Reader) ([]*Entry, error) {
if r == nil {
return nil, fmt.Errorf("reader is nil")
}
d := NewDecoder(r)
var entries []*Entry
for {
e, err := d.Next()
if err == io.EOF {
return entries, nil
}
if err != nil {
return entries, err
}
entries = append(entries, e)
}
}
// decodeLine parses a single reflog line.
// Format: <old-hash> <new-hash> <name> <<email>> <unix-timestamp> <timezone>\t<message>
func decodeLine(line []byte) (*Entry, error) {
e := &Entry{}
// Parse old hash (up to first space)
spaceIdx := bytes.IndexByte(line, ' ')
if spaceIdx == -1 {
return nil, fmt.Errorf("reflog entry too short")
}
oldHashStr := string(line[:spaceIdx])
if !plumbing.IsHash(oldHashStr) {
return nil, fmt.Errorf("invalid old hash in reflog entry: %q", oldHashStr)
}
e.OldHash = plumbing.NewHash(oldHashStr)
line = line[spaceIdx+1:]
// Parse new hash (up to next space)
spaceIdx = bytes.IndexByte(line, ' ')
if spaceIdx == -1 {
return nil, fmt.Errorf("expected space after new hash")
}
newHashStr := string(line[:spaceIdx])
if !plumbing.IsHash(newHashStr) {
return nil, fmt.Errorf("invalid new hash in reflog entry: %q", newHashStr)
}
e.NewHash = plumbing.NewHash(newHashStr)
line = line[spaceIdx+1:]
// Split on tab to separate signature from message
sigBytes := line
before, after, ok := bytes.Cut(line, []byte{'\t'})
if ok {
sigBytes = before
e.Message = string(after)
}
// Parse signature: Name <email> timestamp timezone
open := bytes.LastIndexByte(sigBytes, '<')
closeBracket := bytes.LastIndexByte(sigBytes, '>')
if open == -1 || closeBracket == -1 || closeBracket < open {
return nil, fmt.Errorf("invalid signature in reflog entry")
}
e.Committer.Name = string(bytes.TrimSpace(sigBytes[:open]))
e.Committer.Email = string(sigBytes[open+1 : closeBracket])
// Parse timestamp and timezone after '> '
if closeBracket+2 >= len(sigBytes) {
return nil, fmt.Errorf("missing timestamp in reflog entry")
}
var err error
e.Committer.When, err = decodeTimestamp(sigBytes[closeBracket+2:])
if err != nil {
return nil, err
}
return e, nil
}
func decodeTimestamp(s []byte) (time.Time, error) {
// Format: "1234567890 +0000"
parts := bytes.Fields(s)
if len(parts) != 2 {
return time.Time{}, fmt.Errorf("invalid timestamp in reflog entry: %q", s)
}
secs, err := strconv.ParseInt(string(parts[0]), 10, 64)
if err != nil {
return time.Time{}, fmt.Errorf("invalid timestamp seconds in reflog entry: %w", err)
}
t := time.Unix(secs, 0)
tz := string(parts[1])
if len(tz) != 5 || (tz[0] != '+' && tz[0] != '-') {
return time.Time{}, fmt.Errorf("invalid timezone in reflog entry: %q", tz)
}
h, err := strconv.Atoi(tz[1:3])
if err != nil {
return time.Time{}, fmt.Errorf("invalid timezone hours in reflog entry: %w", err)
}
m, err := strconv.Atoi(tz[3:5])
if err != nil {
return time.Time{}, fmt.Errorf("invalid timezone minutes in reflog entry: %w", err)
}
offset := h*3600 + m*60
if tz[0] == '-' {
offset = -offset
}
t = t.In(time.FixedZone("", offset))
return t, nil
}
// normalizeMessage normalizes a reflog message the same way Git does:
// collapse consecutive whitespace to a single space, strip leading/trailing
// whitespace, and remove newlines.
// See copy_reflog_msg in refs.c:
// https://github.com/git/git/blob/7ff1e8dc1e1680510c96e69965b3fa81372c5037/refs.c#L1026-L1049
func normalizeMessage(msg string) string {
msg = strings.ReplaceAll(msg, "\n", " ")
msg = strings.ReplaceAll(msg, "\r", " ")
fields := strings.Fields(msg)
return strings.Join(fields, " ")
}
// Encode writes a single reflog entry to the writer.
func Encode(w io.Writer, e *Entry) error {
if w == nil {
return fmt.Errorf("writer is nil")
}
if e == nil {
return fmt.Errorf("entry is nil")
}
_, offset := e.Committer.When.Zone()
sign := '+'
if offset < 0 {
sign = '-'
offset = -offset
}
hours := offset / 3600
minutes := (offset % 3600) / 60
msg := normalizeMessage(e.Message)
if msg != "" {
_, err := fmt.Fprintf(w, "%s %s %s <%s> %d %c%02d%02d\t%s\n",
e.OldHash, e.NewHash,
e.Committer.Name, e.Committer.Email,
e.Committer.When.Unix(), sign, hours, minutes,
msg,
)
return err
}
_, err := fmt.Fprintf(w, "%s %s %s <%s> %d %c%02d%02d\n",
e.OldHash, e.NewHash,
e.Committer.Name, e.Committer.Email,
e.Committer.When.Unix(), sign, hours, minutes,
)
return err
}
package revfile
import (
"bufio"
"bytes"
"crypto"
"encoding/hex"
"errors"
"fmt"
"hash"
"io"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/utils/binary"
)
var (
// ErrUnsupportedVersion is returned by Decode when the rev file version
// is not supported.
ErrUnsupportedVersion = errors.New("unsupported version")
// ErrMalformedRevFile is returned by Decode when the rev file is corrupted.
ErrMalformedRevFile = errors.New("malformed rev file")
// ErrUnsupportedHashFunction is returned by Decode when the rev file defines an
// unsupported hash function.
ErrUnsupportedHashFunction = errors.New("unsupported hash function")
// ErrEmptyReverseIndex is returned by Decode when the rev file is empty.
ErrEmptyReverseIndex = errors.New("reverse index is empty")
revHeader = []byte{'R', 'I', 'D', 'X'}
)
// Revfile constants.
const (
VersionSupported = 1
sha1Hash uint32 = 1
sha256Hash uint32 = 2
)
// decoder is the internal state for decoding a rev file.
// It is not exported to prevent reuse - each Decode call creates fresh state.
type decoder struct {
reader io.Reader
hasher crypto.Hash
hash hash.Hash
version uint32
objCount int64
packChecksum plumbing.ObjectID
out chan<- uint32
}
// stateFn defines each individual state within the state machine that
// represents a revfile.
type stateFn func(*decoder) (stateFn, error)
// Decode reads a rev file and sends index positions to out.
// The caller must not close out; Decode closes it when done.
// This function is safe to call concurrently with different parameters.
func Decode(r io.Reader, objCount int64, packChecksum plumbing.ObjectID, out chan<- uint32) error {
if r == nil {
return fmt.Errorf("%w: nil reader", ErrMalformedRevFile)
}
if out == nil {
return errors.New("nil channel")
}
br, ok := r.(*bufio.Reader)
if !ok {
br = bufio.NewReader(r)
}
d := &decoder{
reader: br,
objCount: objCount,
packChecksum: packChecksum,
out: out,
}
defer close(d.out)
for state := readMagicNumber; state != nil; {
var err error
state, err = state(d)
if err != nil {
return err
}
}
return nil
}
func readMagicNumber(d *decoder) (stateFn, error) {
h := make([]byte, 4)
if _, err := io.ReadFull(d.reader, h); err != nil {
return nil, err
}
if !bytes.Equal(h, revHeader) {
return nil, ErrMalformedRevFile
}
return readVersion, nil
}
func readVersion(d *decoder) (stateFn, error) {
v, err := binary.ReadUint32(d.reader)
if err != nil {
return nil, err
}
if v != VersionSupported {
return nil, ErrUnsupportedVersion
}
d.version = v
return readHashFunction, nil
}
func readHashFunction(d *decoder) (stateFn, error) {
hf, err := binary.ReadUint32(d.reader)
if err != nil {
return nil, err
}
switch hf {
case sha1Hash:
d.hasher = crypto.SHA1
case sha256Hash:
d.hasher = crypto.SHA256
default:
return nil, ErrUnsupportedHashFunction
}
if !d.hasher.Available() {
return nil, fmt.Errorf("%w: %v not registered", ErrUnsupportedHashFunction, d.hasher)
}
d.hash = d.hasher.New()
err = binary.Write(d.hash, revHeader, d.version, hf)
if err != nil {
return nil, fmt.Errorf("failed to hash rev header: %w", err)
}
return readEntries, nil
}
func readEntries(d *decoder) (stateFn, error) {
if d.objCount == 0 {
return nil, ErrEmptyReverseIndex
}
var i int64
for i = 0; i < d.objCount; i++ {
idx, err := binary.ReadUint32(d.reader)
if err == io.EOF {
return nil, fmt.Errorf("%w: unexpected EOF at object %d", ErrMalformedRevFile, i)
}
if err != nil {
return nil, err
}
d.out <- idx
err = binary.Write(d.hash, idx)
if err != nil {
return nil, fmt.Errorf("failed to hash entry: %w", err)
}
}
return readPackChecksum, nil
}
func readPackChecksum(d *decoder) (stateFn, error) {
var pack plumbing.Hash
pack.ResetBySize(d.hasher.Size())
n, err := pack.ReadFrom(d.reader)
if err != nil {
return nil, err
}
if n != int64(d.hasher.Size()) {
return nil, fmt.Errorf("%w: wrong checksum size", ErrMalformedRevFile)
}
if pack.Compare(d.packChecksum.Bytes()) != 0 {
return nil, fmt.Errorf("%w: packfile hash mismatch wanted %q got %q",
ErrMalformedRevFile, d.packChecksum.String(), pack.String())
}
err = binary.Write(d.hash, pack.Bytes())
if err != nil {
return nil, fmt.Errorf("failed to hash pack checksum: %w", err)
}
return readRevChecksum, nil
}
func readRevChecksum(d *decoder) (stateFn, error) {
var rev plumbing.Hash
rev.ResetBySize(d.hasher.Size())
n, err := rev.ReadFrom(d.reader)
if err != nil {
return nil, err
}
if n != int64(d.hasher.Size()) {
return nil, fmt.Errorf("%w: wrong checksum size", ErrMalformedRevFile)
}
rh := d.hash.Sum(nil)
if rev.Compare(rh) != 0 {
return nil, fmt.Errorf("%w: rev file checksum mismatch wanted %q got %q",
ErrMalformedRevFile, hex.EncodeToString(rh), rev.String())
}
// Check for unexpected trailing data
var buf [1]byte
extra, err := d.reader.Read(buf[:])
if extra > 0 {
return nil, fmt.Errorf("%w: expected EOF", ErrMalformedRevFile)
}
if err != io.EOF {
return nil, err
}
return nil, nil
}
package revfile
import (
"crypto"
"fmt"
"hash"
"io"
"reflect"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/format/idxfile"
"github.com/go-git/go-git/v6/utils/binary"
)
// encoder is the internal state for encoding a rev file.
// It is not exported to prevent reuse - each Encode call creates fresh state.
type encoder struct {
writer io.Writer
hash hash.Hash
entries []uint32
packChecksum plumbing.Hash
}
// stateFnEncode defines each individual state within the state machine that
// represents encoding a revfile.
type stateFnEncode func(*encoder) (stateFnEncode, error)
// Encode encodes a reverse index from a MemoryIndex to the writer.
// The reverse index maps pack offsets (sorted order) to index positions.
// This function is safe to call concurrently with different parameters.
func Encode(w io.Writer, h hash.Hash, idx *idxfile.MemoryIndex) error {
if w == nil {
return fmt.Errorf("nil writer")
}
v := reflect.ValueOf(w)
switch v.Kind() {
case reflect.Pointer, reflect.Interface:
if v.IsNil() {
return fmt.Errorf("nil writer")
}
}
if idx == nil {
return fmt.Errorf("nil index")
}
h.Reset()
e := &encoder{
writer: w,
hash: h,
}
if err := e.buildReverseIndex(idx); err != nil {
return err
}
for state := writeHeader; state != nil; {
var err error
state, err = state(e)
if err != nil {
return err
}
}
return nil
}
// buildReverseIndex creates the reverse index mapping from the MemoryIndex.
// It maps from pack offset order to index position (sorted by hash).
func (e *encoder) buildReverseIndex(idx *idxfile.MemoryIndex) error {
count, err := idx.Count()
if err != nil {
return err
}
offsetToPos := make(map[uint64]uint32, count)
entries, err := idx.Entries()
if err != nil {
return err
}
defer func() { _ = entries.Close() }()
var pos uint32
for {
entry, err := entries.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
offsetToPos[entry.Offset] = pos
pos++
}
entriesByOffset, err := idx.EntriesByOffset()
if err != nil {
return err
}
defer func() { _ = entriesByOffset.Close() }()
// Build the reverse index array
e.entries = make([]uint32, 0, count)
for {
entry, err := entriesByOffset.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
e.entries = append(e.entries, offsetToPos[entry.Offset])
}
e.packChecksum = idx.PackfileChecksum
return nil
}
func writeHeader(e *encoder) (stateFnEncode, error) {
_, err := e.writer.Write(revHeader)
if err != nil {
return nil, err
}
_, err = e.hash.Write(revHeader)
if err != nil {
return nil, fmt.Errorf("failed to hash header: %w", err)
}
return writeVersion, nil
}
func writeVersion(e *encoder) (stateFnEncode, error) {
err := binary.WriteUint32(io.MultiWriter(e.hash, e.writer), uint32(VersionSupported))
if err != nil {
return nil, fmt.Errorf("failed to hash version: %w", err)
}
return writeHashFunction, nil
}
func writeHashFunction(e *encoder) (stateFnEncode, error) {
hf := sha1Hash
if e.hash.Size() == crypto.SHA256.Size() {
hf = sha256Hash
}
err := binary.WriteUint32(e.writer, uint32(hf))
if err != nil {
return nil, err
}
err = binary.Write(e.hash, hf)
if err != nil {
return nil, fmt.Errorf("failed to hash function %d: %w", hf, err)
}
return writeEntries, nil
}
func writeEntries(e *encoder) (stateFnEncode, error) {
for _, entry := range e.entries {
err := binary.WriteUint32(e.writer, entry)
if err != nil {
return nil, err
}
err = binary.Write(e.hash, entry)
if err != nil {
return nil, fmt.Errorf("failed to hash entry: %w", err)
}
}
return writePackChecksum, nil
}
func writePackChecksum(e *encoder) (stateFnEncode, error) {
_, err := e.writer.Write(e.packChecksum.Bytes())
if err != nil {
return nil, err
}
_, err = e.hash.Write(e.packChecksum.Bytes())
if err != nil {
return nil, fmt.Errorf("failed to hash pack checksum: %w", err)
}
return writeRevChecksum, nil
}
func writeRevChecksum(e *encoder) (stateFnEncode, error) {
checksum := e.hash.Sum(nil)
_, err := e.writer.Write(checksum)
return nil, err
}
package object
import (
"io"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/storer"
"github.com/go-git/go-git/v6/utils/ioutil"
)
// Blob is used to store arbitrary data - it is generally a file.
type Blob struct {
// Hash of the blob.
Hash plumbing.Hash
// Size of the (uncompressed) blob.
Size int64
obj plumbing.EncodedObject
}
// GetBlob gets a blob from an object storer and decodes it.
func GetBlob(s storer.EncodedObjectStorer, h plumbing.Hash) (*Blob, error) {
o, err := s.EncodedObject(plumbing.BlobObject, h)
if err != nil {
return nil, err
}
return DecodeBlob(o)
}
// DecodeBlob decodes an encoded object into a *Blob.
func DecodeBlob(o plumbing.EncodedObject) (*Blob, error) {
b := &Blob{}
if err := b.Decode(o); err != nil {
return nil, err
}
return b, nil
}
// ID returns the object ID of the blob. The returned value will always match
// the current value of Blob.Hash.
//
// ID is present to fulfill the Object interface.
func (b *Blob) ID() plumbing.Hash {
return b.Hash
}
// Type returns the type of object. It always returns plumbing.BlobObject.
//
// Type is present to fulfill the Object interface.
func (b *Blob) Type() plumbing.ObjectType {
return plumbing.BlobObject
}
// Decode transforms a plumbing.EncodedObject into a Blob struct.
func (b *Blob) Decode(o plumbing.EncodedObject) error {
if o.Type() != plumbing.BlobObject {
return ErrUnsupportedObject
}
b.Hash = o.Hash()
b.Size = o.Size()
b.obj = o
return nil
}
// Encode transforms a Blob into a plumbing.EncodedObject.
func (b *Blob) Encode(o plumbing.EncodedObject) (err error) {
o.SetType(plumbing.BlobObject)
w, err := o.Writer()
if err != nil {
return err
}
defer ioutil.CheckClose(w, &err)
r, err := b.Reader()
if err != nil {
return err
}
defer ioutil.CheckClose(r, &err)
_, err = ioutil.CopyBufferPool(w, r)
return err
}
// Reader returns a reader allow the access to the content of the blob
func (b *Blob) Reader() (io.ReadCloser, error) {
return b.obj.Reader()
}
// BlobIter provides an iterator for a set of blobs.
type BlobIter struct {
storer.EncodedObjectIter
s storer.EncodedObjectStorer
}
// NewBlobIter takes a storer.EncodedObjectStorer and a
// storer.EncodedObjectIter and returns a *BlobIter that iterates over all
// blobs contained in the storer.EncodedObjectIter.
//
// Any non-blob object returned by the storer.EncodedObjectIter is skipped.
func NewBlobIter(s storer.EncodedObjectStorer, iter storer.EncodedObjectIter) *BlobIter {
return &BlobIter{iter, s}
}
// Next moves the iterator to the next blob and returns a pointer to it. If
// there are no more blobs, it returns io.EOF.
func (iter *BlobIter) Next() (*Blob, error) {
for {
obj, err := iter.EncodedObjectIter.Next()
if err != nil {
return nil, err
}
if obj.Type() != plumbing.BlobObject {
continue
}
return DecodeBlob(obj)
}
}
// ForEach call the cb function for each blob contained on this iter until
// an error happens or the end of the iter is reached. If ErrStop is sent
// the iteration is stop but no error is returned. The iterator is closed.
func (iter *BlobIter) ForEach(cb func(*Blob) error) error {
return iter.EncodedObjectIter.ForEach(func(obj plumbing.EncodedObject) error {
if obj.Type() != plumbing.BlobObject {
return nil
}
b, err := DecodeBlob(obj)
if err != nil {
return err
}
return cb(b)
})
}
package object
import (
"bytes"
"context"
"fmt"
"strings"
"github.com/go-git/go-git/v6/utils/merkletrie"
)
// Change values represent a detected change between two git trees. For
// modifications, From is the original status of the node and To is its
// final status. For insertions, From is the zero value and for
// deletions To is the zero value.
type Change struct {
From ChangeEntry
To ChangeEntry
}
var empty ChangeEntry
// Action returns the kind of action represented by the change, an
// insertion, a deletion or a modification.
func (c *Change) Action() (merkletrie.Action, error) {
if c.From == empty && c.To == empty {
return merkletrie.Action(0),
fmt.Errorf("malformed change: empty from and to")
}
if c.From == empty {
return merkletrie.Insert, nil
}
if c.To == empty {
return merkletrie.Delete, nil
}
return merkletrie.Modify, nil
}
// Files returns the files before and after a change.
// For insertions from will be nil. For deletions to will be nil.
func (c *Change) Files() (from, to *File, err error) {
action, err := c.Action()
if err != nil {
return from, to, err
}
if action == merkletrie.Insert || action == merkletrie.Modify {
to, err = c.To.Tree.TreeEntryFile(&c.To.TreeEntry)
if !c.To.TreeEntry.Mode.IsFile() {
return nil, nil, nil
}
if err != nil {
return from, to, err
}
}
if action == merkletrie.Delete || action == merkletrie.Modify {
from, err = c.From.Tree.TreeEntryFile(&c.From.TreeEntry)
if !c.From.TreeEntry.Mode.IsFile() {
return nil, nil, nil
}
if err != nil {
return from, to, err
}
}
return from, to, err
}
func (c *Change) String() string {
action, err := c.Action()
if err != nil {
return "malformed change"
}
return fmt.Sprintf("<Action: %s, Path: %s>", action, c.name())
}
// Patch returns a Patch with all the file changes in chunks. This
// representation can be used to create several diff outputs.
func (c *Change) Patch() (*Patch, error) {
return c.PatchContext(context.Background())
}
// PatchContext returns a Patch with all the file changes in chunks. This
// representation can be used to create several diff outputs.
// If context expires, an non-nil error will be returned.
// Provided context must be non-nil.
func (c *Change) PatchContext(ctx context.Context) (*Patch, error) {
return getPatchContext(ctx, "", c)
}
func (c *Change) name() string {
if c.From != empty {
return c.From.Name
}
return c.To.Name
}
// ChangeEntry values represent a node that has suffered a change.
type ChangeEntry struct {
// Full path of the node using "/" as separator.
Name string
// Parent tree of the node that has changed.
Tree *Tree
// The entry of the node.
TreeEntry TreeEntry
}
// Changes represents a collection of changes between two git trees.
// Implements sort.Interface lexicographically over the path of the
// changed files.
type Changes []*Change
func (c Changes) Len() int {
return len(c)
}
func (c Changes) Swap(i, j int) {
c[i], c[j] = c[j], c[i]
}
func (c Changes) Less(i, j int) bool {
return strings.Compare(c[i].name(), c[j].name()) < 0
}
func (c Changes) String() string {
var buffer bytes.Buffer
buffer.WriteString("[")
comma := ""
for _, v := range c {
buffer.WriteString(comma)
buffer.WriteString(v.String())
comma = ", "
}
buffer.WriteString("]")
return buffer.String()
}
// Patch returns a Patch with all the changes in chunks. This
// representation can be used to create several diff outputs.
func (c Changes) Patch() (*Patch, error) {
return c.PatchContext(context.Background())
}
// PatchContext returns a Patch with all the changes in chunks. This
// representation can be used to create several diff outputs.
// If context expires, an non-nil error will be returned.
// Provided context must be non-nil.
func (c Changes) PatchContext(ctx context.Context) (*Patch, error) {
return getPatchContext(ctx, "", c...)
}
package object
import (
"errors"
"fmt"
"github.com/go-git/go-git/v6/utils/merkletrie"
"github.com/go-git/go-git/v6/utils/merkletrie/noder"
)
// The following functions transform changes types form the merkletrie
// package to changes types from this package.
func newChange(c merkletrie.Change) (*Change, error) {
ret := &Change{}
var err error
if ret.From, err = newChangeEntry(c.From); err != nil {
return nil, fmt.Errorf("from field: %s", err)
}
if ret.To, err = newChangeEntry(c.To); err != nil {
return nil, fmt.Errorf("to field: %s", err)
}
return ret, nil
}
func newChangeEntry(p noder.Path) (ChangeEntry, error) {
if p == nil {
return empty, nil
}
asTreeNoder, ok := p.Last().(*treeNoder)
if !ok {
return ChangeEntry{}, errors.New("cannot transform non-TreeNoders")
}
return ChangeEntry{
Name: p.String(),
Tree: asTreeNoder.parent,
TreeEntry: TreeEntry{
Name: asTreeNoder.name,
Mode: asTreeNoder.mode,
Hash: asTreeNoder.hash,
},
}, nil
}
func newChanges(src merkletrie.Changes) (Changes, error) {
ret := make(Changes, len(src))
var err error
for i, e := range src {
ret[i], err = newChange(e)
if err != nil {
return nil, fmt.Errorf("change #%d: %s", i, err)
}
}
return ret, nil
}
package object
import (
"bytes"
"context"
"errors"
"fmt"
"slices"
"strings"
"github.com/ProtonMail/go-crypto/openpgp"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/storer"
"github.com/go-git/go-git/v6/utils/ioutil"
"github.com/go-git/go-git/v6/utils/sync"
)
const (
beginpgp string = "-----BEGIN PGP SIGNATURE-----"
endpgp string = "-----END PGP SIGNATURE-----"
headerpgp string = "gpgsig"
headerpgp256 string = "gpgsig-sha256"
headerencoding string = "encoding"
defaultUtf8CommitMessageEncoding MessageEncoding = "UTF-8"
)
// Hash represents the hash of an object
type Hash plumbing.Hash
// MessageEncoding represents the encoding of a commit
type MessageEncoding string
// Commit points to a single tree, marking it as what the project looked like
// at a certain point in time. It contains meta-information about that point
// in time, such as a timestamp, the author of the changes since the last
// commit, a pointer to the previous commit(s), etc.
// http://shafiulazam.com/gitbook/1_the_git_object_model.html
//
// When a Commit is populated by Decode it retains a reference to the source
// plumbing.EncodedObject so that EncodeWithoutSignature can reproduce the
// exact bytes the signature was computed over. Refer to EncodeWithoutSignature
// for more information.
type Commit struct {
// Hash of the commit object.
Hash plumbing.Hash
// Author is the original author of the commit.
Author Signature
// Committer is the one performing the commit, might be different from
// Author.
Committer Signature
// Signature is the cryptographic signature of the commit (e.g. SSH, X.509).
Signature string
// SignatureSHA256 is the SHA-256 cryptographic signature of the commit,
// stored under the "gpgsig-sha256" header. It may be present alongside
// Signature on commits produced in hash-algorithm compatibility mode.
SignatureSHA256 string
// Message is the commit message, contains arbitrary text.
Message string
// TreeHash is the hash of the root tree of the commit.
TreeHash plumbing.Hash
// ParentHashes are the hashes of the parent commits of the commit.
ParentHashes []plumbing.Hash
// Encoding is the encoding of the commit.
Encoding MessageEncoding
// List of extra headers of the commit
ExtraHeaders []ExtraHeader
s storer.EncodedObjectStorer
// src holds the encoded object this Commit was decoded from, used by
// EncodeWithoutSignature to recover the canonical signed bytes.
src plumbing.EncodedObject
}
// ExtraHeader holds any non-standard header
type ExtraHeader struct {
// Header name
Key string
// Value of the header
Value string
}
// Format implements fmt.Formatter for ExtraHeader.
func (h ExtraHeader) Format(f fmt.State, verb rune) {
switch verb {
case 'v':
_, _ = fmt.Fprintf(f, "ExtraHeader{Key: %v, Value: %v}", h.Key, h.Value)
default:
_, _ = fmt.Fprintf(f, "%s", h.Key)
if len(h.Value) > 0 {
_, _ = fmt.Fprint(f, " ")
// Content may be spread on multiple lines, if so we need to
// prepend each of them with a space for "continuation".
value := strings.TrimSuffix(h.Value, "\n")
lines := strings.Split(value, "\n")
_, _ = fmt.Fprint(f, strings.Join(lines, "\n "))
}
}
}
// Parse an extra header and indicate whether it may be continue on the next line
func parseExtraHeader(line []byte) (ExtraHeader, bool) {
split := bytes.SplitN(line, []byte{' '}, 2)
out := ExtraHeader{
Key: string(bytes.TrimRight(split[0], "\n")),
Value: "",
}
if len(split) == 2 {
out.Value += string(split[1])
return out, true
}
return out, false
}
// GetCommit gets a commit from an object storer and decodes it.
func GetCommit(s storer.EncodedObjectStorer, h plumbing.Hash) (*Commit, error) {
o, err := s.EncodedObject(plumbing.CommitObject, h)
if err != nil {
return nil, err
}
return DecodeCommit(s, o)
}
// DecodeCommit decodes an encoded object into a *Commit and associates it to
// the given object storer.
func DecodeCommit(s storer.EncodedObjectStorer, o plumbing.EncodedObject) (*Commit, error) {
c := &Commit{s: s}
if err := c.Decode(o); err != nil {
return nil, err
}
return c, nil
}
// Tree returns the Tree from the commit.
func (c *Commit) Tree() (*Tree, error) {
return GetTree(c.s, c.TreeHash)
}
// PatchContext returns the Patch between the actual commit and the provided one.
// Error will be return if context expires. Provided context must be non-nil.
//
// NOTE: Since version 5.1.0 the renames are correctly handled, the settings
// used are the recommended options DefaultDiffTreeOptions.
func (c *Commit) PatchContext(ctx context.Context, to *Commit) (*Patch, error) {
fromTree, err := c.Tree()
if err != nil {
return nil, err
}
var toTree *Tree
if to != nil {
toTree, err = to.Tree()
if err != nil {
return nil, err
}
}
return fromTree.PatchContext(ctx, toTree)
}
// Patch returns the Patch between the actual commit and the provided one.
//
// NOTE: Since version 5.1.0 the renames are correctly handled, the settings
// used are the recommended options DefaultDiffTreeOptions.
func (c *Commit) Patch(to *Commit) (*Patch, error) {
return c.PatchContext(context.Background(), to)
}
// Parents return a CommitIter to the parent Commits.
func (c *Commit) Parents() CommitIter {
return NewCommitIter(c.s,
storer.NewEncodedObjectLookupIter(c.s, plumbing.CommitObject, c.ParentHashes),
)
}
// NumParents returns the number of parents in a commit.
func (c *Commit) NumParents() int {
return len(c.ParentHashes)
}
// ErrParentNotFound is returned when the parent commit is not found.
var ErrParentNotFound = errors.New("commit parent not found")
// ErrMalformedCommit is returned when a commit object cannot be decoded
// because its standard headers (tree, parent, author, committer) are missing,
// duplicated, or out of order.
var ErrMalformedCommit = errors.New("malformed commit")
// Parent returns the ith parent of a commit.
func (c *Commit) Parent(i int) (*Commit, error) {
if len(c.ParentHashes) == 0 || i > len(c.ParentHashes)-1 {
return nil, ErrParentNotFound
}
return GetCommit(c.s, c.ParentHashes[i])
}
// File returns the file with the specified "path" in the commit and a
// nil error if the file exists. If the file does not exist, it returns
// a nil file and the ErrFileNotFound error.
func (c *Commit) File(path string) (*File, error) {
tree, err := c.Tree()
if err != nil {
return nil, err
}
return tree.File(path)
}
// Files returns a FileIter allowing to iterate over the Tree
func (c *Commit) Files() (*FileIter, error) {
tree, err := c.Tree()
if err != nil {
return nil, err
}
return tree.Files(), nil
}
// ID returns the object ID of the commit. The returned value will always match
// the current value of Commit.Hash.
//
// ID is present to fulfill the Object interface.
func (c *Commit) ID() plumbing.Hash {
return c.Hash
}
// Type returns the type of object. It always returns plumbing.CommitObject.
//
// Type is present to fulfill the Object interface.
func (c *Commit) Type() plumbing.ObjectType {
return plumbing.CommitObject
}
func (c *Commit) reset() {
storer := c.s
*c = Commit{
Encoding: defaultUtf8CommitMessageEncoding,
s: storer,
}
}
// Decode transforms a plumbing.EncodedObject into a Commit struct.
func (c *Commit) Decode(o plumbing.EncodedObject) (err error) {
if o.Type() != plumbing.CommitObject {
return ErrUnsupportedObject
}
c.reset()
c.Hash = o.Hash()
c.src = o
reader, err := o.Reader()
if err != nil {
return err
}
defer ioutil.CheckClose(reader, &err)
r := sync.GetBufioReader(reader)
defer sync.PutBufioReader(r)
s := &commitScanner{r: r, c: c}
for state := scanTree; state != nil; {
state, err = state(s)
if err != nil {
return err
}
}
if !s.sawTree {
return fmt.Errorf("%w: missing tree header", ErrMalformedCommit)
}
c.Message = s.msgbuf.String()
return nil
}
// Encode transforms a Commit into a plumbing.EncodedObject.
func (c *Commit) Encode(o plumbing.EncodedObject) error {
return c.encode(o, true)
}
// EncodeWithoutSignature exports a Commit into a plumbing.EncodedObject
// without any signature headers, producing the payload that PGP/GPG
// signatures are computed over.
//
// Behaviour depends on how the Commit was created:
//
// - For Commits populated by Decode whose exported fields still match the
// source object, the payload is streamed from the raw source bytes with
// gpgsig and gpgsig-sha256 headers (and their continuation lines)
// stripped verbatim. This preserves the exact bytes the signature was
// computed over, regardless of any normalization performed by Decode.
//
// - For Commits constructed in memory, or for decoded Commits whose
// exported fields have been mutated, the payload is derived from the
// current struct fields. Mutation is detected by re-decoding the source
// object and comparing exported fields; if any differ, the in-memory
// representation prevails.
func (c *Commit) EncodeWithoutSignature(o plumbing.EncodedObject) error {
if c.matchesSource() {
return stripObjectSignatures(o, c.src, plumbing.CommitObject)
}
return c.encode(o, false)
}
// matchesSource reports whether c.src is set and re-decoding it produces a
// Commit whose payload-affecting exported fields are identical to those of
// c. It is the auto-detection used by EncodeWithoutSignature to decide
// between the raw bytes and the struct-encoded payload.
//
// Signature and SignatureSHA256 are intentionally excluded from the
// comparison: neither path emits them, so mutating them must not trigger a
// switch to struct-encode (which would change the byte layout the caller is
// trying to verify against).
func (c *Commit) matchesSource() bool {
if c.src == nil {
return false
}
fresh := &Commit{}
if err := fresh.Decode(c.src); err != nil {
return false
}
return c.Hash == fresh.Hash &&
signatureEqual(c.Author, fresh.Author) &&
signatureEqual(c.Committer, fresh.Committer) &&
c.Message == fresh.Message &&
c.TreeHash == fresh.TreeHash &&
c.Encoding == fresh.Encoding &&
slices.Equal(c.ParentHashes, fresh.ParentHashes) &&
slices.Equal(c.ExtraHeaders, fresh.ExtraHeaders)
}
func signatureEqual(a, b Signature) bool {
return a.Name == b.Name &&
a.Email == b.Email &&
a.When.Unix() == b.When.Unix() &&
a.When.Format("-0700") == b.When.Format("-0700")
}
func isStandardHeader(key string) bool {
switch key {
case "tree", "parent", "author", "committer",
headerencoding, headerpgp, headerpgp256:
return true
}
return false
}
func (c *Commit) encode(o plumbing.EncodedObject, includeSig bool) (err error) {
o.SetType(plumbing.CommitObject)
w, err := o.Writer()
if err != nil {
return err
}
defer ioutil.CheckClose(w, &err)
if _, err = fmt.Fprintf(w, "tree %s\n", c.TreeHash.String()); err != nil {
return err
}
for _, parent := range c.ParentHashes {
if _, err = fmt.Fprintf(w, "parent %s\n", parent.String()); err != nil {
return err
}
}
if _, err = fmt.Fprint(w, "author "); err != nil {
return err
}
if err = c.Author.Encode(w); err != nil {
return err
}
if _, err = fmt.Fprint(w, "\ncommitter "); err != nil {
return err
}
if err = c.Committer.Encode(w); err != nil {
return err
}
if string(c.Encoding) != "" && c.Encoding != defaultUtf8CommitMessageEncoding {
if _, err = fmt.Fprintf(w, "\n%s %s", headerencoding, c.Encoding); err != nil {
return err
}
}
for _, header := range c.ExtraHeaders {
if isStandardHeader(header.Key) {
continue
}
if _, err = fmt.Fprintf(w, "\n%s", header); err != nil {
return err
}
}
if c.Signature != "" && includeSig {
if _, err = fmt.Fprint(w, "\n"+headerpgp+" "); err != nil {
return err
}
// Split all the signature lines and re-write with a left padding and
// newline. Use join for this so it's clear that a newline should not be
// added after this section, as it will be added when the message is
// printed.
signature := strings.TrimSuffix(c.Signature, "\n")
lines := strings.Split(signature, "\n")
if _, err = fmt.Fprint(w, strings.Join(lines, "\n ")); err != nil {
return err
}
}
if c.SignatureSHA256 != "" && includeSig {
if _, err = fmt.Fprint(w, "\n"+headerpgp256+" "); err != nil {
return err
}
signature := strings.TrimSuffix(c.SignatureSHA256, "\n")
lines := strings.Split(signature, "\n")
if _, err = fmt.Fprint(w, strings.Join(lines, "\n ")); err != nil {
return err
}
}
if _, err = fmt.Fprintf(w, "\n\n%s", c.Message); err != nil {
return err
}
return err
}
// Stats returns the stats of a commit.
func (c *Commit) Stats() (FileStats, error) {
return c.StatsContext(context.Background())
}
// StatsContext returns the stats of a commit. Error will be return if context
// expires. Provided context must be non-nil.
func (c *Commit) StatsContext(ctx context.Context) (FileStats, error) {
fromTree, err := c.Tree()
if err != nil {
return nil, err
}
toTree := &Tree{}
if c.NumParents() != 0 {
firstParent, err := c.Parents().Next()
if err != nil {
return nil, err
}
toTree, err = firstParent.Tree()
if err != nil {
return nil, err
}
}
patch, err := toTree.PatchContext(ctx, fromTree)
if err != nil {
return nil, err
}
return getFileStatsFromFilePatches(patch.FilePatches()), nil
}
func (c *Commit) String() string {
return fmt.Sprintf(
"%s %s\nAuthor: %s\nDate: %s\n\n%s\n",
plumbing.CommitObject, c.Hash, c.Author.String(),
c.Author.When.Format(DateFormat), indent(c.Message),
)
}
// ErrMultipleSignatures is returned by Verify when the commit carries more
// than one armored signature block. Mirrors upstream's parse_gpg_output
// rejection of GOODSIG/BADSIG status lines after the first
// (gpg-interface.c:257-269): multi-signature commits are intentionally
// unsupported because their provenance cannot be reduced to a single
// authoritative signer.
var ErrMultipleSignatures = errors.New("commit has multiple signatures")
// Verify performs PGP verification of the commit with a provided armored
// keyring and returns openpgp.Entity associated with verifying key on success.
func (c *Commit) Verify(armoredKeyRing string) (*openpgp.Entity, error) {
if countSignatureBlocks([]byte(c.Signature)) > 1 {
return nil, ErrMultipleSignatures
}
keyRingReader := strings.NewReader(armoredKeyRing)
keyring, err := openpgp.ReadArmoredKeyRing(keyRingReader)
if err != nil {
return nil, err
}
// Extract signature.
signature := strings.NewReader(c.Signature)
encoded := &plumbing.MemoryObject{}
// Encode commit components, excluding signature and get a reader object.
if err := c.EncodeWithoutSignature(encoded); err != nil {
return nil, err
}
er, err := encoded.Reader()
if err != nil {
return nil, err
}
return openpgp.CheckArmoredDetachedSignature(keyring, er, signature, nil)
}
// Less defines a compare function to determine which commit is 'earlier' by:
// - First use Committer.When
// - If Committer.When are equal then use Author.When
// - If Author.When also equal then compare the string value of the hash
func (c *Commit) Less(rhs *Commit) bool {
return c.Committer.When.Before(rhs.Committer.When) ||
(c.Committer.When.Equal(rhs.Committer.When) &&
(c.Author.When.Before(rhs.Author.When) ||
(c.Author.When.Equal(rhs.Author.When) && c.Hash.Compare(rhs.Hash.Bytes()) < 0)))
}
func indent(t string) string {
output := make([]string, 0, strings.Count(t, "\n")+1)
for line := range strings.SplitSeq(t, "\n") {
if len(line) != 0 {
line = " " + line
}
output = append(output, line)
}
return strings.Join(output, "\n")
}
// CommitIter is a generic closable interface for iterating over commits.
type CommitIter interface {
Next() (*Commit, error)
ForEach(func(*Commit) error) error
Close()
}
// storerCommitIter provides an iterator from commits in an EncodedObjectStorer.
type storerCommitIter struct {
storer.EncodedObjectIter
s storer.EncodedObjectStorer
}
// NewCommitIter takes a storer.EncodedObjectStorer and a
// storer.EncodedObjectIter and returns a CommitIter that iterates over all
// commits contained in the storer.EncodedObjectIter.
//
// Any non-commit object returned by the storer.EncodedObjectIter is skipped.
func NewCommitIter(s storer.EncodedObjectStorer, iter storer.EncodedObjectIter) CommitIter {
return &storerCommitIter{iter, s}
}
// Next moves the iterator to the next commit and returns a pointer to it. If
// there are no more commits, it returns io.EOF.
func (iter *storerCommitIter) Next() (*Commit, error) {
obj, err := iter.EncodedObjectIter.Next()
if err != nil {
return nil, err
}
return DecodeCommit(iter.s, obj)
}
// ForEach call the cb function for each commit contained on this iter until
// an error appends or the end of the iter is reached. If ErrStop is sent
// the iteration is stopped but no error is returned. The iterator is closed.
func (iter *storerCommitIter) ForEach(cb func(*Commit) error) error {
return iter.EncodedObjectIter.ForEach(func(obj plumbing.EncodedObject) error {
c, err := DecodeCommit(iter.s, obj)
if err != nil {
return err
}
return cb(c)
})
}
func (iter *storerCommitIter) Close() {
iter.EncodedObjectIter.Close()
}
package object
import (
"bufio"
"bytes"
"fmt"
"io"
"strings"
"github.com/go-git/go-git/v6/plumbing"
)
// commitScanner holds the working state of the commit decoder driven by the
// stateFn loop in (*Commit).Decode. Each commitState reads one or more lines
// from r, updates the in-progress *Commit and the scanner's bookkeeping, and
// returns the state that should run next (or nil to stop).
type commitScanner struct {
r *bufio.Reader
c *Commit
msgbuf bytes.Buffer
// pending holds a line that was read but the current state decided to
// hand back to the next state, paired with the io.EOF flag that was
// returned when the line was originally read.
pending []byte
pendingErr error
// First-occurrence tracking — once the corresponding field has been
// decoded, subsequent occurrences are silently dropped (matches
// upstream's find_commit_header / first-wins semantics).
//
// gpgsig/gpgsig-sha256 are NOT tracked here: upstream's
// parse_buffer_signed_by_header (commit.c:1186) accumulates every
// occurrence into one signature buffer, so we do the same on the
// scanner side to keep verification payloads byte-aligned.
sawTree, sawAuthor, sawCommitter bool
sawEncoding bool
// extra is the multi-line ExtraHeader currently being assembled.
extra *ExtraHeader
}
// commitState is one step of the decoder state machine. Each function reads
// the lines it needs, mutates *Commit via s.c, and returns the next state to
// run (or nil to terminate the loop).
type commitState func(*commitScanner) (commitState, error)
// readLine returns the next line from the buffer, transparently consuming any
// line that was previously pushed back by a state that decided not to handle
// it.
func (s *commitScanner) readLine() ([]byte, error) {
if s.pending != nil {
line, err := s.pending, s.pendingErr
s.pending, s.pendingErr = nil, nil
return line, err
}
line, err := s.r.ReadBytes('\n')
if err != nil && err != io.EOF {
return line, err
}
return line, err
}
// pushBack stashes an unconsumed line so the next state's readLine call sees
// it. Only one line can be pushed back at a time.
func (s *commitScanner) pushBack(line []byte, err error) {
s.pending = line
s.pendingErr = err
}
// scanTree expects the first non-empty header to be `tree HASH`. Anything
// else (or an empty buffer) is rejected with ErrMalformedCommit, matching
// upstream's `bogus commit object` check.
func scanTree(s *commitScanner) (commitState, error) {
line, err := s.readLine()
if err != nil && err != io.EOF {
return nil, err
}
if len(line) == 0 || isBlankLine(line) {
return nil, fmt.Errorf("%w: missing tree header", ErrMalformedCommit)
}
key, data := splitHeader(line)
if key != "tree" {
return nil, fmt.Errorf("%w: tree header must be first", ErrMalformedCommit)
}
h, herr := parseObjectIDHex(data, ErrMalformedCommit, "tree")
if herr != nil {
return nil, herr
}
s.c.TreeHash = h
s.sawTree = true
if err == io.EOF {
return nil, nil
}
return scanParents, nil
}
// scanParents consumes contiguous `parent HASH` lines. The first non-parent
// line ends the parent block and is handed off to scanAuthor; any later
// `parent` line is silently dropped (matches upstream's parse_commit_buffer
// exiting its parent loop at the first non-parent line and
// read_commit_extra_header_lines filtering `parent` out of extras).
func scanParents(s *commitScanner) (commitState, error) {
line, err := s.readLine()
if err != nil && err != io.EOF {
return nil, err
}
if len(line) == 0 {
return nil, nil
}
if isBlankLine(line) {
return scanMessage, nil
}
key, data := splitHeader(line)
if key == "parent" {
h, herr := parseObjectIDHex(data, ErrMalformedCommit, "parent")
if herr != nil {
return nil, herr
}
s.c.ParentHashes = append(s.c.ParentHashes, h)
if err == io.EOF {
return nil, nil
}
return scanParents, nil
}
s.pushBack(line, err)
return scanAuthor, nil
}
// scanAuthor accepts an `author` line at its canonical position immediately
// after the parent block. Any other header here is pushed back for
// scanCommitter; an out-of-place author is therefore silently dropped.
// Mirrors upstream's parse_commit_date func.
func scanAuthor(s *commitScanner) (commitState, error) {
line, err := s.readLine()
if err != nil && err != io.EOF {
return nil, err
}
if len(line) == 0 {
return nil, nil
}
if isBlankLine(line) {
return scanMessage, nil
}
key, data := splitHeader(line)
if key == "author" {
s.c.Author.Decode(data)
s.sawAuthor = true
if err == io.EOF {
return nil, nil
}
return scanCommitter, nil
}
s.pushBack(line, err)
return scanCommitter, nil
}
// scanCommitter accepts a `committer` line at its canonical position
// immediately after the author. Any other header is pushed back for
// scanHeaders. Same upstream rationale as scanAuthor.
func scanCommitter(s *commitScanner) (commitState, error) {
line, err := s.readLine()
if err != nil && err != io.EOF {
return nil, err
}
if len(line) == 0 {
return nil, nil
}
if isBlankLine(line) {
return scanMessage, nil
}
key, data := splitHeader(line)
if key == "committer" {
s.c.Committer.Decode(data)
s.sawCommitter = true
if err == io.EOF {
return nil, nil
}
return scanHeaders, nil
}
s.pushBack(line, err)
return scanHeaders, nil
}
// scanHeaders dispatches one header line. Continuation-bearing headers
// (mergetag, gpgsig, gpgsig-sha256, and unknown extras whose value is
// continued on subsequent lines) hand off to a dedicated continuation state
// that handles the `<space>...` lines and then returns here.
func scanHeaders(s *commitScanner) (commitState, error) {
line, err := s.readLine()
if err != nil && err != io.EOF {
return nil, err
}
if len(line) == 0 {
return nil, nil
}
if isBlankLine(line) {
return scanMessage, nil
}
originalLine := line
key, data := splitHeader(line)
var next commitState = scanHeaders
switch key {
case "tree", "parent", "author", "committer":
// Anything reaching scanHeaders with one of these keys is out of
// canonical position — duplicate tree, parent past the contiguous
// block, or author/committer not at their expected slot. Drop them
// the same way upstream's standard_header_field filter excludes
// them from the extras list (read_commit_extra_header_lines,
// commit.c:1520-1522).
case headerencoding:
if !s.sawEncoding {
s.c.Encoding = MessageEncoding(data)
s.sawEncoding = true
}
case headerpgp:
s.c.Signature += string(data) + "\n"
next = scanPgpCont
case headerpgp256:
s.c.SignatureSHA256 += string(data) + "\n"
next = scanPgp256Cont
default:
h, multiline := parseExtraHeader(originalLine)
if multiline {
s.extra = &h
next = scanExtraCont
} else {
s.c.ExtraHeaders = append(s.c.ExtraHeaders, h)
}
}
if err == io.EOF {
return nil, nil
}
return next, nil
}
// scanPgpCont and scanPgp256Cont accumulate continuation lines for a
// signature header. Continuations strip exactly one leading space,
// mirroring upstream's `line + 1` (commit.c:1509). The first
// non-continuation line is pushed back so scanHeaders can dispatch it —
// repeat occurrences of the same signature header land back here and
// concatenate, matching upstream's parse_buffer_signed_by_header
// (commit.c:1186). Mergetag continuations go through scanExtraCont
// because mergetag is modelled as an entry in ExtraHeaders.
func scanPgpCont(s *commitScanner) (commitState, error) {
return continuationCont(s, &s.c.Signature, scanPgpCont)
}
func scanPgp256Cont(s *commitScanner) (commitState, error) {
return continuationCont(s, &s.c.SignatureSHA256, scanPgp256Cont)
}
func continuationCont(s *commitScanner, dst *string, self commitState) (commitState, error) {
line, err := s.readLine()
if err != nil && err != io.EOF {
return nil, err
}
if len(line) > 0 && line[0] == ' ' {
*dst += string(line[1:])
if err == io.EOF {
return nil, nil
}
return self, nil
}
if len(line) > 0 {
s.pushBack(line, err)
}
return scanHeaders, nil
}
// scanExtraCont accumulates continuation lines for an unknown ExtraHeader
// whose value spans multiple lines, then finalises the entry once the
// continuation block ends.
func scanExtraCont(s *commitScanner) (commitState, error) {
line, err := s.readLine()
if err != nil && err != io.EOF {
return nil, err
}
if len(line) > 0 && line[0] == ' ' {
s.extra.Value += string(line[1:])
if err == io.EOF {
s.finaliseExtra()
return nil, nil
}
return scanExtraCont, nil
}
s.finaliseExtra()
if len(line) > 0 {
s.pushBack(line, err)
}
return scanHeaders, nil
}
func (s *commitScanner) finaliseExtra() {
s.extra.Value = strings.TrimRight(s.extra.Value, "\n")
s.c.ExtraHeaders = append(s.c.ExtraHeaders, *s.extra)
s.extra = nil
}
// scanMessage drains the remaining bytes into the message buffer.
func scanMessage(s *commitScanner) (commitState, error) {
for {
line, err := s.readLine()
if err != nil && err != io.EOF {
return nil, err
}
if len(line) > 0 {
s.msgbuf.Write(line)
}
if err == io.EOF {
return nil, nil
}
}
}
// isBlankLine reports whether line is the canonical header/body separator —
// a single newline. Mirrors upstream's `*line == '\n'` test in
// read_commit_extra_header_lines (commit.c:1502).
func isBlankLine(line []byte) bool {
return len(line) == 1 && line[0] == '\n'
}
// splitHeader returns the header keyword (everything before the first space)
// and the value (everything after, with the trailing newline stripped). If
// the header has no value the returned data is nil.
func splitHeader(line []byte) (string, []byte) {
trimmed := bytes.TrimRight(line, "\n")
key, value, ok := bytes.Cut(trimmed, []byte{' '})
if !ok {
return string(trimmed), nil
}
return string(key), value
}
func parseObjectIDHex(data []byte, malformedErr error, header string) (plumbing.Hash, error) {
if len(data) != 40 && len(data) != 64 {
return plumbing.ZeroHash, fmt.Errorf("%w: bad %s hash", malformedErr, header)
}
h, ok := plumbing.FromHex(string(data))
if !ok {
return plumbing.ZeroHash, fmt.Errorf("%w: bad %s hash", malformedErr, header)
}
return h, nil
}
package object
import (
"container/list"
"errors"
"io"
"slices"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/storer"
"github.com/go-git/go-git/v6/storage"
)
type commitPreIterator struct {
seenExternal map[plumbing.Hash]bool
seen map[plumbing.Hash]bool
stack []CommitIter
start *Commit
}
func forEachCommit(next func() (*Commit, error), cb func(*Commit) error) error {
for {
c, err := next()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return err
}
err = cb(c)
if err == storer.ErrStop {
break
}
if err != nil {
return err
}
}
return nil
}
// NewCommitPreorderIter returns a CommitIter that walks the commit history,
// starting at the given commit and visiting its parents in pre-order.
// The given callback will be called for each visited commit. Each commit will
// be visited only once. If the callback returns an error, walking will stop
// and will return the error. Other errors might be returned if the history
// cannot be traversed (e.g. missing objects). Ignore allows to skip some
// commits from being iterated.
func NewCommitPreorderIter(
c *Commit,
seenExternal map[plumbing.Hash]bool,
ignore []plumbing.Hash,
) CommitIter {
seen := make(map[plumbing.Hash]bool)
for _, h := range ignore {
seen[h] = true
}
return &commitPreIterator{
seenExternal: seenExternal,
seen: seen,
stack: make([]CommitIter, 0),
start: c,
}
}
func (w *commitPreIterator) Next() (*Commit, error) {
var c *Commit
for {
if w.start != nil {
c = w.start
w.start = nil
} else {
current := len(w.stack) - 1
if current < 0 {
return nil, io.EOF
}
var err error
c, err = w.stack[current].Next()
if err == io.EOF {
w.stack = w.stack[:current]
continue
}
if err != nil {
return nil, err
}
}
if w.seen[c.Hash] || w.seenExternal[c.Hash] {
continue
}
w.seen[c.Hash] = true
if c.NumParents() > 0 {
w.stack = append(w.stack, filteredParentIter(c, w.seen))
}
return c, nil
}
}
func filteredParentIter(c *Commit, seen map[plumbing.Hash]bool) CommitIter {
var hashes []plumbing.Hash
for _, h := range c.ParentHashes {
if !seen[h] {
hashes = append(hashes, h)
}
}
return NewCommitIter(c.s,
storer.NewEncodedObjectLookupIter(c.s, plumbing.CommitObject, hashes),
)
}
func (w *commitPreIterator) ForEach(cb func(*Commit) error) error {
return forEachCommit(w.Next, cb)
}
func (w *commitPreIterator) Close() {}
type commitPostIterator struct {
stack []*Commit
seen map[plumbing.Hash]bool
}
// NewCommitPostorderIter returns a CommitIter that walks the commit
// history like WalkCommitHistory but in post-order. This means that after
// walking a merge commit, the merged commit will be walked before the base
// it was merged on. This can be useful if you wish to see the history in
// chronological order. Ignore allows to skip some commits from being iterated.
func NewCommitPostorderIter(c *Commit, ignore []plumbing.Hash) CommitIter {
seen := make(map[plumbing.Hash]bool)
for _, h := range ignore {
seen[h] = true
}
return &commitPostIterator{
stack: []*Commit{c},
seen: seen,
}
}
func (w *commitPostIterator) Next() (*Commit, error) {
for {
if len(w.stack) == 0 {
return nil, io.EOF
}
c := w.stack[len(w.stack)-1]
w.stack = w.stack[:len(w.stack)-1]
if w.seen[c.Hash] {
continue
}
w.seen[c.Hash] = true
return c, c.Parents().ForEach(func(p *Commit) error {
w.stack = append(w.stack, p)
return nil
})
}
}
func (w *commitPostIterator) ForEach(cb func(*Commit) error) error {
return forEachCommit(w.Next, cb)
}
func (w *commitPostIterator) Close() {}
type commitPostIteratorFirstParent struct {
stack []*Commit
seen map[plumbing.Hash]bool
}
// NewCommitPostorderIterFirstParent returns a CommitIter that walks the commit
// history like WalkCommitHistory but in post-order.
//
// This option acts like the git log --first-parent flag, skipping intermediate
// commits that were brought in via a merge commit.
// Ignore allows to skip some commits from being iterated.
func NewCommitPostorderIterFirstParent(c *Commit, ignore []plumbing.Hash) CommitIter {
seen := make(map[plumbing.Hash]bool)
for _, h := range ignore {
seen[h] = true
}
return &commitPostIteratorFirstParent{
stack: []*Commit{c},
seen: seen,
}
}
func (w *commitPostIteratorFirstParent) Next() (*Commit, error) {
for {
if len(w.stack) == 0 {
return nil, io.EOF
}
c := w.stack[len(w.stack)-1]
w.stack = w.stack[:len(w.stack)-1]
if w.seen[c.Hash] {
continue
}
w.seen[c.Hash] = true
return c, c.Parents().ForEach(func(p *Commit) error {
if len(c.ParentHashes) > 0 && p.Hash == c.ParentHashes[0] {
w.stack = append(w.stack, p)
}
return nil
})
}
}
func (w *commitPostIteratorFirstParent) ForEach(cb func(*Commit) error) error {
return forEachCommit(w.Next, cb)
}
func (w *commitPostIteratorFirstParent) Close() {}
// commitAllIterator stands for commit iterator for all refs.
type commitAllIterator struct {
// currCommit points to the current commit.
currCommit *list.Element
}
// NewCommitAllIter returns a new commit iterator for all refs.
// repoStorer is a repo Storer used to get commits and references.
// commitIterFunc is a commit iterator function, used to iterate through ref commits in chosen order
func NewCommitAllIter(repoStorer storage.Storer, commitIterFunc func(*Commit) CommitIter) (CommitIter, error) {
commitsPath := list.New()
commitsLookup := make(map[plumbing.Hash]*list.Element)
head, err := storer.ResolveReference(repoStorer, plumbing.HEAD)
if err == nil {
err = addReference(repoStorer, commitIterFunc, head, commitsPath, commitsLookup)
}
if err != nil && err != plumbing.ErrReferenceNotFound {
return nil, err
}
// add all references along with the HEAD
refIter, err := repoStorer.IterReferences()
if err != nil {
return nil, err
}
defer refIter.Close()
for {
ref, err := refIter.Next()
if err == io.EOF {
break
}
if err == plumbing.ErrReferenceNotFound {
continue
}
if err != nil {
return nil, err
}
if err = addReference(repoStorer, commitIterFunc, ref, commitsPath, commitsLookup); err != nil {
return nil, err
}
}
return &commitAllIterator{commitsPath.Front()}, nil
}
func addReference(
repoStorer storage.Storer,
commitIterFunc func(*Commit) CommitIter,
ref *plumbing.Reference,
commitsPath *list.List,
commitsLookup map[plumbing.Hash]*list.Element,
) error {
_, exists := commitsLookup[ref.Hash()]
if exists {
// we already have it - skip the reference.
return nil
}
refCommit, _ := GetCommit(repoStorer, ref.Hash())
if refCommit == nil {
// if it's not a commit - skip it.
return nil
}
var (
refCommits []*Commit
parent *list.Element
)
// collect all ref commits to add
commitIter := commitIterFunc(refCommit)
for c, e := commitIter.Next(); e == nil; {
parent, exists = commitsLookup[c.Hash]
if exists {
break
}
refCommits = append(refCommits, c)
c, e = commitIter.Next()
}
commitIter.Close()
if parent == nil {
// common parent - not found
// add all commits to the path from this ref (maybe it's a HEAD and we don't have anything, yet)
for _, c := range refCommits {
parent = commitsPath.PushBack(c)
commitsLookup[c.Hash] = parent
}
} else {
// add ref's commits to the path in reverse order (from the latest)
for _, c := range slices.Backward(refCommits) {
// insert before found common parent
parent = commitsPath.InsertBefore(c, parent)
commitsLookup[c.Hash] = parent
}
}
return nil
}
func (it *commitAllIterator) Next() (*Commit, error) {
if it.currCommit == nil {
return nil, io.EOF
}
c := it.currCommit.Value.(*Commit)
it.currCommit = it.currCommit.Next()
return c, nil
}
func (it *commitAllIterator) ForEach(cb func(*Commit) error) error {
return forEachCommit(it.Next, cb)
}
func (it *commitAllIterator) Close() {
it.currCommit = nil
}
package object
import (
"errors"
"io"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/storer"
)
type bfsCommitIterator struct {
seenExternal map[plumbing.Hash]bool
seen map[plumbing.Hash]bool
queue []*Commit
}
// NewCommitIterBSF returns a CommitIter that walks the commit history,
// starting at the given commit and visiting its parents in pre-order.
// The given callback will be called for each visited commit. Each commit will
// be visited only once. If the callback returns an error, walking will stop
// and will return the error. Other errors might be returned if the history
// cannot be traversed (e.g. missing objects). Ignore allows to skip some
// commits from being iterated.
func NewCommitIterBSF(
c *Commit,
seenExternal map[plumbing.Hash]bool,
ignore []plumbing.Hash,
) CommitIter {
seen := make(map[plumbing.Hash]bool)
for _, h := range ignore {
seen[h] = true
}
return &bfsCommitIterator{
seenExternal: seenExternal,
seen: seen,
queue: []*Commit{c},
}
}
func (w *bfsCommitIterator) appendHash(store storer.EncodedObjectStorer, h plumbing.Hash) error {
if w.seen[h] || w.seenExternal[h] {
return nil
}
c, err := GetCommit(store, h)
if err != nil {
return err
}
w.queue = append(w.queue, c)
return nil
}
func (w *bfsCommitIterator) Next() (*Commit, error) {
var c *Commit
for {
if len(w.queue) == 0 {
return nil, io.EOF
}
c = w.queue[0]
w.queue = w.queue[1:]
if w.seen[c.Hash] || w.seenExternal[c.Hash] {
continue
}
w.seen[c.Hash] = true
for _, h := range c.ParentHashes {
err := w.appendHash(c.s, h)
if err != nil {
return nil, err
}
}
return c, nil
}
}
func (w *bfsCommitIterator) ForEach(cb func(*Commit) error) error {
for {
c, err := w.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
err = cb(c)
if errors.Is(err, storer.ErrStop) {
break
}
if err != nil {
return err
}
}
return nil
}
func (w *bfsCommitIterator) Close() {}
package object
import (
"errors"
"io"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/storer"
)
// NewFilterCommitIter returns a CommitIter that walks the commit history,
// starting at the passed commit and visiting its parents in Breadth-first order.
// The commits returned by the CommitIter will validate the passed CommitFilter.
// The history won't be transversed beyond a commit if isLimit is true for it.
// Each commit will be visited only once.
// If the commit history can not be traversed, or the Close() method is called,
// the CommitIter won't return more commits.
// If no isValid is passed, all ancestors of from commit will be valid.
// If no isLimit is limit, all ancestors of all commits will be visited.
func NewFilterCommitIter(
from *Commit,
isValid *CommitFilter,
isLimit *CommitFilter,
) CommitIter {
var validFilter CommitFilter
if isValid == nil {
validFilter = func(_ *Commit) bool {
return true
}
} else {
validFilter = *isValid
}
var limitFilter CommitFilter
if isLimit == nil {
limitFilter = func(_ *Commit) bool {
return false
}
} else {
limitFilter = *isLimit
}
return &filterCommitIter{
isValid: validFilter,
isLimit: limitFilter,
visited: map[plumbing.Hash]struct{}{},
queue: []*Commit{from},
}
}
// CommitFilter returns a boolean for the passed Commit
type CommitFilter func(*Commit) bool
// filterCommitIter implements CommitIter
type filterCommitIter struct {
isValid CommitFilter
isLimit CommitFilter
visited map[plumbing.Hash]struct{}
queue []*Commit
lastErr error
}
// Next returns the next commit of the CommitIter.
// It will return io.EOF if there are no more commits to visit,
// or an error if the history could not be traversed.
func (w *filterCommitIter) Next() (*Commit, error) {
var commit *Commit
var err error
for {
commit, err = w.popNewFromQueue()
if err != nil {
return nil, w.close(err)
}
w.visited[commit.Hash] = struct{}{}
if !w.isLimit(commit) {
err = w.addToQueue(commit.s, commit.ParentHashes...)
if err != nil {
return nil, w.close(err)
}
}
if w.isValid(commit) {
return commit, nil
}
}
}
// ForEach runs the passed callback over each Commit returned by the CommitIter
// until the callback returns an error or there is no more commits to traverse.
func (w *filterCommitIter) ForEach(cb func(*Commit) error) error {
for {
commit, err := w.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
if err := cb(commit); errors.Is(err, storer.ErrStop) {
break
} else if err != nil {
return err
}
}
return nil
}
// Error returns the error that caused that the CommitIter is no longer returning commits
func (w *filterCommitIter) Error() error {
return w.lastErr
}
// Close closes the CommitIter
func (w *filterCommitIter) Close() {
w.visited = map[plumbing.Hash]struct{}{}
w.queue = []*Commit{}
w.isLimit = nil
w.isValid = nil
}
// close closes the CommitIter with an error
func (w *filterCommitIter) close(err error) error {
w.Close()
w.lastErr = err
return err
}
// popNewFromQueue returns the first new commit from the internal fifo queue,
// or an io.EOF error if the queue is empty
func (w *filterCommitIter) popNewFromQueue() (*Commit, error) {
var first *Commit
for {
if len(w.queue) == 0 {
if w.lastErr != nil {
return nil, w.lastErr
}
return nil, io.EOF
}
first = w.queue[0]
w.queue = w.queue[1:]
if _, ok := w.visited[first.Hash]; ok {
continue
}
return first, nil
}
}
// addToQueue adds the passed commits to the internal fifo queue if they weren't seen
// or returns an error if the passed hashes could not be used to get valid commits
func (w *filterCommitIter) addToQueue(
store storer.EncodedObjectStorer,
hashes ...plumbing.Hash,
) error {
for _, hash := range hashes {
if _, ok := w.visited[hash]; ok {
continue
}
commit, err := GetCommit(store, hash)
if err != nil {
return err
}
w.queue = append(w.queue, commit)
}
return nil
}
package object
import (
"errors"
"io"
"github.com/emirpasic/gods/trees/binaryheap"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/storer"
)
type commitIteratorByCTime struct {
seenExternal map[plumbing.Hash]bool
seen map[plumbing.Hash]bool
heap *binaryheap.Heap
}
// NewCommitIterCTime returns a CommitIter that walks the commit history,
// starting at the given commit and visiting its parents while preserving Committer Time order.
// this appears to be the closest order to `git log`
// The given callback will be called for each visited commit. Each commit will
// be visited only once. If the callback returns an error, walking will stop
// and will return the error. Other errors might be returned if the history
// cannot be traversed (e.g. missing objects). Ignore allows to skip some
// commits from being iterated.
func NewCommitIterCTime(
c *Commit,
seenExternal map[plumbing.Hash]bool,
ignore []plumbing.Hash,
) CommitIter {
seen := make(map[plumbing.Hash]bool)
for _, h := range ignore {
seen[h] = true
}
heap := binaryheap.NewWith(func(a, b any) int {
if a.(*Commit).Committer.When.Before(b.(*Commit).Committer.When) {
return 1
}
return -1
})
heap.Push(c)
return &commitIteratorByCTime{
seenExternal: seenExternal,
seen: seen,
heap: heap,
}
}
func (w *commitIteratorByCTime) Next() (*Commit, error) {
var c *Commit
for {
cIn, ok := w.heap.Pop()
if !ok {
return nil, io.EOF
}
c = cIn.(*Commit)
if w.seen[c.Hash] || w.seenExternal[c.Hash] {
continue
}
w.seen[c.Hash] = true
for _, h := range c.ParentHashes {
if w.seen[h] || w.seenExternal[h] {
continue
}
pc, err := GetCommit(c.s, h)
if err != nil {
return nil, err
}
w.heap.Push(pc)
}
return c, nil
}
}
func (w *commitIteratorByCTime) ForEach(cb func(*Commit) error) error {
for {
c, err := w.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
err = cb(c)
if errors.Is(err, storer.ErrStop) {
break
}
if err != nil {
return err
}
}
return nil
}
func (w *commitIteratorByCTime) Close() {}
package object
import (
"errors"
"io"
"time"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/storer"
)
type commitLimitIter struct {
sourceIter CommitIter
limitOptions LogLimitOptions
}
// LogLimitOptions defines limits for log traversal.
type LogLimitOptions struct {
Since *time.Time
Until *time.Time
TailHash plumbing.Hash
}
// NewCommitLimitIterFromIter creates a new commit iterator with limits applied.
func NewCommitLimitIterFromIter(commitIter CommitIter, limitOptions LogLimitOptions) CommitIter {
iterator := new(commitLimitIter)
iterator.sourceIter = commitIter
iterator.limitOptions = limitOptions
return iterator
}
func (c *commitLimitIter) Next() (*Commit, error) {
for {
commit, err := c.sourceIter.Next()
if err != nil {
return nil, err
}
if c.limitOptions.Since != nil && commit.Committer.When.Before(*c.limitOptions.Since) {
continue
}
if c.limitOptions.Until != nil && commit.Committer.When.After(*c.limitOptions.Until) {
continue
}
if c.limitOptions.TailHash == commit.Hash {
return commit, storer.ErrStop
}
return commit, nil
}
}
func (c *commitLimitIter) ForEach(cb func(*Commit) error) error {
for {
commit, nextErr := c.Next()
if nextErr == io.EOF {
break
}
if nextErr != nil && !errors.Is(nextErr, storer.ErrStop) {
return nextErr
}
err := cb(commit)
if errors.Is(err, storer.ErrStop) || errors.Is(nextErr, storer.ErrStop) {
return nil
} else if err != nil {
return err
}
}
return nil
}
func (c *commitLimitIter) Close() {
c.sourceIter.Close()
}
package object
import (
"errors"
"io"
"slices"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/storer"
)
type commitPathIter struct {
pathFilter func(string) bool
sourceIter CommitIter
currentCommit *Commit
checkParent bool
}
// NewCommitPathIterFromIter returns a commit iterator which performs diffTree between
// successive trees returned from the commit iterator from the argument. The purpose of this is
// to find the commits that explain how the files that match the path came to be.
// If checkParent is true then the function double checks if potential parent (next commit in a path)
// is one of the parents in the tree (it's used by `git log --all`).
// pathFilter is a function that takes path of file as argument and returns true if we want it
func NewCommitPathIterFromIter(pathFilter func(string) bool, commitIter CommitIter, checkParent bool) CommitIter {
iterator := new(commitPathIter)
iterator.sourceIter = commitIter
iterator.pathFilter = pathFilter
iterator.checkParent = checkParent
return iterator
}
// NewCommitFileIterFromIter is kept for compatibility, can be replaced with NewCommitPathIterFromIter
func NewCommitFileIterFromIter(fileName string, commitIter CommitIter, checkParent bool) CommitIter {
return NewCommitPathIterFromIter(
func(path string) bool {
return path == fileName
},
commitIter,
checkParent,
)
}
func (c *commitPathIter) Next() (*Commit, error) {
if c.currentCommit == nil {
var err error
c.currentCommit, err = c.sourceIter.Next()
if err != nil {
return nil, err
}
}
commit, commitErr := c.getNextFileCommit()
// Setting current-commit to nil to prevent unwanted states when errors are raised
if commitErr != nil {
c.currentCommit = nil
}
return commit, commitErr
}
func (c *commitPathIter) getNextFileCommit() (*Commit, error) {
var parentTree, currentTree *Tree
for {
// Parent-commit can be nil if the current-commit is the initial commit
parentCommit, parentCommitErr := c.sourceIter.Next()
if parentCommitErr != nil {
// If the parent-commit is beyond the initial commit, keep it nil
if parentCommitErr != io.EOF {
return nil, parentCommitErr
}
parentCommit = nil
}
if parentTree == nil {
var currTreeErr error
currentTree, currTreeErr = c.currentCommit.Tree()
if currTreeErr != nil {
return nil, currTreeErr
}
} else {
currentTree = parentTree
parentTree = nil
}
if parentCommit != nil {
var parentTreeErr error
parentTree, parentTreeErr = parentCommit.Tree()
if parentTreeErr != nil {
return nil, parentTreeErr
}
}
// Find diff between current and parent trees
changes, diffErr := DiffTree(currentTree, parentTree)
if diffErr != nil {
return nil, diffErr
}
found := c.hasFileChange(changes, parentCommit)
// Storing the current-commit in-case a change is found, and
// Updating the current-commit for the next-iteration
prevCommit := c.currentCommit
c.currentCommit = parentCommit
if found {
return prevCommit, nil
}
// If not matches found and if parent-commit is beyond the initial commit, then return with EOF
if parentCommit == nil {
return nil, io.EOF
}
}
}
func (c *commitPathIter) hasFileChange(changes Changes, parent *Commit) bool {
for _, change := range changes {
if !c.pathFilter(change.name()) {
continue
}
// filename matches, now check if source iterator contains all commits (from all refs)
if c.checkParent {
// Check if parent is beyond the initial commit
if parent == nil || isParentHash(parent.Hash, c.currentCommit) {
return true
}
continue
}
return true
}
return false
}
func isParentHash(hash plumbing.Hash, commit *Commit) bool {
return slices.Contains(commit.ParentHashes, hash)
}
func (c *commitPathIter) ForEach(cb func(*Commit) error) error {
for {
commit, nextErr := c.Next()
if nextErr == io.EOF {
break
}
if nextErr != nil {
return nextErr
}
err := cb(commit)
if errors.Is(err, storer.ErrStop) {
return nil
} else if err != nil {
return err
}
}
return nil
}
func (c *commitPathIter) Close() {
c.sourceIter.Close()
}
package object
import (
"bytes"
"context"
"errors"
"github.com/go-git/go-git/v6/utils/merkletrie"
"github.com/go-git/go-git/v6/utils/merkletrie/noder"
)
// DiffTree compares the content and mode of the blobs found via two
// tree objects.
// DiffTree does not perform rename detection, use DiffTreeWithOptions
// instead to detect renames.
func DiffTree(a, b *Tree) (Changes, error) {
return DiffTreeContext(context.Background(), a, b)
}
// DiffTreeContext compares the content and mode of the blobs found via two
// tree objects. Provided context must be non-nil.
// An error will be returned if context expires.
func DiffTreeContext(ctx context.Context, a, b *Tree) (Changes, error) {
return DiffTreeWithOptions(ctx, a, b, nil)
}
// DiffTreeOptions are the configurable options when performing a diff tree.
type DiffTreeOptions struct {
// DetectRenames is whether the diff tree will use rename detection.
DetectRenames bool
// RenameScore is the threshold to of similarity between files to consider
// that a pair of delete and insert are a rename. The number must be
// exactly between 0 and 100.
RenameScore uint
// RenameLimit is the maximum amount of files that can be compared when
// detecting renames. The number of comparisons that have to be performed
// is equal to the number of deleted files * the number of added files.
// That means, that if 100 files were deleted and 50 files were added, 5000
// file comparisons may be needed. So, if the rename limit is 50, the number
// of both deleted and added needs to be equal or less than 50.
// A value of 0 means no limit.
RenameLimit uint
// OnlyExactRenames performs only detection of exact renames and will not perform
// any detection of renames based on file similarity.
OnlyExactRenames bool
}
// DefaultDiffTreeOptions are the default and recommended options for the
// diff tree.
var DefaultDiffTreeOptions = &DiffTreeOptions{
DetectRenames: true,
RenameScore: 60,
RenameLimit: 0,
OnlyExactRenames: false,
}
// DiffTreeWithOptions compares the content and mode of the blobs found
// via two tree objects with the given options. The provided context
// must be non-nil.
// If no options are passed, no rename detection will be performed. The
// recommended options are DefaultDiffTreeOptions.
// An error will be returned if the context expires.
// This function will be deprecated and removed in v6 so the default
// behaviour of DiffTree is to detect renames.
func DiffTreeWithOptions(
ctx context.Context,
a, b *Tree,
opts *DiffTreeOptions,
) (Changes, error) {
from := NewTreeRootNode(a)
to := NewTreeRootNode(b)
hashEqual := func(a, b noder.Hasher) bool {
return bytes.Equal(a.Hash(), b.Hash())
}
merkletrieChanges, err := merkletrie.DiffTreeContext(ctx, from, to, hashEqual)
if err != nil {
if errors.Is(err, merkletrie.ErrCanceled) {
return nil, ErrCanceled
}
return nil, err
}
changes, err := newChanges(merkletrieChanges)
if err != nil {
return nil, err
}
if opts == nil {
opts = new(DiffTreeOptions)
}
if opts.DetectRenames {
return DetectRenames(changes, opts)
}
return changes, nil
}
package object
import (
"bytes"
"errors"
"io"
"strings"
"github.com/go-git/go-git/v6/plumbing/filemode"
"github.com/go-git/go-git/v6/plumbing/storer"
"github.com/go-git/go-git/v6/utils/binary"
"github.com/go-git/go-git/v6/utils/ioutil"
)
// File represents git file objects.
type File struct {
// Name is the path of the file. It might be relative to a tree,
// depending of the function that generates it.
Name string
// Mode is the file mode.
Mode filemode.FileMode
// Blob with the contents of the file.
Blob
}
// NewFile returns a File based on the given blob object
func NewFile(name string, m filemode.FileMode, b *Blob) *File {
return &File{Name: name, Mode: m, Blob: *b}
}
// Contents returns the contents of a file as a string.
func (f *File) Contents() (content string, err error) {
reader, err := f.Reader()
if err != nil {
return "", err
}
defer ioutil.CheckClose(reader, &err)
buf := new(bytes.Buffer)
if _, err := buf.ReadFrom(reader); err != nil {
return "", err
}
return buf.String(), nil
}
// IsBinary returns if the file is binary or not
func (f *File) IsBinary() (bin bool, err error) {
reader, err := f.Reader()
if err != nil {
return false, err
}
defer ioutil.CheckClose(reader, &err)
return binary.IsBinary(reader)
}
// Lines returns a slice of lines from the contents of a file, stripping
// all end of line characters. If the last line is empty (does not end
// in an end of line), it is also stripped.
func (f *File) Lines() ([]string, error) {
content, err := f.Contents()
if err != nil {
return nil, err
}
splits := strings.Split(content, "\n")
// remove the last line if it is empty
if splits[len(splits)-1] == "" {
return splits[:len(splits)-1], nil
}
return splits, nil
}
// FileIter provides an iterator for the files in a tree.
type FileIter struct {
s storer.EncodedObjectStorer
w TreeWalker
}
// NewFileIter takes a storer.EncodedObjectStorer and a Tree and returns a
// *FileIter that iterates over all files contained in the tree, recursively.
func NewFileIter(s storer.EncodedObjectStorer, t *Tree) *FileIter {
return &FileIter{s: s, w: *NewTreeWalker(t, true, nil)}
}
// Next moves the iterator to the next file and returns a pointer to it. If
// there are no more files, it returns io.EOF.
func (iter *FileIter) Next() (*File, error) {
for {
name, entry, err := iter.w.Next()
if err != nil {
return nil, err
}
if entry.Mode == filemode.Dir || entry.Mode == filemode.Submodule {
continue
}
blob, err := GetBlob(iter.s, entry.Hash)
if err != nil {
return nil, err
}
return NewFile(name, entry.Mode, blob), nil
}
}
// ForEach call the cb function for each file contained in this iter until
// an error happens or the end of the iter is reached. If plumbing.ErrStop is sent
// the iteration is stop but no error is returned. The iterator is closed.
func (iter *FileIter) ForEach(cb func(*File) error) error {
defer iter.Close()
for {
f, err := iter.Next()
if err != nil {
if err == io.EOF {
return nil
}
return err
}
if err := cb(f); err != nil {
if errors.Is(err, storer.ErrStop) {
return nil
}
return err
}
}
}
// Close releases resources associated with the iterator.
func (iter *FileIter) Close() {
iter.w.Close()
}
package object
import (
"errors"
"fmt"
"sort"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/storer"
)
// errIsReachable is thrown when first commit is an ancestor of the second
var errIsReachable = fmt.Errorf("first is reachable from second")
// MergeBase mimics the behavior of `git merge-base actual other`, returning the
// best common ancestor between the actual and the passed one.
// The best common ancestors can not be reached from other common ancestors.
func (c *Commit) MergeBase(other *Commit) ([]*Commit, error) {
// use sortedByCommitDateDesc strategy
sorted := sortByCommitDateDesc(c, other)
newer := sorted[0]
older := sorted[1]
newerHistory, err := ancestorsIndex(older, newer)
if errors.Is(err, errIsReachable) {
return []*Commit{older}, nil
}
if err != nil {
return nil, err
}
var res []*Commit
inNewerHistory := isInIndexCommitFilter(newerHistory)
resIter := NewFilterCommitIter(older, &inNewerHistory, &inNewerHistory)
_ = resIter.ForEach(func(commit *Commit) error {
res = append(res, commit)
return nil
})
return Independents(res)
}
// IsAncestor returns true if the actual commit is ancestor of the passed one.
// It returns an error if the history is not transversable
// It mimics the behavior of `git merge --is-ancestor actual other`
func (c *Commit) IsAncestor(other *Commit) (bool, error) {
found := false
iter := NewCommitPreorderIter(other, nil, nil)
err := iter.ForEach(func(comm *Commit) error {
if comm.Hash != c.Hash {
return nil
}
found = true
return storer.ErrStop
})
return found, err
}
// ancestorsIndex returns a map with the ancestors of the starting commit if the
// excluded one is not one of them. It returns errIsReachable if the excluded commit
// is ancestor of the starting, or another error if the history is not traversable.
func ancestorsIndex(excluded, starting *Commit) (map[plumbing.Hash]struct{}, error) {
if excluded.Hash.String() == starting.Hash.String() {
return nil, errIsReachable
}
startingHistory := map[plumbing.Hash]struct{}{}
startingIter := NewCommitIterBSF(starting, nil, nil)
err := startingIter.ForEach(func(commit *Commit) error {
if commit.Hash == excluded.Hash {
return errIsReachable
}
startingHistory[commit.Hash] = struct{}{}
return nil
})
if err != nil {
return nil, err
}
return startingHistory, nil
}
// Independents returns a subset of the passed commits, that are not reachable the others
// It mimics the behavior of `git merge-base --independent commit...`.
func Independents(commits []*Commit) ([]*Commit, error) {
// use sortedByCommitDateDesc strategy
candidates := sortByCommitDateDesc(commits...)
candidates = removeDuplicated(candidates)
seen := map[plumbing.Hash]struct{}{}
var isLimit CommitFilter = func(commit *Commit) bool {
_, ok := seen[commit.Hash]
return ok
}
if len(candidates) < 2 {
return candidates, nil
}
pos := 0
for {
from := candidates[pos]
others := remove(candidates, from)
fromHistoryIter := NewFilterCommitIter(from, nil, &isLimit)
err := fromHistoryIter.ForEach(func(fromAncestor *Commit) error {
for _, other := range others {
if fromAncestor.Hash == other.Hash {
candidates = remove(candidates, other)
others = remove(others, other)
}
}
if len(candidates) == 1 {
return storer.ErrStop
}
seen[fromAncestor.Hash] = struct{}{}
return nil
})
if err != nil {
return nil, err
}
nextPos := indexOf(candidates, from) + 1
if nextPos >= len(candidates) {
break
}
pos = nextPos
}
return candidates, nil
}
// sortByCommitDateDesc returns the passed commits, sorted by `committer.When desc`
//
// Following this strategy, it is tried to reduce the time needed when walking
// the history from one commit to reach the others. It is assumed that ancestors
// use to be committed before its descendant;
// That way `Independents(A^, A)` will be processed as being `Independents(A, A^)`;
// so starting by `A` it will be reached `A^` way sooner than walking from `A^`
// to the initial commit, and then from `A` to `A^`.
func sortByCommitDateDesc(commits ...*Commit) []*Commit {
sorted := make([]*Commit, len(commits))
copy(sorted, commits)
sort.Slice(sorted, func(i, j int) bool {
return sorted[i].Committer.When.After(sorted[j].Committer.When)
})
return sorted
}
// indexOf returns the first position where target was found in the passed commits
func indexOf(commits []*Commit, target *Commit) int {
for i, commit := range commits {
if target.Hash == commit.Hash {
return i
}
}
return -1
}
// remove returns the passed commits excluding the commit toDelete
func remove(commits []*Commit, toDelete *Commit) []*Commit {
res := make([]*Commit, len(commits))
j := 0
for _, commit := range commits {
if commit.Hash == toDelete.Hash {
continue
}
res[j] = commit
j++
}
return res[:j]
}
// removeDuplicated removes duplicated commits from the passed slice of commits
func removeDuplicated(commits []*Commit) []*Commit {
seen := make(map[plumbing.Hash]struct{}, len(commits))
res := make([]*Commit, len(commits))
j := 0
for _, commit := range commits {
if _, ok := seen[commit.Hash]; ok {
continue
}
seen[commit.Hash] = struct{}{}
res[j] = commit
j++
}
return res[:j]
}
// isInIndexCommitFilter returns a commitFilter that returns true
// if the commit is in the passed index.
func isInIndexCommitFilter(index map[plumbing.Hash]struct{}) CommitFilter {
return func(c *Commit) bool {
_, ok := index[c.Hash]
return ok
}
}
// Package object contains implementations of all Git objects and utility
// functions to work with them.
package object
import (
"bytes"
"errors"
"fmt"
"io"
"strconv"
"time"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/storer"
)
// ErrUnsupportedObject trigger when a non-supported object is being decoded.
var ErrUnsupportedObject = errors.New("unsupported object type")
// Object is a generic representation of any git object. It is implemented by
// Commit, Tree, Blob, and Tag, and includes the functions that are common to
// them.
//
// Object is returned when an object can be of any type. It is frequently used
// with a type cast to acquire the specific type of object:
//
// func process(obj Object) {
// switch o := obj.(type) {
// case *Commit:
// // o is a Commit
// case *Tree:
// // o is a Tree
// case *Blob:
// // o is a Blob
// case *Tag:
// // o is a Tag
// }
// }
//
// This interface is intentionally different from plumbing.EncodedObject, which
// is a lower level interface used by storage implementations to read and write
// objects in its encoded form.
type Object interface {
ID() plumbing.Hash
Type() plumbing.ObjectType
Decode(plumbing.EncodedObject) error
Encode(plumbing.EncodedObject) error
}
// GetObject gets an object from an object storer and decodes it.
func GetObject(s storer.EncodedObjectStorer, h plumbing.Hash) (Object, error) {
o, err := s.EncodedObject(plumbing.AnyObject, h)
if err != nil {
return nil, err
}
return DecodeObject(s, o)
}
// DecodeObject decodes an encoded object into an Object and associates it to
// the given object storer.
func DecodeObject(s storer.EncodedObjectStorer, o plumbing.EncodedObject) (Object, error) {
switch o.Type() {
case plumbing.CommitObject:
return DecodeCommit(s, o)
case plumbing.TreeObject:
return DecodeTree(s, o)
case plumbing.BlobObject:
return DecodeBlob(o)
case plumbing.TagObject:
return DecodeTag(s, o)
default:
return nil, plumbing.ErrInvalidType
}
}
// DateFormat is the format being used in the original git implementation
const DateFormat = "Mon Jan 02 15:04:05 2006 -0700"
// Signature is used to identify who and when created a commit or tag.
type Signature struct {
// Name represents a person name. It is an arbitrary string.
Name string
// Email is an email, but it cannot be assumed to be well-formed.
Email string
// When is the timestamp of the signature.
When time.Time
}
// Decode decodes a byte slice into a signature
func (s *Signature) Decode(b []byte) {
open := bytes.LastIndexByte(b, '<')
closeBracket := bytes.LastIndexByte(b, '>')
if open == -1 || closeBracket == -1 {
return
}
if closeBracket < open {
return
}
s.Name = string(bytes.Trim(b[:open], " "))
s.Email = string(b[open+1 : closeBracket])
hasTime := closeBracket+2 < len(b)
if hasTime {
s.decodeTimeAndTimeZone(b[closeBracket+2:])
}
}
// Encode encodes a Signature into a writer.
func (s *Signature) Encode(w io.Writer) error {
if _, err := fmt.Fprintf(w, "%s <%s> ", s.Name, s.Email); err != nil {
return err
}
if err := s.encodeTimeAndTimeZone(w); err != nil {
return err
}
return nil
}
var timeZoneLength = 5
func (s *Signature) decodeTimeAndTimeZone(b []byte) {
space := bytes.IndexByte(b, ' ')
if space == -1 {
space = len(b)
}
ts, err := strconv.ParseInt(string(b[:space]), 10, 64)
if err != nil {
return
}
s.When = time.Unix(ts, 0).In(time.UTC)
tzStart := space + 1
if tzStart >= len(b) || tzStart+timeZoneLength > len(b) {
return
}
timezone := string(b[tzStart : tzStart+timeZoneLength])
tzhours, err1 := strconv.ParseInt(timezone[0:3], 10, 64)
tzmins, err2 := strconv.ParseInt(timezone[3:], 10, 64)
if err1 != nil || err2 != nil {
return
}
if tzhours < 0 {
tzmins *= -1
}
tz := time.FixedZone("", int(tzhours*60*60+tzmins*60))
s.When = s.When.In(tz)
}
func (s *Signature) encodeTimeAndTimeZone(w io.Writer) error {
u := max(s.When.Unix(), 0)
_, err := fmt.Fprintf(w, "%d %s", u, s.When.Format("-0700"))
return err
}
func (s *Signature) String() string {
return fmt.Sprintf("%s <%s>", s.Name, s.Email)
}
// ObjectIter provides an iterator for a set of objects.
type ObjectIter struct { //nolint:revive // stutters but is a well-established name
storer.EncodedObjectIter
s storer.EncodedObjectStorer
}
// NewObjectIter takes a storer.EncodedObjectStorer and a
// storer.EncodedObjectIter and returns an *ObjectIter that iterates over all
// objects contained in the storer.EncodedObjectIter.
func NewObjectIter(s storer.EncodedObjectStorer, iter storer.EncodedObjectIter) *ObjectIter {
return &ObjectIter{iter, s}
}
// Next moves the iterator to the next object and returns a pointer to it. If
// there are no more objects, it returns io.EOF.
func (iter *ObjectIter) Next() (Object, error) {
for {
obj, err := iter.EncodedObjectIter.Next()
if err != nil {
return nil, err
}
o, err := iter.toObject(obj)
if errors.Is(err, plumbing.ErrInvalidType) {
continue
}
if err != nil {
return nil, err
}
return o, nil
}
}
// ForEach call the cb function for each object contained on this iter until
// an error happens or the end of the iter is reached. If ErrStop is sent
// the iteration is stop but no error is returned. The iterator is closed.
func (iter *ObjectIter) ForEach(cb func(Object) error) error {
return iter.EncodedObjectIter.ForEach(func(obj plumbing.EncodedObject) error {
o, err := iter.toObject(obj)
if errors.Is(err, plumbing.ErrInvalidType) {
return nil
}
if err != nil {
return err
}
return cb(o)
})
}
func (iter *ObjectIter) toObject(obj plumbing.EncodedObject) (Object, error) {
switch obj.Type() {
case plumbing.BlobObject:
blob := &Blob{}
return blob, blob.Decode(obj)
case plumbing.TreeObject:
tree := &Tree{s: iter.s}
return tree, tree.Decode(obj)
case plumbing.CommitObject:
commit := &Commit{}
return commit, commit.Decode(obj)
case plumbing.TagObject:
tag := &Tag{}
return tag, tag.Decode(obj)
default:
return nil, plumbing.ErrInvalidType
}
}
package object
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"strconv"
"strings"
dmp "github.com/sergi/go-diff/diffmatchpatch"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/filemode"
fdiff "github.com/go-git/go-git/v6/plumbing/format/diff"
"github.com/go-git/go-git/v6/utils/diff"
)
// ErrCanceled is returned when the operation is canceled.
var ErrCanceled = errors.New("operation canceled")
func getPatch(message string, changes ...*Change) (*Patch, error) {
ctx := context.Background()
return getPatchContext(ctx, message, changes...)
}
func getPatchContext(ctx context.Context, message string, changes ...*Change) (*Patch, error) {
if len(changes) == 0 {
return &Patch{message: message}, nil
}
filePatches := make([]fdiff.FilePatch, 0, len(changes))
for _, c := range changes {
select {
case <-ctx.Done():
return nil, ErrCanceled
default:
}
fp, err := filePatchWithContext(ctx, c)
if err != nil {
return nil, err
}
filePatches = append(filePatches, fp)
}
return &Patch{message, filePatches}, nil
}
func filePatchWithContext(ctx context.Context, c *Change) (fdiff.FilePatch, error) {
// Submodules (gitlinks) are not blob objects, so their contents cannot be
// read as files. Git represents them in a diff by a single
// "Subproject commit <hash>" line, so build that content synthetically
// instead of dropping the change from the patch.
if isSubmodule(c.From) || isSubmodule(c.To) {
return submoduleFilePatch(ctx, c)
}
from, to, err := c.Files()
if err != nil {
return nil, err
}
fromContent, fIsBinary, err := fileContent(from)
if err != nil {
return nil, err
}
toContent, tIsBinary, err := fileContent(to)
if err != nil {
return nil, err
}
if fIsBinary || tIsBinary {
return &textFilePatch{from: c.From, to: c.To}, nil
}
diffs := diff.Do(fromContent, toContent)
chunks := make([]fdiff.Chunk, 0, len(diffs))
for _, d := range diffs {
select {
case <-ctx.Done():
return nil, ErrCanceled
default:
}
var op fdiff.Operation
switch d.Type {
case dmp.DiffEqual:
op = fdiff.Equal
case dmp.DiffDelete:
op = fdiff.Delete
case dmp.DiffInsert:
op = fdiff.Add
}
chunks = append(chunks, &textChunk{d.Text, op})
}
return &textFilePatch{
chunks: chunks,
from: c.From,
to: c.To,
}, nil
}
func isSubmodule(e ChangeEntry) bool {
return e != empty && e.TreeEntry.Mode == filemode.Submodule
}
// submoduleContent returns the textual representation git uses for a submodule
// (gitlink) in a diff: a single "Subproject commit <hash>" line. It returns an
// empty string when the entry does not point to a submodule.
func submoduleContent(e ChangeEntry) string {
if !isSubmodule(e) {
return ""
}
return fmt.Sprintf("Subproject commit %s\n", e.TreeEntry.Hash)
}
// submoduleFilePatch builds a file patch for a change that adds, removes or
// updates a submodule.
func submoduleFilePatch(ctx context.Context, c *Change) (fdiff.FilePatch, error) {
fromContent := submoduleContent(c.From)
toContent := submoduleContent(c.To)
diffs := diff.Do(fromContent, toContent)
chunks := make([]fdiff.Chunk, 0, len(diffs))
for _, d := range diffs {
select {
case <-ctx.Done():
return nil, ErrCanceled
default:
}
var op fdiff.Operation
switch d.Type {
case dmp.DiffEqual:
op = fdiff.Equal
case dmp.DiffDelete:
op = fdiff.Delete
case dmp.DiffInsert:
op = fdiff.Add
}
chunks = append(chunks, &textChunk{d.Text, op})
}
return &textFilePatch{
chunks: chunks,
from: c.From,
to: c.To,
}, nil
}
func fileContent(f *File) (content string, isBinary bool, err error) {
if f == nil {
return content, isBinary, err
}
isBinary, err = f.IsBinary()
if err != nil || isBinary {
return content, isBinary, err
}
content, err = f.Contents()
return content, isBinary, err
}
// Patch is an implementation of fdiff.Patch interface
type Patch struct {
message string
filePatches []fdiff.FilePatch
}
// FilePatches returns the file patches.
func (p *Patch) FilePatches() []fdiff.FilePatch {
return p.filePatches
}
// Message returns the patch message.
func (p *Patch) Message() string {
return p.message
}
// Encode encodes the patch to the given writer.
func (p *Patch) Encode(w io.Writer) error {
ue := fdiff.NewUnifiedEncoder(w, fdiff.DefaultContextLines)
return ue.Encode(p)
}
// Stats returns the file stats.
func (p *Patch) Stats() FileStats {
return getFileStatsFromFilePatches(p.FilePatches())
}
func (p *Patch) String() string {
buf := bytes.NewBuffer(nil)
err := p.Encode(buf)
if err != nil {
return fmt.Sprintf("malformed patch: %s", err.Error())
}
return buf.String()
}
// changeEntryWrapper is an implementation of fdiff.File interface
type changeEntryWrapper struct {
ce ChangeEntry
}
func (f *changeEntryWrapper) Hash() plumbing.Hash {
if f.Empty() {
return plumbing.ZeroHash
}
return f.ce.TreeEntry.Hash
}
func (f *changeEntryWrapper) Mode() filemode.FileMode {
return f.ce.TreeEntry.Mode
}
func (f *changeEntryWrapper) Path() string {
if f.Empty() {
return ""
}
return f.ce.Name
}
func (f *changeEntryWrapper) Empty() bool {
// Submodules (gitlinks) are not files, but they still take part in a diff
// and thus must not be treated as empty entries.
return !f.ce.TreeEntry.Mode.IsFile() &&
f.ce.TreeEntry.Mode != filemode.Submodule
}
// textFilePatch is an implementation of fdiff.FilePatch interface
type textFilePatch struct {
chunks []fdiff.Chunk
from, to ChangeEntry
}
func (tf *textFilePatch) Files() (from, to fdiff.File) {
f := &changeEntryWrapper{tf.from}
t := &changeEntryWrapper{tf.to}
if !f.Empty() {
from = f
}
if !t.Empty() {
to = t
}
return from, to
}
func (tf *textFilePatch) IsBinary() bool {
return len(tf.chunks) == 0
}
func (tf *textFilePatch) Chunks() []fdiff.Chunk {
return tf.chunks
}
// textChunk is an implementation of fdiff.Chunk interface
type textChunk struct {
content string
op fdiff.Operation
}
func (t *textChunk) Content() string {
return t.content
}
func (t *textChunk) Type() fdiff.Operation {
return t.op
}
// FileStat stores the status of changes in content of a file.
type FileStat struct {
Name string
Addition int
Deletion int
}
func (fs FileStat) String() string {
return printStat([]FileStat{fs})
}
// FileStats is a collection of FileStat.
type FileStats []FileStat
func (fileStats FileStats) String() string {
return printStat(fileStats)
}
// printStat prints the stats of changes in content of files.
// Original implementation: https://github.com/git/git/blob/1a87c842ece327d03d08096395969aca5e0a6996/diff.c#L2615
// Parts of the output:
// <pad><filename><pad>|<pad><changeNumber><pad><+++/---><newline>
// example: " main.go | 10 +++++++--- "
func printStat(fileStats []FileStat) string {
maxGraphWidth := uint(53)
maxNameLen := 0
maxChangeLen := 0
scaleLinear := func(it, width, maxVal uint) uint {
if it == 0 || maxVal == 0 {
return 0
}
return 1 + (it * (width - 1) / maxVal)
}
for _, fs := range fileStats {
if len(fs.Name) > maxNameLen {
maxNameLen = len(fs.Name)
}
changes := strconv.Itoa(fs.Addition + fs.Deletion)
if len(changes) > maxChangeLen {
maxChangeLen = len(changes)
}
}
var result strings.Builder
for _, fs := range fileStats {
add := uint(fs.Addition)
del := uint(fs.Deletion)
np := maxNameLen - len(fs.Name)
cp := maxChangeLen - len(strconv.Itoa(fs.Addition+fs.Deletion))
total := add + del
if total > maxGraphWidth {
add = scaleLinear(add, maxGraphWidth, total)
del = scaleLinear(del, maxGraphWidth, total)
}
adds := strings.Repeat("+", int(add))
dels := strings.Repeat("-", int(del))
namePad := strings.Repeat(" ", np)
changePad := strings.Repeat(" ", cp)
fmt.Fprintf(&result, " %s%s | %s%d %s%s\n", fs.Name, namePad, changePad, total, adds, dels)
}
return result.String()
}
func getFileStatsFromFilePatches(filePatches []fdiff.FilePatch) FileStats {
fileStats := make(FileStats, 0, len(filePatches))
for _, fp := range filePatches {
// ignore empty patches (binary files, submodule refs updates)
if len(fp.Chunks()) == 0 {
continue
}
cs := FileStat{}
from, to := fp.Files()
switch {
case from == nil:
// New File is created.
cs.Name = to.Path()
case to == nil:
// File is deleted.
cs.Name = from.Path()
case from.Path() != to.Path():
// File is renamed.
cs.Name = fmt.Sprintf("%s => %s", from.Path(), to.Path())
default:
cs.Name = from.Path()
}
for _, chunk := range fp.Chunks() {
s := chunk.Content()
if len(s) == 0 {
continue
}
switch chunk.Type() {
case fdiff.Add:
cs.Addition += strings.Count(s, "\n")
if s[len(s)-1] != '\n' {
cs.Addition++
}
case fdiff.Delete:
cs.Deletion += strings.Count(s, "\n")
if s[len(s)-1] != '\n' {
cs.Deletion++
}
}
}
fileStats = append(fileStats, cs)
}
return fileStats
}
package object
import (
"errors"
"io"
"slices"
"sort"
"strings"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/filemode"
"github.com/go-git/go-git/v6/utils/ioutil"
"github.com/go-git/go-git/v6/utils/merkletrie"
)
// DetectRenames detects the renames in the given changes on two trees with
// the given options. It will return the given changes grouping additions and
// deletions into modifications when possible.
// If options is nil, the default diff tree options will be used.
func DetectRenames(
changes Changes,
opts *DiffTreeOptions,
) (Changes, error) {
if opts == nil {
opts = DefaultDiffTreeOptions
}
detector := &renameDetector{
renameScore: int(opts.RenameScore),
renameLimit: int(opts.RenameLimit),
onlyExact: opts.OnlyExactRenames,
}
for _, c := range changes {
action, err := c.Action()
if err != nil {
return nil, err
}
switch action {
case merkletrie.Insert:
detector.added = append(detector.added, c)
case merkletrie.Delete:
detector.deleted = append(detector.deleted, c)
default:
detector.modified = append(detector.modified, c)
}
}
return detector.detect()
}
// renameDetector will detect and resolve renames in a set of changes.
// see: https://github.com/eclipse/jgit/blob/master/org.eclipse.jgit/src/org/eclipse/jgit/diff/RenameDetector.java
type renameDetector struct {
added []*Change
deleted []*Change
modified []*Change
renameScore int
renameLimit int
onlyExact bool
}
// detectExactRenames detects matches files that were deleted with files that
// were added where the hash is the same on both. If there are multiple targets
// the one with the most similar path will be chosen as the rename and the
// rest as either deletions or additions.
func (d *renameDetector) detectExactRenames() {
added := groupChangesByHash(d.added)
deletes := groupChangesByHash(d.deleted)
var uniqueAdds []*Change
var nonUniqueAdds [][]*Change
var addedLeft []*Change
for _, cs := range added {
if len(cs) == 1 {
uniqueAdds = append(uniqueAdds, cs[0])
} else {
nonUniqueAdds = append(nonUniqueAdds, cs)
}
}
for _, c := range uniqueAdds {
hash := changeHash(c)
deleted := deletes[hash]
switch {
case len(deleted) == 1:
if sameMode(c, deleted[0]) {
d.modified = append(d.modified, &Change{From: deleted[0].From, To: c.To})
delete(deletes, hash)
} else {
addedLeft = append(addedLeft, c)
}
case len(deleted) > 1:
bestMatch := bestNameMatch(c, deleted)
if bestMatch != nil && sameMode(c, bestMatch) {
d.modified = append(d.modified, &Change{From: bestMatch.From, To: c.To})
delete(deletes, hash)
newDeletes := make([]*Change, 0, len(deleted)-1)
for _, d := range deleted {
if d != bestMatch {
newDeletes = append(newDeletes, d)
}
}
deletes[hash] = newDeletes
}
default:
addedLeft = append(addedLeft, c)
}
}
for _, added := range nonUniqueAdds {
hash := changeHash(added[0])
deleted := deletes[hash]
switch {
case len(deleted) == 1:
deleted := deleted[0]
bestMatch := bestNameMatch(deleted, added)
if bestMatch != nil && sameMode(deleted, bestMatch) {
d.modified = append(d.modified, &Change{From: deleted.From, To: bestMatch.To})
delete(deletes, hash)
for _, c := range added {
if c != bestMatch {
addedLeft = append(addedLeft, c)
}
}
} else {
addedLeft = append(addedLeft, added...)
}
case len(deleted) > 1:
maxSize := len(deleted) * len(added)
if d.renameLimit > 0 && d.renameLimit < maxSize {
maxSize = d.renameLimit
}
matrix := make(similarityMatrix, 0, maxSize)
for delIdx, del := range deleted {
deletedName := changeName(del)
for addIdx, add := range added {
addedName := changeName(add)
score := nameSimilarityScore(addedName, deletedName)
matrix = append(matrix, similarityPair{added: addIdx, deleted: delIdx, score: score})
if len(matrix) >= maxSize {
break
}
}
if len(matrix) >= maxSize {
break
}
}
sort.Stable(matrix)
usedAdds := make(map[*Change]struct{})
usedDeletes := make(map[*Change]struct{})
for _, m := range slices.Backward(matrix) {
del := deleted[m.deleted]
add := added[m.added]
if add == nil || del == nil {
// it was already matched
continue
}
usedAdds[add] = struct{}{}
usedDeletes[del] = struct{}{}
d.modified = append(d.modified, &Change{From: del.From, To: add.To})
added[m.added] = nil
deleted[m.deleted] = nil
}
for _, c := range added {
if _, ok := usedAdds[c]; !ok && c != nil {
addedLeft = append(addedLeft, c)
}
}
newDeletes := make([]*Change, 0, len(deleted)-len(usedDeletes))
for _, c := range deleted {
if _, ok := usedDeletes[c]; !ok && c != nil {
newDeletes = append(newDeletes, c)
}
}
deletes[hash] = newDeletes
default:
addedLeft = append(addedLeft, added...)
}
}
d.added = addedLeft
d.deleted = nil
for _, dels := range deletes {
d.deleted = append(d.deleted, dels...)
}
}
// detectContentRenames detects renames based on the similarity of the content
// in the files by building a matrix of pairs between sources and destinations
// and matching by the highest score.
// see: https://github.com/eclipse/jgit/blob/master/org.eclipse.jgit/src/org/eclipse/jgit/diff/SimilarityRenameDetector.java
func (d *renameDetector) detectContentRenames() error {
cnt := max(len(d.added), len(d.deleted))
if d.renameLimit > 0 && cnt > d.renameLimit {
return nil
}
srcs, dsts := d.deleted, d.added
matrix, err := buildSimilarityMatrix(srcs, dsts, d.renameScore)
if err != nil {
return err
}
renames := make([]*Change, 0, min(len(matrix), len(dsts)))
// Match rename pairs on a first come, first serve basis until
// we have looked at everything that is above the minimum score.
for _, pair := range slices.Backward(matrix) {
src := srcs[pair.deleted]
dst := dsts[pair.added]
if dst == nil || src == nil {
// It was already matched before
continue
}
renames = append(renames, &Change{From: src.From, To: dst.To})
// Claim destination and source as matched
dsts[pair.added] = nil
srcs[pair.deleted] = nil
}
d.modified = append(d.modified, renames...)
d.added = compactChanges(dsts)
d.deleted = compactChanges(srcs)
return nil
}
func (d *renameDetector) detect() (Changes, error) {
if len(d.added) > 0 && len(d.deleted) > 0 {
d.detectExactRenames()
if !d.onlyExact {
if err := d.detectContentRenames(); err != nil {
return nil, err
}
}
}
result := make(Changes, 0, len(d.added)+len(d.deleted)+len(d.modified))
result = append(result, d.added...)
result = append(result, d.deleted...)
result = append(result, d.modified...)
sort.Stable(result)
return result, nil
}
func bestNameMatch(change *Change, changes []*Change) *Change {
var best *Change
var bestScore int
cname := changeName(change)
for _, c := range changes {
score := nameSimilarityScore(cname, changeName(c))
if score > bestScore {
bestScore = score
best = c
}
}
return best
}
func nameSimilarityScore(a, b string) int {
aDirLen := strings.LastIndexByte(a, '/') + 1
bDirLen := strings.LastIndexByte(b, '/') + 1
dirMin := min(aDirLen, bDirLen)
dirMax := max(aDirLen, bDirLen)
var dirScoreLtr, dirScoreRtl int
if dirMax == 0 {
dirScoreLtr = 100
dirScoreRtl = 100
} else {
var dirSim int
for ; dirSim < dirMin; dirSim++ {
if a[dirSim] != b[dirSim] {
break
}
}
dirScoreLtr = dirSim * 100 / dirMax
if dirScoreLtr == 100 {
dirScoreRtl = 100
} else {
for dirSim = 0; dirSim < dirMin; dirSim++ {
if a[aDirLen-1-dirSim] != b[bDirLen-1-dirSim] {
break
}
}
dirScoreRtl = dirSim * 100 / dirMax
}
}
fileMin := min(len(a)-aDirLen, len(b)-bDirLen)
fileMax := max(len(a)-aDirLen, len(b)-bDirLen)
fileSim := 0
for ; fileSim < fileMin; fileSim++ {
if a[len(a)-1-fileSim] != b[len(b)-1-fileSim] {
break
}
}
fileScore := fileSim * 100 / fileMax
return (((dirScoreLtr + dirScoreRtl) * 25) + (fileScore * 50)) / 100
}
func changeName(c *Change) string {
if c.To != empty {
return c.To.Name
}
return c.From.Name
}
func changeHash(c *Change) plumbing.Hash {
if c.To != empty {
return c.To.TreeEntry.Hash
}
return c.From.TreeEntry.Hash
}
func changeMode(c *Change) filemode.FileMode {
if c.To != empty {
return c.To.TreeEntry.Mode
}
return c.From.TreeEntry.Mode
}
func sameMode(a, b *Change) bool {
return changeMode(a) == changeMode(b)
}
func groupChangesByHash(changes []*Change) map[plumbing.Hash][]*Change {
result := make(map[plumbing.Hash][]*Change)
for _, c := range changes {
hash := changeHash(c)
result[hash] = append(result[hash], c)
}
return result
}
type similarityMatrix []similarityPair
func (m similarityMatrix) Len() int { return len(m) }
func (m similarityMatrix) Swap(i, j int) { m[i], m[j] = m[j], m[i] }
func (m similarityMatrix) Less(i, j int) bool {
if m[i].score == m[j].score {
if m[i].added == m[j].added {
return m[i].deleted < m[j].deleted
}
return m[i].added < m[j].added
}
return m[i].score < m[j].score
}
type similarityPair struct {
// index of the added file
added int
// index of the deleted file
deleted int
// similarity score
score int
}
const maxMatrixSize = 10000
func buildSimilarityMatrix(srcs, dsts []*Change, renameScore int) (similarityMatrix, error) {
// Allocate for the worst-case scenario where every pair has a score
// that we need to consider. We might not need that many.
matrixSize := min(len(srcs)*len(dsts), maxMatrixSize)
matrix := make(similarityMatrix, 0, matrixSize)
srcSizes := make([]int64, len(srcs))
dstSizes := make([]int64, len(dsts))
dstTooLarge := make(map[int]bool)
// Consider each pair of files, if the score is above the minimum
// threshold we need to record that scoring in the matrix so we can
// later find the best matches.
outerLoop:
for srcIdx, src := range srcs {
if changeMode(src) != filemode.Regular {
continue
}
// Declare the from file and the similarity index here to be able to
// reuse it inside the inner loop. The reason to not initialize them
// here is so we can skip the initialization in case they happen to
// not be needed later. They will be initialized inside the inner
// loop if and only if they're needed and reused in subsequent passes.
var from *File
var s *similarityIndex
var err error
for dstIdx, dst := range dsts {
if changeMode(dst) != filemode.Regular {
continue
}
if dstTooLarge[dstIdx] {
continue
}
var to *File
srcSize := srcSizes[srcIdx]
if srcSize == 0 {
from, _, err = src.Files()
if err != nil {
return nil, err
}
srcSize = from.Size + 1
srcSizes[srcIdx] = srcSize
}
dstSize := dstSizes[dstIdx]
if dstSize == 0 {
_, to, err = dst.Files()
if err != nil {
return nil, err
}
dstSize = to.Size + 1
dstSizes[dstIdx] = dstSize
}
minSize, maxSize := srcSize, dstSize
if dstSize < srcSize {
minSize, maxSize = dstSize, srcSize
}
if int(minSize*100/maxSize) < renameScore {
// File sizes are too different to be a match
continue
}
if s == nil {
s, err = fileSimilarityIndex(from)
if err != nil {
if errors.Is(err, errIndexFull) {
continue outerLoop
}
return nil, err
}
}
if to == nil {
_, to, err = dst.Files()
if err != nil {
return nil, err
}
}
di, err := fileSimilarityIndex(to)
if err != nil {
if errors.Is(err, errIndexFull) {
dstTooLarge[dstIdx] = true
}
return nil, err
}
contentScore := s.score(di, 10000)
// The name score returns a value between 0 and 100, so we need to
// convert it to the same range as the content score.
nameScore := nameSimilarityScore(src.From.Name, dst.To.Name) * 100
score := (contentScore*99 + nameScore*1) / 10000
if score < renameScore {
continue
}
matrix = append(matrix, similarityPair{added: dstIdx, deleted: srcIdx, score: score})
}
}
sort.Stable(matrix)
return matrix, nil
}
func compactChanges(changes []*Change) []*Change {
var result []*Change
for _, c := range changes {
if c != nil {
result = append(result, c)
}
}
return result
}
const (
keyShift = 32
maxCountValue = (1 << keyShift) - 1
)
var errIndexFull = errors.New("index is full")
// similarityIndex is an index structure of lines/blocks in one file.
// This structure can be used to compute an approximation of the similarity
// between two files.
// To save space in memory, this index uses a space efficient encoding which
// will not exceed 1MiB per instance. The index starts out at a smaller size
// (closer to 2KiB), but may grow as more distinct blocks within the scanned
// file are discovered.
// see: https://github.com/eclipse/jgit/blob/master/org.eclipse.jgit/src/org/eclipse/jgit/diff/SimilarityIndex.java
type similarityIndex struct {
hashed uint64
// number of non-zero entries in hashes
numHashes int
growAt int
hashes []keyCountPair
hashBits int
}
func fileSimilarityIndex(f *File) (*similarityIndex, error) {
idx := newSimilarityIndex()
if err := idx.hash(f); err != nil {
return nil, err
}
sort.Stable(keyCountPairs(idx.hashes))
return idx, nil
}
func newSimilarityIndex() *similarityIndex {
return &similarityIndex{
hashBits: 8,
hashes: make([]keyCountPair, 1<<8),
growAt: shouldGrowAt(8),
}
}
func (i *similarityIndex) hash(f *File) error {
isBin, err := f.IsBinary()
if err != nil {
return err
}
r, err := f.Reader()
if err != nil {
return err
}
defer ioutil.CheckClose(r, &err)
return i.hashContent(r, f.Size, isBin)
}
func (i *similarityIndex) hashContent(r io.Reader, size int64, isBin bool) error {
buf := make([]byte, 4096)
var ptr, cnt int
remaining := size
for 0 < remaining {
hash := 5381
var blockHashedCnt uint64
// Hash one line or block, whatever happens first
n := int64(0)
for {
if ptr == cnt {
ptr = 0
var err error
cnt, err = io.ReadFull(r, buf)
if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) {
return err
}
if cnt == 0 {
return io.EOF
}
}
n++
c := buf[ptr] & 0xff
ptr++
// Ignore CR in CRLF sequence if it's text
if !isBin && c == '\r' && ptr < cnt && buf[ptr] == '\n' {
continue
}
blockHashedCnt++
if c == '\n' {
break
}
hash = (hash << 5) + hash + int(c)
if n >= 64 || n >= remaining {
break
}
}
i.hashed += blockHashedCnt
if err := i.add(hash, blockHashedCnt); err != nil {
return err
}
remaining -= n
}
return nil
}
// score computes the similarity score between this index and another one.
// A region of a file is defined as a line in a text file or a fixed-size
// block in a binary file. To prepare an index, each region in the file is
// hashed; the values and counts of hashes are retained in a sorted table.
// Define the similarity fraction F as the count of matching regions between
// the two files divided between the maximum count of regions in either file.
// The similarity score is F multiplied by the maxScore constant, yielding a
// range [0, maxScore]. It is defined as maxScore for the degenerate case of
// two empty files.
// The similarity score is symmetrical; i.e. a.score(b) == b.score(a).
func (i *similarityIndex) score(other *similarityIndex, maxScore int) int {
maxHashed := max(i.hashed, other.hashed)
if maxHashed == 0 {
return maxScore
}
return int(i.common(other) * uint64(maxScore) / maxHashed)
}
func (i *similarityIndex) common(dst *similarityIndex) uint64 {
srcIdx, dstIdx := 0, 0
if i.numHashes == 0 || dst.numHashes == 0 {
return 0
}
var common uint64
srcKey, dstKey := i.hashes[srcIdx].key(), dst.hashes[dstIdx].key()
mainLoop:
for {
switch {
case srcKey == dstKey:
srcCnt, dstCnt := i.hashes[srcIdx].count(), dst.hashes[dstIdx].count()
if srcCnt < dstCnt {
common += srcCnt
} else {
common += dstCnt
}
srcIdx++
if srcIdx == len(i.hashes) {
break mainLoop
}
srcKey = i.hashes[srcIdx].key()
dstIdx++
if dstIdx == len(dst.hashes) {
break mainLoop
}
dstKey = dst.hashes[dstIdx].key()
case srcKey < dstKey:
// Region of src that is not in dst
srcIdx++
if srcIdx == len(i.hashes) {
break mainLoop
}
srcKey = i.hashes[srcIdx].key()
default:
// Region of dst that is not in src
dstIdx++
if dstIdx == len(dst.hashes) {
break mainLoop
}
dstKey = dst.hashes[dstIdx].key()
}
}
return common
}
func (i *similarityIndex) add(key int, cnt uint64) error {
key = int(uint32(key) * 0x9e370001 >> 1)
j := i.slot(key)
for {
v := i.hashes[j]
switch {
case v == 0:
// It's an empty slot, so we can store it here.
if i.growAt <= i.numHashes {
if err := i.grow(); err != nil {
return err
}
j = i.slot(key)
continue
}
var err error
i.hashes[j], err = newKeyCountPair(key, cnt)
if err != nil {
return err
}
i.numHashes++
return nil
case v.key() == key:
// It's the same key, so increment the counter.
var err error
i.hashes[j], err = newKeyCountPair(key, v.count()+cnt)
return err
case j+1 >= len(i.hashes):
j = 0
default:
j++
}
}
}
type keyCountPair uint64
func newKeyCountPair(key int, cnt uint64) (keyCountPair, error) {
if cnt > maxCountValue {
return 0, errIndexFull
}
return keyCountPair((uint64(key) << keyShift) | cnt), nil
}
func (p keyCountPair) key() int {
return int(p >> keyShift)
}
func (p keyCountPair) count() uint64 {
return uint64(p) & maxCountValue
}
func (i *similarityIndex) slot(key int) int {
// We use 31 - hashBits because the upper bit was already forced
// to be 0 and we want the remaining high bits to be used as the
// table slot.
return int(uint32(key) >> uint(31-i.hashBits))
}
func shouldGrowAt(hashBits int) int {
return (1 << uint(hashBits)) * (hashBits - 3) / hashBits
}
func (i *similarityIndex) grow() error {
if i.hashBits == 30 {
return errIndexFull
}
old := i.hashes
i.hashBits++
i.growAt = shouldGrowAt(i.hashBits)
// TODO(erizocosmico): find a way to check if it will OOM and return
// errIndexFull instead.
i.hashes = make([]keyCountPair, 1<<uint(i.hashBits))
for _, v := range old {
if v != 0 {
j := i.slot(v.key())
for i.hashes[j] != 0 {
j++
if j >= len(i.hashes) {
j = 0
}
}
i.hashes[j] = v
}
}
return nil
}
type keyCountPairs []keyCountPair
func (p keyCountPairs) Len() int { return len(p) }
func (p keyCountPairs) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p keyCountPairs) Less(i, j int) bool { return p[i] < p[j] }
package object
import (
"bytes"
"io"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/utils/ioutil"
"github.com/go-git/go-git/v6/utils/sync"
)
const (
signatureTypeUnknown signatureType = iota
signatureTypeOpenPGP
signatureTypeX509
signatureTypeSSH
)
var (
// openPGPSignatureFormat is the format of an OpenPGP signature.
openPGPSignatureFormat = signatureFormat{
[]byte("-----BEGIN PGP SIGNATURE-----"),
[]byte("-----BEGIN PGP MESSAGE-----"),
}
// x509SignatureFormat is the format of an X509 signature, which is
// a PKCS#7 (S/MIME) signature.
x509SignatureFormat = signatureFormat{
[]byte("-----BEGIN SIGNED MESSAGE-----"),
}
// sshSignatureFormat is the format of an SSH signature.
sshSignatureFormat = signatureFormat{
[]byte("-----BEGIN SSH SIGNATURE-----"),
}
)
// knownSignatureFormats is a map of known signature formats, indexed by
// their signatureType.
var knownSignatureFormats = map[signatureType]signatureFormat{
signatureTypeOpenPGP: openPGPSignatureFormat,
signatureTypeX509: x509SignatureFormat,
signatureTypeSSH: sshSignatureFormat,
}
// signatureType represents the type of the signature.
type signatureType int8
// signatureFormat represents the beginning of a signature.
type signatureFormat [][]byte
// typeForSignature returns the type of the signature based on its format.
func typeForSignature(b []byte) signatureType {
for t, i := range knownSignatureFormats {
for _, begin := range i {
if bytes.HasPrefix(b, begin) {
return t
}
}
}
return signatureTypeUnknown
}
// parseSignedBytes returns the position of the last signature block found in
// the given bytes. If no signature block is found, it returns -1.
//
// When multiple signature blocks are found, the position of the last one is
// returned. Any tailing bytes after this signature block start should be
// considered part of the signature.
//
// Given this, it would be safe to use the returned position to split the bytes
// into two parts: the first part containing the message, the second part
// containing the signature.
//
// Example:
//
// message := []byte(`Message with signature
//
// -----BEGIN SSH SIGNATURE-----
// ...`)
//
// var signature string
// if pos, _ := parseSignedBytes(message); pos != -1 {
// signature = string(message[pos:])
// message = message[:pos]
// }
//
// This logic is on par with git's gpg-interface.c:parse_signed_buffer().
// https://github.com/git/git/blob/7c2ef319c52c4997256f5807564523dfd4acdfc7/gpg-interface.c#L668
func parseSignedBytes(b []byte) (int, signatureType) {
n, match := 0, -1
var t signatureType
for n < len(b) {
i := b[n:]
if st := typeForSignature(i); st != signatureTypeUnknown {
match = n
t = st
}
if eol := bytes.IndexByte(i, '\n'); eol >= 0 {
n += eol + 1
continue
}
// If we reach this point, we've reached the end.
break
}
return match, t
}
// countSignatureBlocks reports how many distinct armored signature blocks
// start at a line boundary in b. Used by verification paths to reject
// multi-signature payloads, matching upstream's check in gpg-interface.c
// where parse_gpg_output bails out the first time it sees a second
// exclusive status line (a second GOODSIG/BADSIG/etc.).
func countSignatureBlocks(b []byte) int {
n, count := 0, 0
for n < len(b) {
i := b[n:]
if typeForSignature(i) != signatureTypeUnknown {
count++
}
if eol := bytes.IndexByte(i, '\n'); eol >= 0 {
n += eol + 1
continue
}
break
}
return count
}
// isSignatureHeader reports whether line is a canonical "gpgsig "/
// "gpgsig-sha256 " header line. Other "gpgsig"-prefixed extra headers
// are intentionally not matched.
func isSignatureHeader(line []byte) bool {
return bytes.HasPrefix(line, []byte(headerpgp+" ")) ||
bytes.HasPrefix(line, []byte(headerpgp256+" "))
}
// stripObjectSignatures streams src into dst, producing the byte sequence
// over which a PGP/GPG signature is computed:
//
// - Canonical "gpgsig" and "gpgsig-sha256" headers (and their
// continuation lines) are dropped, mirroring upstream's
// remove_signature in commit.c.
// - For tag objects, the inline trailing PGP signature is additionally
// truncated, mirroring upstream's parse_signature in gpg-interface.c
// used by gpg_verify_tag.
//
// The returned object's type is set to objType. Used by both
// Commit.EncodeWithoutSignature and Tag.EncodeWithoutSignature to
// reproduce the exact bytes the signature was computed over.
func stripObjectSignatures(dst, src plumbing.EncodedObject, objType plumbing.ObjectType) (err error) {
dst.SetType(objType)
r, err := src.Reader()
if err != nil {
return err
}
defer ioutil.CheckClose(r, &err)
var input io.Reader = r
if objType == plumbing.TagObject {
raw, err := io.ReadAll(r)
if err != nil {
return err
}
if sm, _ := parseSignedBytes(raw); sm >= 0 {
raw = raw[:sm]
}
input = bytes.NewReader(raw)
}
w, err := dst.Writer()
if err != nil {
return err
}
defer ioutil.CheckClose(w, &err)
return stripHeaderSignatures(w, input)
}
// stripHeaderSignatures copies r to w, dropping canonical signature header
// lines (gpgsig and gpgsig-sha256) and their continuation lines. Lines
// past the blank line that closes the header block are copied verbatim.
func stripHeaderSignatures(w io.Writer, r io.Reader) error {
br := sync.GetBufioReader(r)
defer sync.PutBufioReader(br)
var inBody, skipping bool
for {
line, rerr := br.ReadBytes('\n')
if rerr != nil && rerr != io.EOF {
return rerr
}
write := true
if !inBody {
switch {
case skipping && len(line) > 0 && line[0] == ' ':
write = false
case isSignatureHeader(line):
skipping = true
write = false
case len(line) == 1 && line[0] == '\n':
skipping = false
inBody = true
default:
skipping = false
}
}
if write && len(line) > 0 {
if _, werr := w.Write(line); werr != nil {
return werr
}
}
if rerr == io.EOF {
return nil
}
}
}
package object
import (
"errors"
"fmt"
"strings"
"github.com/ProtonMail/go-crypto/openpgp"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/storer"
"github.com/go-git/go-git/v6/utils/ioutil"
"github.com/go-git/go-git/v6/utils/sync"
)
// ErrMalformedTag is returned when a tag object cannot be decoded because
// its required headers (object, type, tag) are missing or out of order.
var ErrMalformedTag = errors.New("malformed tag")
// Tag represents an annotated tag object. It points to a single git object of
// any type, but tags typically are applied to commit or blob objects. It
// provides a reference that associates the target with a tag name. It also
// contains meta-information about the tag, including the tagger, tag date and
// message.
//
// Note that this is not used for lightweight tags.
//
// https://git-scm.com/book/en/v2/Git-Internals-Git-References#Tags
type Tag struct {
// Hash of the tag.
Hash plumbing.Hash
// Name of the tag.
Name string
// Tagger is the one who created the tag.
Tagger Signature
// Message is an arbitrary text message.
Message string
// Signature is the cryptographic signature appended after the message
// body. This is the canonical tag signature in upstream Git.
Signature string
// SignatureSHA256 is the cryptographic signature stored under the
// "gpgsig-sha256" header.
SignatureSHA256 string
// TargetType is the object type of the target.
TargetType plumbing.ObjectType
// Target is the hash of the target object.
Target plumbing.Hash
s storer.EncodedObjectStorer
// src holds the encoded object this Tag was decoded from, used by
// EncodeWithoutSignature to recover the canonical signed bytes.
src plumbing.EncodedObject
}
// GetTag gets a tag from an object storer and decodes it.
func GetTag(s storer.EncodedObjectStorer, h plumbing.Hash) (*Tag, error) {
o, err := s.EncodedObject(plumbing.TagObject, h)
if err != nil {
return nil, err
}
return DecodeTag(s, o)
}
// DecodeTag decodes an encoded object into a *Commit and associates it to the
// given object storer.
func DecodeTag(s storer.EncodedObjectStorer, o plumbing.EncodedObject) (*Tag, error) {
t := &Tag{s: s}
if err := t.Decode(o); err != nil {
return nil, err
}
return t, nil
}
// ID returns the object ID of the tag, not the object that the tag references.
// The returned value will always match the current value of Tag.Hash.
//
// ID is present to fulfill the Object interface.
func (t *Tag) ID() plumbing.Hash {
return t.Hash
}
// Type returns the type of object. It always returns plumbing.TagObject.
//
// Type is present to fulfill the Object interface.
func (t *Tag) Type() plumbing.ObjectType {
return plumbing.TagObject
}
func (t *Tag) reset() {
storer := t.s
*t = Tag{s: storer}
}
// Decode transforms a plumbing.EncodedObject into a Tag struct.
func (t *Tag) Decode(o plumbing.EncodedObject) (err error) {
if o.Type() != plumbing.TagObject {
return ErrUnsupportedObject
}
t.reset()
t.Hash = o.Hash()
t.src = o
reader, err := o.Reader()
if err != nil {
return err
}
defer ioutil.CheckClose(reader, &err)
r := sync.GetBufioReader(reader)
defer sync.PutBufioReader(r)
s := &tagScanner{r: r, t: t}
for state := scanTagObject; state != nil; {
state, err = state(s)
if err != nil {
return err
}
}
data := s.msgbuf.Bytes()
if sm, _ := parseSignedBytes(data); sm >= 0 {
t.Signature = string(data[sm:])
data = data[:sm]
}
t.Message = string(data)
return nil
}
// Encode transforms a Tag into a plumbing.EncodedObject.
func (t *Tag) Encode(o plumbing.EncodedObject) error {
return t.encode(o, true)
}
// EncodeWithoutSignature exports a Tag into a plumbing.EncodedObject without
// any signature data, producing the payload that PGP/GPG signatures are
// computed over.
//
// Behaviour mirrors Commit.EncodeWithoutSignature:
//
// - For Tags populated by Decode whose exported fields still match the
// source object, the payload is streamed from the raw source bytes with
// the inline trailing signature truncated and gpgsig/gpgsig-sha256
// headers (and their continuation lines) stripped verbatim. This
// preserves the exact bytes the signature was computed over, regardless
// of any normalization performed by Decode.
//
// - For Tags constructed in memory, or for decoded Tags whose exported
// fields have been mutated, the payload is derived from the current
// struct fields. Mutation is detected by re-decoding the source object
// and comparing exported fields; if any differ, the in-memory
// representation prevails.
func (t *Tag) EncodeWithoutSignature(o plumbing.EncodedObject) error {
if t.matchesSource() {
return stripObjectSignatures(o, t.src, plumbing.TagObject)
}
return t.encode(o, false)
}
// matchesSource reports whether t.src is set and re-decoding it produces a
// Tag whose payload-affecting exported fields are identical to those of t.
//
// Signature and SignatureSHA256 are intentionally excluded from the
// comparison: neither path emits them as part of the verification payload,
// so mutating them must not trigger a switch to struct-encode (which would
// change the byte layout the caller is trying to verify against).
func (t *Tag) matchesSource() bool {
if t.src == nil {
return false
}
fresh := &Tag{}
if err := fresh.Decode(t.src); err != nil {
return false
}
return t.Hash == fresh.Hash &&
t.Name == fresh.Name &&
signatureEqual(t.Tagger, fresh.Tagger) &&
t.Message == fresh.Message &&
t.TargetType == fresh.TargetType &&
t.Target == fresh.Target
}
func (t *Tag) encode(o plumbing.EncodedObject, includeSig bool) (err error) {
o.SetType(plumbing.TagObject)
w, err := o.Writer()
if err != nil {
return err
}
defer ioutil.CheckClose(w, &err)
if _, err = fmt.Fprintf(w,
"object %s\ntype %s\ntag %s\n",
t.Target.String(), t.TargetType.Bytes(), t.Name); err != nil {
return err
}
if !isZeroSignature(t.Tagger) {
if _, err = fmt.Fprint(w, "tagger "); err != nil {
return err
}
if err = t.Tagger.Encode(w); err != nil {
return err
}
if _, err = fmt.Fprint(w, "\n"); err != nil {
return err
}
}
// gpgsig-sha256 is emitted between the tagger line and the blank line
// that separates headers from the body, matching upstream's
// add_header_signature insertion point (commit.c:1142-1171), which
// builtin/tag.c:do_sign reuses when signing tags in compat mode.
if t.SignatureSHA256 != "" && includeSig {
if _, err = fmt.Fprint(w, headerpgp256+" "); err != nil {
return err
}
sig := strings.TrimSuffix(t.SignatureSHA256, "\n")
lines := strings.Split(sig, "\n")
if _, err = fmt.Fprint(w, strings.Join(lines, "\n ")); err != nil {
return err
}
if _, err = fmt.Fprint(w, "\n"); err != nil {
return err
}
}
if _, err = fmt.Fprint(w, "\n"); err != nil {
return err
}
if _, err = fmt.Fprint(w, t.Message); err != nil {
return err
}
// Note that this is highly sensitive to what is sent along in the
// message. Message *always* needs to end with a newline, or else the
// message and the trailing signature will be concatenated into a
// corrupt object. Since this is a lower-level method, we assume you
// know what you are doing and have already done the needful on the
// message in the caller.
if includeSig {
if _, err = fmt.Fprint(w, t.Signature); err != nil {
return err
}
}
return err
}
func isZeroSignature(s Signature) bool {
return s.Name == "" && s.Email == "" && s.When.IsZero()
}
// Commit returns the commit pointed to by the tag. If the tag points to a
// different type of object ErrUnsupportedObject will be returned.
func (t *Tag) Commit() (*Commit, error) {
if t.TargetType != plumbing.CommitObject {
return nil, ErrUnsupportedObject
}
o, err := t.s.EncodedObject(plumbing.CommitObject, t.Target)
if err != nil {
return nil, err
}
return DecodeCommit(t.s, o)
}
// Tree returns the tree pointed to by the tag. If the tag points to a commit
// object the tree of that commit will be returned. If the tag does not point
// to a commit or tree object ErrUnsupportedObject will be returned.
func (t *Tag) Tree() (*Tree, error) {
switch t.TargetType {
case plumbing.CommitObject:
c, err := t.Commit()
if err != nil {
return nil, err
}
return c.Tree()
case plumbing.TreeObject:
return GetTree(t.s, t.Target)
default:
return nil, ErrUnsupportedObject
}
}
// Blob returns the blob pointed to by the tag. If the tag points to a
// different type of object ErrUnsupportedObject will be returned.
func (t *Tag) Blob() (*Blob, error) {
if t.TargetType != plumbing.BlobObject {
return nil, ErrUnsupportedObject
}
return GetBlob(t.s, t.Target)
}
// Object returns the object pointed to by the tag.
func (t *Tag) Object() (Object, error) {
o, err := t.s.EncodedObject(t.TargetType, t.Target)
if err != nil {
return nil, err
}
return DecodeObject(t.s, o)
}
// String returns the meta information contained in the tag as a formatted
// string.
func (t *Tag) String() string {
obj, _ := t.Object()
return fmt.Sprintf(
"%s %s\nTagger: %s\nDate: %s\n\n%s\n%s",
plumbing.TagObject, t.Name, t.Tagger.String(), t.Tagger.When.Format(DateFormat),
t.Message, objectAsString(obj),
)
}
// Verify performs PGP verification of the tag with a provided armored
// keyring and returns openpgp.Entity associated with verifying key on
// success.
func (t *Tag) Verify(armoredKeyRing string) (*openpgp.Entity, error) {
keyRingReader := strings.NewReader(armoredKeyRing)
keyring, err := openpgp.ReadArmoredKeyRing(keyRingReader)
if err != nil {
return nil, err
}
// Extract signature.
signature := strings.NewReader(t.Signature)
encoded := &plumbing.MemoryObject{}
// Encode tag components, excluding signature and get a reader object.
if err := t.EncodeWithoutSignature(encoded); err != nil {
return nil, err
}
er, err := encoded.Reader()
if err != nil {
return nil, err
}
return openpgp.CheckArmoredDetachedSignature(keyring, er, signature, nil)
}
// TagIter provides an iterator for a set of tags.
type TagIter struct {
storer.EncodedObjectIter
s storer.EncodedObjectStorer
}
// NewTagIter takes a storer.EncodedObjectStorer and a
// storer.EncodedObjectIter and returns a *TagIter that iterates over all
// tags contained in the storer.EncodedObjectIter.
//
// Any non-tag object returned by the storer.EncodedObjectIter is skipped.
func NewTagIter(s storer.EncodedObjectStorer, iter storer.EncodedObjectIter) *TagIter {
return &TagIter{iter, s}
}
// Next moves the iterator to the next tag and returns a pointer to it. If
// there are no more tags, it returns io.EOF.
func (iter *TagIter) Next() (*Tag, error) {
obj, err := iter.EncodedObjectIter.Next()
if err != nil {
return nil, err
}
return DecodeTag(iter.s, obj)
}
// ForEach call the cb function for each tag contained on this iter until
// an error happens or the end of the iter is reached. If ErrStop is sent
// the iteration is stop but no error is returned. The iterator is closed.
func (iter *TagIter) ForEach(cb func(*Tag) error) error {
return iter.EncodedObjectIter.ForEach(func(obj plumbing.EncodedObject) error {
t, err := DecodeTag(iter.s, obj)
if err != nil {
return err
}
return cb(t)
})
}
func objectAsString(obj Object) string {
switch o := obj.(type) {
case *Commit:
return o.String()
default:
return ""
}
}
package object
import (
"bufio"
"bytes"
"fmt"
"io"
"github.com/go-git/go-git/v6/plumbing"
)
// tagScanner holds the working state of the tag decoder driven by the
// stateFn loop in (*Tag).Decode. Each tagState reads one or more lines
// from r, updates the in-progress *Tag and the scanner's bookkeeping,
// and returns the state that should run next (or nil to stop).
type tagScanner struct {
r *bufio.Reader
t *Tag
msgbuf bytes.Buffer
// pending holds a line that was read but the current state decided to
// hand back to the next state, paired with the io.EOF flag returned
// when the line was originally read.
pending []byte
pendingErr error
// First-occurrence tracking — once the corresponding canonical
// header has been decoded at its expected position, subsequent
// occurrences (or out-of-position lines) are silently dropped,
// matching the strict layout enforced by upstream's
// parse_tag_buffer (tag.c:130).
//
// gpgsig-sha256 is NOT tracked here: upstream's
// parse_buffer_signed_by_header (commit.c:1186) accumulates every
// occurrence into one signature buffer, so we do the same.
sawObject, sawType, sawName, sawTagger bool
}
// tagState is one step of the decoder state machine. Each function reads
// the lines it needs, mutates *Tag via s.t, and returns the next state
// to run (or nil to terminate the loop).
type tagState func(*tagScanner) (tagState, error)
// readLine returns the next line from the buffer, transparently
// consuming any line that was previously pushed back by a state that
// decided not to handle it.
func (s *tagScanner) readLine() ([]byte, error) {
if s.pending != nil {
line, err := s.pending, s.pendingErr
s.pending, s.pendingErr = nil, nil
return line, err
}
return s.r.ReadBytes('\n')
}
// pushBack stashes an unconsumed line so the next state's readLine call
// sees it. Only one line can be pushed back at a time.
func (s *tagScanner) pushBack(line []byte, err error) {
s.pending = line
s.pendingErr = err
}
// scanTagObject requires the first line to be `object HASH`, mirroring
// upstream's strict parse_tag_buffer (tag.c:151-156). Anything else
// returns ErrMalformedTag.
func scanTagObject(s *tagScanner) (tagState, error) {
line, err := s.readLine()
if err != nil && err != io.EOF {
return nil, err
}
if len(line) == 0 || isBlankLine(line) {
return nil, fmt.Errorf("%w: missing object header", ErrMalformedTag)
}
key, data := splitHeader(line)
if key != "object" {
return nil, fmt.Errorf("%w: object header must be first", ErrMalformedTag)
}
h, herr := parseObjectIDHex(data, ErrMalformedTag, "object")
if herr != nil {
return nil, herr
}
s.t.Target = h
s.sawObject = true
if err == io.EOF {
return nil, nil
}
return scanTagType, nil
}
// scanTagType requires a `type` line immediately after the object header,
// mirroring upstream's parse_tag_buffer (tag.c:158-166).
func scanTagType(s *tagScanner) (tagState, error) {
line, err := s.readLine()
if err != nil && err != io.EOF {
return nil, err
}
if len(line) == 0 || isBlankLine(line) {
return nil, fmt.Errorf("%w: missing type header", ErrMalformedTag)
}
key, data := splitHeader(line)
if key != "type" {
return nil, fmt.Errorf("%w: type header must follow object", ErrMalformedTag)
}
ot, perr := plumbing.ParseObjectType(string(data))
if perr != nil {
return nil, perr
}
s.t.TargetType = ot
s.sawType = true
if err == io.EOF {
return nil, nil
}
return scanTagName, nil
}
// scanTagName requires a `tag` line immediately after the type header,
// mirroring upstream's parse_tag_buffer (tag.c:186-194).
func scanTagName(s *tagScanner) (tagState, error) {
line, err := s.readLine()
if err != nil && err != io.EOF {
return nil, err
}
if len(line) == 0 || isBlankLine(line) {
return nil, fmt.Errorf("%w: missing tag header", ErrMalformedTag)
}
key, data := splitHeader(line)
if key != "tag" {
return nil, fmt.Errorf("%w: tag header must follow type", ErrMalformedTag)
}
s.t.Name = string(data)
s.sawName = true
if err == io.EOF {
return nil, nil
}
return scanTagTagger, nil
}
// scanTagTagger accepts a `tagger` line at its canonical position. Any
// other header is pushed back for scanTagHeaders.
func scanTagTagger(s *tagScanner) (tagState, error) {
line, err := s.readLine()
if err != nil && err != io.EOF {
return nil, err
}
if len(line) == 0 {
return nil, nil
}
if isBlankLine(line) {
return scanTagMessage, nil
}
key, data := splitHeader(line)
if key == "tagger" {
s.t.Tagger.Decode(data)
s.sawTagger = true
if err == io.EOF {
return nil, nil
}
return scanTagHeaders, nil
}
s.pushBack(line, err)
return scanTagHeaders, nil
}
// scanTagHeaders dispatches one header line. gpgsig-sha256 hands off to
// scanTagPgp256Cont so the continuation block can be consumed; out-of-
// canonical-position fields and unknown headers are silently dropped.
func scanTagHeaders(s *tagScanner) (tagState, error) {
line, err := s.readLine()
if err != nil && err != io.EOF {
return nil, err
}
if len(line) == 0 {
return nil, nil
}
if isBlankLine(line) {
return scanTagMessage, nil
}
key, data := splitHeader(line)
next := scanTagHeaders
switch key {
case "object", "type", "tag", "tagger":
// Out-of-canonical-position duplicates are dropped, mirroring the
// strict ordering of upstream's parse_tag_buffer.
case headerpgp256:
s.t.SignatureSHA256 += string(data) + "\n"
next = scanTagPgp256Cont
default:
// Unknown header — silently dropped (the Tag struct does not
// expose ExtraHeaders).
}
if err == io.EOF {
return nil, nil
}
return next, nil
}
// scanTagPgp256Cont accumulates continuation lines for the gpgsig-sha256
// header. Continuations strip exactly one leading space, mirroring
// upstream's `line + 1` (commit.c:1509). The first non-continuation line
// is pushed back so scanTagHeaders can dispatch it — repeat occurrences
// of the same header land back here and concatenate, matching upstream's
// parse_buffer_signed_by_header (commit.c:1186).
func scanTagPgp256Cont(s *tagScanner) (tagState, error) {
line, err := s.readLine()
if err != nil && err != io.EOF {
return nil, err
}
if len(line) > 0 && line[0] == ' ' {
s.t.SignatureSHA256 += string(line[1:])
if err == io.EOF {
return nil, nil
}
return scanTagPgp256Cont, nil
}
if len(line) > 0 {
s.pushBack(line, err)
}
return scanTagHeaders, nil
}
// scanTagMessage drains the remaining bytes into the message buffer.
// (*Tag).Decode then runs parseSignedBytes over those bytes to peel off
// the optional inline trailing PGP signature.
func scanTagMessage(s *tagScanner) (tagState, error) {
for {
line, err := s.readLine()
if err != nil && err != io.EOF {
return nil, err
}
if len(line) > 0 {
s.msgbuf.Write(line)
}
if err == io.EOF {
return nil, nil
}
}
}
package object
import (
"bufio"
"context"
"errors"
"fmt"
"io"
"path"
"path/filepath"
"sort"
"strings"
"github.com/go-git/go-git/v6/internal/pathutil"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/filemode"
"github.com/go-git/go-git/v6/plumbing/storer"
"github.com/go-git/go-git/v6/utils/ioutil"
"github.com/go-git/go-git/v6/utils/sync"
)
const (
maxTreeDepth = 1024
startingStackSize = 8
)
// New errors defined by this package.
var (
ErrMaxTreeDepth = errors.New("maximum tree depth exceeded")
ErrFileNotFound = errors.New("file not found")
ErrDirectoryNotFound = errors.New("directory not found")
ErrEntryNotFound = errors.New("entry not found")
ErrEntriesNotSorted = errors.New("entries in tree are not sorted")
ErrMalformedTree = errors.New("malformed tree")
ErrDuplicateEntry = errors.New("duplicate entry in tree")
ErrInvalidTree = errors.New("invalid tree")
)
// maxTreeEntryNameLen mirrors the default of upstream Git's
// `fsck.treeEntryLargeName.maxTreeEntryLen` configuration (fsck.c
// in v2.54.0[1]). 4096 bytes is well above any realistic tree-entry
// name; entries longer than this almost always indicate a malformed
// or hand-crafted tree object.
//
// [1]: https://github.com/git/git/blob/v2.54.0/fsck.c#L26
const maxTreeEntryNameLen = 4096
// Tree is basically like a directory - it references a bunch of other trees
// and/or blobs (i.e. files and sub-directories)
type Tree struct {
Entries []TreeEntry
Hash plumbing.Hash
s storer.EncodedObjectStorer
t map[string]*Tree // tree path cache
entriesSorted bool
}
// GetTree gets a tree from an object storer and decodes it.
func GetTree(s storer.EncodedObjectStorer, h plumbing.Hash) (*Tree, error) {
o, err := s.EncodedObject(plumbing.TreeObject, h)
if err != nil {
return nil, err
}
return DecodeTree(s, o)
}
// DecodeTree decodes an encoded object into a *Tree and associates it to the
// given object storer.
func DecodeTree(s storer.EncodedObjectStorer, o plumbing.EncodedObject) (*Tree, error) {
t := &Tree{s: s}
if err := t.Decode(o); err != nil {
return nil, err
}
return t, nil
}
// TreeEntry represents a file
type TreeEntry struct {
Name string
Mode filemode.FileMode
Hash plumbing.Hash
}
// File returns the hash of the file identified by the `path` argument.
// The path is interpreted as relative to the tree receiver.
func (t *Tree) File(path string) (*File, error) {
e, err := t.FindEntry(path)
if err != nil {
return nil, ErrFileNotFound
}
blob, err := GetBlob(t.s, e.Hash)
if err != nil {
if errors.Is(err, plumbing.ErrObjectNotFound) {
return nil, ErrFileNotFound
}
return nil, err
}
return NewFile(path, e.Mode, blob), nil
}
// Size returns the plaintext size of an object, without reading it
// into memory.
func (t *Tree) Size(path string) (int64, error) {
e, err := t.FindEntry(path)
if err != nil {
return 0, ErrEntryNotFound
}
return t.s.EncodedObjectSize(e.Hash)
}
// Tree returns the tree identified by the `path` argument.
// The path is interpreted as relative to the tree receiver.
func (t *Tree) Tree(path string) (*Tree, error) {
e, err := t.FindEntry(path)
if err != nil {
return nil, ErrDirectoryNotFound
}
tree, err := GetTree(t.s, e.Hash)
if errors.Is(err, plumbing.ErrObjectNotFound) {
return nil, ErrDirectoryNotFound
}
return tree, err
}
// TreeEntryFile returns the *File for a given *TreeEntry.
//
// The entry's name is validated against pathutil.ValidTreePath for
// the same reason FindEntry validates: TreeEntryFile is a boundary
// where attacker-controlled tree data leaves the trusted store as a
// *File whose Name a caller can hand to filesystem ops.
func (t *Tree) TreeEntryFile(e *TreeEntry) (*File, error) {
if err := pathutil.ValidTreePath(e.Name); err != nil {
return nil, err
}
blob, err := GetBlob(t.s, e.Hash)
if err != nil {
return nil, err
}
return NewFile(e.Name, e.Mode, blob), nil
}
// FindEntry search a TreeEntry in this tree or any subtree.
//
// The lookup path is validated against pathutil.ValidTreePath to
// prevent attacker-controlled tree contents from leaking past this
// boundary as `.git`-shaped or path-traversal-shaped names. Callers
// that legitimately need to look up unsafe paths should walk the
// tree manually.
func (t *Tree) FindEntry(path string) (*TreeEntry, error) {
if err := pathutil.ValidTreePath(path); err != nil {
return nil, err
}
if t.t == nil {
t.t = make(map[string]*Tree)
}
pathParts := strings.Split(path, "/")
startingTree := t
pathCurrent := ""
// search for the longest path in the tree path cache
for i := len(pathParts) - 1; i >= 1; i-- {
path := filepath.Join(pathParts[:i]...)
tree, ok := t.t[path]
if ok {
startingTree = tree
pathParts = pathParts[i:]
pathCurrent = path
break
}
}
var tree *Tree
var err error
for tree = startingTree; len(pathParts) > 1; pathParts = pathParts[1:] {
if tree, err = tree.dir(pathParts[0]); err != nil {
return nil, err
}
pathCurrent = filepath.Join(pathCurrent, pathParts[0])
t.t[pathCurrent] = tree
}
return tree.entry(pathParts[0])
}
func (t *Tree) dir(baseName string) (*Tree, error) {
entry, err := t.entry(baseName)
if err != nil {
return nil, ErrDirectoryNotFound
}
obj, err := t.s.EncodedObject(plumbing.TreeObject, entry.Hash)
if err != nil {
return nil, err
}
tree := &Tree{s: t.s}
err = tree.Decode(obj)
return tree, err
}
func (t *Tree) entry(baseName string) (*TreeEntry, error) {
if t.entriesSorted {
if entry := t.searchEntry(baseName); entry != nil {
return entry, nil
}
return nil, ErrEntryNotFound
}
pastName := baseName + "/"
for i := range t.Entries {
entry := &t.Entries[i]
if entry.Name == baseName {
return entry, nil
}
if treeEntrySortName(entry) > pastName {
break
}
}
return nil, ErrEntryNotFound
}
func (t *Tree) searchEntry(baseName string) *TreeEntry {
if i := t.searchEntryIndex(baseName); i < len(t.Entries) && t.Entries[i].Name == baseName {
return &t.Entries[i]
}
if i := t.searchEntryIndex(baseName + "/"); i < len(t.Entries) && t.Entries[i].Name == baseName {
return &t.Entries[i]
}
return nil
}
func (t *Tree) searchEntryIndex(name string) int {
return sort.Search(len(t.Entries), func(i int) bool {
return treeEntrySortName(&t.Entries[i]) >= name
})
}
// Files returns a FileIter allowing to iterate over the Tree
func (t *Tree) Files() *FileIter {
return NewFileIter(t.s, t)
}
// ID returns the object ID of the tree. The returned value will always match
// the current value of Tree.Hash.
//
// ID is present to fulfill the Object interface.
func (t *Tree) ID() plumbing.Hash {
return t.Hash
}
// Type returns the type of object. It always returns plumbing.TreeObject.
func (t *Tree) Type() plumbing.ObjectType {
return plumbing.TreeObject
}
func (t *Tree) reset() {
storer := t.s
*t = Tree{s: storer}
}
// Decode transform an plumbing.EncodedObject into a Tree struct
func (t *Tree) Decode(o plumbing.EncodedObject) (err error) {
if o.Type() != plumbing.TreeObject {
return ErrUnsupportedObject
}
t.reset()
t.Hash = o.Hash()
// assume tree is sorted as a valid tree should always be sorted.
t.entriesSorted = true
if o.Size() == 0 {
return nil
}
reader, err := o.Reader()
if err != nil {
return err
}
defer ioutil.CheckClose(reader, &err)
r := sync.GetBufioReader(reader)
defer sync.PutBufioReader(r)
var prevSortName string
for {
// Use ReadSlice to get a view into bufio's internal buffer,
// avoiding a string allocation for the mode (which is parsed
// into a uint32 immediately and doesn't need to persist).
modeSlice, err := r.ReadSlice(' ')
if err != nil {
if err == io.EOF {
if len(modeSlice) != 0 {
return fmt.Errorf("%w: missing mode terminator", ErrMalformedTree)
}
break
}
return err
}
modeSlice = modeSlice[:len(modeSlice)-1] // strip delimiter
mode, err := filemode.FromBytes(modeSlice)
if err != nil {
return fmt.Errorf("%w: malformed mode", ErrMalformedTree)
}
mode = canonicalTreeMode(mode)
nameSlice, err := r.ReadSlice(0)
if err == bufio.ErrBufferFull {
// Rare: name exceeds bufio's buffer. Accumulate the rest.
buf := append([]byte(nil), nameSlice...)
for err == bufio.ErrBufferFull {
var more []byte
more, err = r.ReadSlice(0)
buf = append(buf, more...)
}
nameSlice = buf
}
if err != nil {
if err == io.EOF {
return fmt.Errorf("%w: missing filename terminator", ErrMalformedTree)
}
return err
}
if len(nameSlice) == 1 {
return fmt.Errorf("%w: empty filename", ErrMalformedTree)
}
name := string(nameSlice[:len(nameSlice)-1]) // strip delimiter
var hash plumbing.Hash
hash.ResetBySize(t.Hash.Size())
if _, err = hash.ReadFrom(r); err != nil {
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
return fmt.Errorf("%w: truncated object id", ErrMalformedTree)
}
return err
}
entry := TreeEntry{
Hash: hash,
Mode: mode,
Name: name,
}
sortName := treeEntrySortName(&entry)
if len(t.Entries) != 0 && prevSortName > sortName {
t.entriesSorted = false
}
prevSortName = sortName
t.Entries = append(t.Entries, entry)
}
return nil
}
// TreeEntrySorter is a helper type for sorting TreeEntry slices.
type TreeEntrySorter []TreeEntry
func (s TreeEntrySorter) Len() int {
return len(s)
}
func (s TreeEntrySorter) Less(i, j int) bool {
return treeEntrySortName(&s[i]) < treeEntrySortName(&s[j])
}
func (s TreeEntrySorter) Swap(i, j int) {
s[i], s[j] = s[j], s[i]
}
// Git compares tree entries as if directory names had a trailing slash.
func treeEntrySortName(e *TreeEntry) string {
if e.Mode == filemode.Dir {
return e.Name + "/"
}
return e.Name
}
func canonicalTreeMode(mode filemode.FileMode) filemode.FileMode {
switch mode & 0o170000 {
case 0o040000:
return filemode.Dir
case 0o100000:
if mode&0o111 != 0 {
return filemode.Executable
}
return filemode.Regular
case 0o120000:
return filemode.Symlink
default:
return filemode.Submodule
}
}
// Encode transforms a Tree into a plumbing.EncodedObject.
//
// The tree is run through Tree.Validate before any bytes are written,
// so the encoder cannot produce a tree object containing components
// such as ".git", "..", control characters, HFS+/NTFS variants of
// ".git", null entry hashes, oversize names, mis-sorted or duplicate
// entries, or symlinks disguised as ".gitmodules"/".gitattributes"/
// ".gitignore"/".mailmap". Callers that need to emit such bytes for
// testing or recovery should write them directly via
// plumbing.EncodedObject rather than through this method.
func (t *Tree) Encode(o plumbing.EncodedObject) (err error) {
if err := t.Validate(); err != nil {
return err
}
o.SetType(plumbing.TreeObject)
w, err := o.Writer()
if err != nil {
return err
}
defer ioutil.CheckClose(w, &err)
for _, entry := range t.Entries {
if _, err = fmt.Fprintf(w, "%o %s", entry.Mode, entry.Name); err != nil {
return err
}
if _, err = w.Write([]byte{0x00}); err != nil {
return err
}
if _, err = entry.Hash.WriteTo(w); err != nil {
return err
}
}
return err
}
// Validate reports whether the tree object obeys the same structural
// rules upstream Git's fsck_tree[1] enforces. It is the read-side
// counterpart to Tree.Encode's producer-side gate: Decode is permissive
// so that inspection and recovery tools can read trees with unusual
// entries, and callers that want fsck-shaped reporting call Validate.
//
// The returned error wraps ErrInvalidTree (and, where applicable,
// ErrEntriesNotSorted, ErrDuplicateEntry, or pathutil.ErrInvalidPath)
// so callers can match either the umbrella or specific rule with
// errors.Is. When multiple rules are violated they are reported
// together via errors.Join.
//
// Two fsck_tree warnings — zero-padded modes and the non-canonical
// 0100664 bits — are not surfaced here. Both rely on inspecting the
// original octal string from the wire, which canonicalTreeMode
// discards during Decode. Detecting them would require a parallel
// raw-mode field on TreeEntry; the structural rules below are the
// load-bearing ones for refusing malformed trees.
//
// [1]: https://github.com/git/git/blob/v2.54.0/fsck.c#L616-L800
func (t *Tree) Validate() error {
var errs []error
add := func(err error) {
errs = append(errs, fmt.Errorf("%w: %w", ErrInvalidTree, err))
}
seen := make(map[string]struct{}, len(t.Entries))
var prevSortName string
for i := range t.Entries {
e := &t.Entries[i]
if e.Hash.IsZero() {
add(fmt.Errorf("entry %q points to null hash", e.Name))
}
switch {
case e.Name == "":
add(errors.New("contains empty entry name"))
case strings.ContainsRune(e.Name, '/'):
add(fmt.Errorf("entry name %q contains a slash", e.Name))
default:
if err := pathutil.ValidTreePath(e.Name); err != nil {
add(err)
}
if _, dup := seen[e.Name]; dup {
add(fmt.Errorf("%w: %q", ErrDuplicateEntry, e.Name))
}
seen[e.Name] = struct{}{}
if len(e.Name) > maxTreeEntryNameLen {
add(fmt.Errorf("entry name length %d exceeds %d", len(e.Name), maxTreeEntryNameLen))
}
}
// Mode validation against the canonical set. Decode normalises
// the wire bytes via canonicalTreeMode, so this rule mainly
// catches programmatically-built trees with garbage modes;
// the zero-padded-mode and non-canonical-bit checks fsck_tree
// runs against the raw wire form are out of reach without
// retaining the original octal string.
if !isValidTreeMode(e.Mode) {
add(fmt.Errorf("entry %q has bad mode %o", e.Name, e.Mode))
}
// Symlink-disguised metadata files. Mirrors the four FSCK_MSG_*
// _SYMLINK reports in fsck_tree.
if e.Mode == filemode.Symlink {
switch {
case pathutil.IsHFSDotGitmodules(e.Name) || pathutil.IsNTFSDotGitmodules(e.Name):
add(errors.New(".gitmodules is a symlink"))
case pathutil.IsHFSDotGitattributes(e.Name) || pathutil.IsNTFSDotGitattributes(e.Name):
add(errors.New(".gitattributes is a symlink"))
case pathutil.IsHFSDotGitignore(e.Name) || pathutil.IsNTFSDotGitignore(e.Name):
add(errors.New(".gitignore is a symlink"))
case pathutil.IsHFSDotMailmap(e.Name) || pathutil.IsNTFSDotMailmap(e.Name):
add(errors.New(".mailmap is a symlink"))
}
}
sortName := treeEntrySortName(e)
if i > 0 && prevSortName > sortName {
add(ErrEntriesNotSorted)
}
prevSortName = sortName
}
return errors.Join(errs...)
}
// isValidTreeMode reports whether mode is one of the canonical tree
// modes upstream Git accepts in fsck_tree, including the non-canonical
// 0100664 that upstream tolerates outside --strict mode.
func isValidTreeMode(mode filemode.FileMode) bool {
switch mode {
case filemode.Regular,
filemode.Executable,
filemode.Symlink,
filemode.Dir,
filemode.Submodule,
filemode.Deprecated:
return true
}
return false
}
// Diff returns a list of changes between this tree and the provided one
func (t *Tree) Diff(to *Tree) (Changes, error) {
return t.DiffContext(context.Background(), to)
}
// DiffContext returns a list of changes between this tree and the provided one
// Error will be returned if context expires. Provided context must be non nil.
//
// NOTE: Since version 5.1.0 the renames are correctly handled, the settings
// used are the recommended options DefaultDiffTreeOptions.
func (t *Tree) DiffContext(ctx context.Context, to *Tree) (Changes, error) {
return DiffTreeWithOptions(ctx, t, to, DefaultDiffTreeOptions)
}
// Patch returns a slice of Patch objects with all the changes between trees
// in chunks. This representation can be used to create several diff outputs.
func (t *Tree) Patch(to *Tree) (*Patch, error) {
return t.PatchContext(context.Background(), to)
}
// PatchContext returns a slice of Patch objects with all the changes between
// trees in chunks. This representation can be used to create several diff
// outputs. If context expires, an error will be returned. Provided context must
// be non-nil.
//
// NOTE: Since version 5.1.0 the renames are correctly handled, the settings
// used are the recommended options DefaultDiffTreeOptions.
func (t *Tree) PatchContext(ctx context.Context, to *Tree) (*Patch, error) {
changes, err := t.DiffContext(ctx, to)
if err != nil {
return nil, err
}
return changes.PatchContext(ctx)
}
// treeEntryIter facilitates iterating through the TreeEntry objects in a Tree.
type treeEntryIter struct {
t *Tree
pos int
}
func (iter *treeEntryIter) Next() (TreeEntry, error) {
if iter.pos >= len(iter.t.Entries) {
return TreeEntry{}, io.EOF
}
iter.pos++
return iter.t.Entries[iter.pos-1], nil
}
// TreeWalker provides a means of walking through all of the entries in a Tree.
type TreeWalker struct {
stack []*treeEntryIter
base string
recursive bool
seen map[plumbing.Hash]bool
// skipPathValidation disables the pathutil.ValidTreePath check in Next.
// It is set by inspection-only callers (e.g. the diff treeNoder) that
// never funnel entry names into the filesystem and must enumerate trees
// faithfully, including entries with names upstream Git accepts but that
// are unsafe to materialise (control characters, `.git`-shaped names).
skipPathValidation bool
s storer.EncodedObjectStorer
t *Tree
}
// NewTreeWalker returns a new TreeWalker for the given tree.
//
// It is the caller's responsibility to call Close() when finished with the
// tree walker.
func NewTreeWalker(t *Tree, recursive bool, seen map[plumbing.Hash]bool) *TreeWalker {
stack := make([]*treeEntryIter, 0, startingStackSize)
stack = append(stack, &treeEntryIter{t, 0})
return &TreeWalker{
stack: stack,
recursive: recursive,
seen: seen,
s: t.s,
t: t,
}
}
// Next returns the next object from the tree. Objects are returned in order
// and subtrees are included. After the last object has been returned further
// calls to Next() will return io.EOF.
//
// Each entry's name is validated against pathutil.ValidTreePath as it
// surfaces, so callers that funnel the returned name into filesystem
// or archive output can trust it is free of `.git`-shaped components,
// HFS+/NTFS variants, Windows reserved names, and traversal sequences.
// A malformed entry stops the walk with the validator's error;
// inspection-only callers that need to enumerate raw, unvalidated
// names can read Tree.Entries directly or set skipPathValidation.
//
// In the current implementation any objects which cannot be found in the
// underlying repository will be skipped automatically. It is possible that this
// may change in future versions.
func (w *TreeWalker) Next() (name string, entry TreeEntry, err error) {
var obj *Tree
for {
current := len(w.stack) - 1
if current < 0 {
// Nothing left on the stack so we're finished
err = io.EOF
return name, entry, err
}
if current > maxTreeDepth {
// We're probably following bad data or some self-referencing tree
err = ErrMaxTreeDepth
return name, entry, err
}
entry, err = w.stack[current].Next()
if err == io.EOF {
// Finished with the current tree, move back up to the parent
w.stack = w.stack[:current]
w.base, _ = path.Split(w.base)
w.base = strings.TrimSuffix(w.base, "/")
continue
}
if err != nil {
return name, entry, err
}
if w.seen[entry.Hash] {
continue
}
if !w.skipPathValidation {
if err := pathutil.ValidTreePath(entry.Name); err != nil {
return name, entry, err
}
}
if entry.Mode == filemode.Dir {
obj, err = GetTree(w.s, entry.Hash)
}
name = simpleJoin(w.base, entry.Name)
if err != nil {
err = io.EOF
return name, entry, err
}
break
}
if !w.recursive {
return name, entry, err
}
if obj != nil {
w.stack = append(w.stack, &treeEntryIter{obj, 0})
w.base = simpleJoin(w.base, entry.Name)
}
return name, entry, err
}
// Tree returns the tree that the tree walker most recently operated on.
func (w *TreeWalker) Tree() *Tree {
current := len(w.stack) - 1
if w.stack[current].pos == 0 {
current--
}
if current < 0 {
return nil
}
return w.stack[current].t
}
// Close releases any resources used by the TreeWalker.
func (w *TreeWalker) Close() {
w.stack = nil
}
// TreeIter provides an iterator for a set of trees.
type TreeIter struct {
storer.EncodedObjectIter
s storer.EncodedObjectStorer
}
// NewTreeIter takes a storer.EncodedObjectStorer and a
// storer.EncodedObjectIter and returns a *TreeIter that iterates over all
// tree contained in the storer.EncodedObjectIter.
//
// Any non-tree object returned by the storer.EncodedObjectIter is skipped.
func NewTreeIter(s storer.EncodedObjectStorer, iter storer.EncodedObjectIter) *TreeIter {
return &TreeIter{iter, s}
}
// Next moves the iterator to the next tree and returns a pointer to it. If
// there are no more trees, it returns io.EOF.
func (iter *TreeIter) Next() (*Tree, error) {
for {
obj, err := iter.EncodedObjectIter.Next()
if err != nil {
return nil, err
}
if obj.Type() != plumbing.TreeObject {
continue
}
return DecodeTree(iter.s, obj)
}
}
// ForEach call the cb function for each tree contained on this iter until
// an error happens or the end of the iter is reached. If ErrStop is sent
// the iteration is stop but no error is returned. The iterator is closed.
func (iter *TreeIter) ForEach(cb func(*Tree) error) error {
return iter.EncodedObjectIter.ForEach(func(obj plumbing.EncodedObject) error {
if obj.Type() != plumbing.TreeObject {
return nil
}
t, err := DecodeTree(iter.s, obj)
if err != nil {
return err
}
return cb(t)
})
}
func simpleJoin(parent, child string) string {
if len(parent) > 0 {
return parent + "/" + child
}
return child
}
package object
import (
"io"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/filemode"
"github.com/go-git/go-git/v6/utils/merkletrie/noder"
)
// A treenoder is a helper type that wraps git trees into merkletrie
// noders.
//
// As a merkletrie noder doesn't understand the concept of modes (e.g.
// file permissions), the treenoder includes the mode of the git tree in
// the hash, so changes in the modes will be detected as modifications
// to the file contents by the merkletrie difftree algorithm. This is
// consistent with how the "git diff-tree" command works.
type treeNoder struct {
parent *Tree // the root node is its own parent
name string // empty string for the root node
mode filemode.FileMode
hash plumbing.Hash
children []noder.Noder // memoized
}
// NewTreeRootNode returns the root node of a Tree
func NewTreeRootNode(t *Tree) noder.Noder {
if t == nil {
return &treeNoder{}
}
return &treeNoder{
parent: t,
name: "",
mode: filemode.Dir,
hash: t.Hash,
}
}
func (t *treeNoder) Skip() bool {
return false
}
func (t *treeNoder) isRoot() bool {
return t.name == ""
}
func (t *treeNoder) String() string {
return "treeNoder <" + t.name + ">"
}
func (t *treeNoder) Hash() []byte {
if t.mode == filemode.Deprecated {
return append(t.hash.Bytes(), filemode.Regular.Bytes()...)
}
return append(t.hash.Bytes(), t.mode.Bytes()...)
}
func (t *treeNoder) Name() string {
return t.name
}
func (t *treeNoder) IsDir() bool {
return t.mode == filemode.Dir
}
// Children will return the children of a treenoder as treenoders,
// building them from the children of the wrapped git tree.
func (t *treeNoder) Children() ([]noder.Noder, error) {
if t.mode != filemode.Dir {
return noder.NoChildren, nil
}
// children are memoized for efficiency
if t.children != nil {
return t.children, nil
}
// the parent of the returned children will be ourself as a tree if
// we are a not the root treenoder. The root is special as it
// is is own parent.
parent := t.parent
if !t.isRoot() {
var err error
if parent, err = t.parent.Tree(t.name); err != nil {
return nil, err
}
}
var err error
t.children, err = transformChildren(parent)
return t.children, err
}
// Returns the children of a tree as treenoders.
// Efficiency is key here.
func transformChildren(t *Tree) ([]noder.Noder, error) {
var err error
var e TreeEntry
// there will be more tree entries than children in the tree,
// due to submodules and empty directories, but I think it is still
// worth it to pre-allocate the whole array now, even if sometimes
// is bigger than needed.
ret := make([]noder.Noder, 0, len(t.Entries))
walker := NewTreeWalker(t, false, nil) // don't recurse
// The diff walk is read-only and never materialises entry names into the
// filesystem, so it must enumerate the tree faithfully — including entries
// with names that are unsafe to check out but valid per upstream Git (e.g.
// control characters). Path safety is enforced at materialisation
// boundaries (FindEntry, TreeEntryFile, archive, FileIter), not here.
walker.skipPathValidation = true
// don't defer walker.Close() for efficiency reasons.
for {
_, e, err = walker.Next()
if err == io.EOF {
break
}
if err != nil {
walker.Close()
return nil, err
}
ret = append(ret, &treeNoder{
parent: t,
name: e.Name,
mode: e.Mode,
hash: e.Hash,
})
}
walker.Close()
return ret, nil
}
// len(t.tree.Entries) != the number of elements walked by treewalker
// for some reason because of empty directories, submodules, etc, so we
// have to walk here.
func (t *treeNoder) NumChildren() (int, error) {
children, err := t.Children()
if err != nil {
return 0, err
}
return len(children), nil
}
// Package capability defines the server and client capabilities.
package capability
import (
"errors"
"fmt"
"os"
"slices"
"strings"
"github.com/go-git/go-git/v6/plumbing/format/pktline"
)
var (
// ErrArguments is returned when a capability is given an argument but
// doesn't accept any.
ErrArguments = errors.New("capability does not accept arguments")
// ErrArgumentsRequired is returned when a capability requires an argument
// but none was provided.
ErrArgumentsRequired = errors.New("capability requires an argument")
// ErrMultipleArguments is returned when a capability accepts only one argument
// but multiple were provided.
ErrMultipleArguments = errors.New("capability accepts only one argument")
// ErrEmptyArgument is returned when an argument is empty.
ErrEmptyArgument = errors.New("capability argument cannot be empty")
)
// Capability describes a server or client capability.
type Capability = string
const (
// MultiACK capability allows the server to return "ACK obj-id continue" as
// soon as it finds a commit that it can use as a common base, between the
// client's wants and the client's have set.
//
// By sending this early, the server can potentially head off the client
// from walking any further down that particular branch of the client's
// repository history. The client may still need to walk down other
// branches, sending have lines for those, until the server has a
// complete cut across the DAG, or the client has said "done".
//
// Without multi_ack, a client sends have lines in --date-order until
// the server has found a common base. That means the client will send
// have lines that are already known by the server to be common, because
// they overlap in time with another branch that the server hasn't found
// a common base on yet.
//
// For example suppose the client has commits in caps that the server
// doesn't and the server has commits in lower case that the client
// doesn't, as in the following diagram:
//
// +---- u ---------------------- x
// / +----- y
// / /
// a -- b -- c -- d -- E -- F
// \
// +--- Q -- R -- S
//
// If the client wants x,y and starts out by saying have F,S, the server
// doesn't know what F,S is. Eventually the client says "have d" and
// the server sends "ACK d continue" to let the client know to stop
// walking down that line (so don't send c-b-a), but it's not done yet,
// it needs a base for x. The client keeps going with S-R-Q, until a
// gets reached, at which point the server has a clear base and it all
// ends.
//
// Without multi_ack the client would have sent that c-b-a chain anyway,
// interleaved with S-R-Q.
MultiACK Capability = "multi_ack"
// MultiACKDetailed is an extension of multi_ack that permits client to
// better understand the server's in-memory state.
MultiACKDetailed Capability = "multi_ack_detailed"
// NoDone should only be used with the smart HTTP protocol. If
// multi_ack_detailed and no-done are both present, then the sender is
// free to immediately send a pack following its first "ACK obj-id ready"
// message.
//
// Without no-done in the smart HTTP protocol, the server session would
// end and the client has to make another trip to send "done" before
// the server can send the pack. no-done removes the last round and
// thus slightly reduces latency.
NoDone Capability = "no-done"
// ThinPack is one with deltas which reference base objects not
// contained within the pack (but are known to exist at the receiving
// end). This can reduce the network traffic significantly, but it
// requires the receiving end to know how to "thicken" these packs by
// adding the missing bases to the pack.
//
// The upload-pack server advertises 'thin-pack' when it can generate
// and send a thin pack. A client requests the 'thin-pack' capability
// when it understands how to "thicken" it, notifying the server that
// it can receive such a pack. A client MUST NOT request the
// 'thin-pack' capability if it cannot turn a thin pack into a
// self-contained pack.
//
// Receive-pack, on the other hand, is assumed by default to be able to
// handle thin packs, but can ask the client not to use the feature by
// advertising the 'no-thin' capability. A client MUST NOT send a thin
// pack if the server advertises the 'no-thin' capability.
//
// The reasons for this asymmetry are historical. The receive-pack
// program did not exist until after the invention of thin packs, so
// historically the reference implementation of receive-pack always
// understood thin packs. Adding 'no-thin' later allowed receive-pack
// to disable the feature in a backwards-compatible manner.
ThinPack Capability = "thin-pack"
// NoThin is the opposite of the ThinPack capability.
NoThin Capability = "no-thin"
// Sideband means that server can send, and client understand multiplexed
// progress reports and error info interleaved with the packfile itself.
//
// These two options are mutually exclusive. A modern client always
// favors Sideband64k.
//
// Either mode indicates that the packfile data will be streamed broken
// up into packets of up to either 1000 bytes in the case of 'side_band',
// or 65520 bytes in the case of 'side_band_64k'. Each packet is made up
// of a leading 4-byte pkt-line length of how much data is in the packet,
// followed by a 1-byte stream code, followed by the actual data.
//
// The stream code can be one of:
//
// 1 - pack data
// 2 - progress messages
// 3 - fatal error message just before stream aborts
//
// The "side-band-64k" capability came about as a way for newer clients
// that can handle much larger packets to request packets that are
// actually crammed nearly full, while maintaining backward compatibility
// for the older clients.
//
// Further, with side-band and its up to 1000-byte messages, it's actually
// 999 bytes of payload and 1 byte for the stream code. With side-band-64k,
// same deal, you have up to 65519 bytes of data and 1 byte for the stream
// code.
//
// The client MUST send only maximum of one of "side-band" and "side-
// band-64k". Server MUST diagnose it as an error if client requests
// both.
Sideband Capability = "side-band"
// Sideband64k is the 64k variant of Sideband.
Sideband64k Capability = "side-band-64k"
// OFSDelta server can send, and client understand PACKv2 with delta
// referring to its base by position in pack rather than by an obj-id. That
// is, they can send/read OBJ_OFS_DELTA (aka type 6) in a packfile.
OFSDelta Capability = "ofs-delta"
// Agent the server may optionally send this capability to notify the client
// that the server is running version `X`. The client may optionally return
// its own agent string by responding with an `agent=Y` capability (but it
// MUST NOT do so if the server did not mention the agent capability). The
// `X` and `Y` strings may contain any printable ASCII characters except
// space (i.e., the byte range 32 < x < 127), and are typically of the form
// "package/version" (e.g., "git/1.8.3.1"). The agent strings are purely
// informative for statistics and debugging purposes, and MUST NOT be used
// to programmatically assume the presence or absence of particular features.
Agent Capability = "agent"
// Shallow capability adds "deepen", "shallow" and "unshallow" commands to
// the fetch-pack/upload-pack protocol so clients can request shallow
// clones.
Shallow Capability = "shallow"
// DeepenSince adds "deepen-since" command to fetch-pack/upload-pack
// protocol so the client can request shallow clones that are cut at a
// specific time, instead of depth. Internally it's equivalent of doing
// "rev-list --max-age=<timestamp>" on the server side. "deepen-since"
// cannot be used with "deepen".
DeepenSince Capability = "deepen-since"
// DeepenNot adds "deepen-not" command to fetch-pack/upload-pack
// protocol so the client can request shallow clones that are cut at a
// specific revision, instead of depth. Internally it's equivalent of
// doing "rev-list --not <rev>" on the server side. "deepen-not"
// cannot be used with "deepen", but can be used with "deepen-since".
DeepenNot Capability = "deepen-not"
// DeepenRelative if this capability is requested by the client, the
// semantics of "deepen" command is changed. The "depth" argument is the
// depth from the current shallow boundary, instead of the depth from
// remote refs.
DeepenRelative Capability = "deepen-relative"
// NoProgress the client was started with "git clone -q" or something, and
// doesn't want that side band 2. Basically the client just says "I do not
// wish to receive stream 2 on sideband, so do not send it to me, and if
// you did, I will drop it on the floor anyway". However, the sideband
// channel 3 is still used for error responses.
NoProgress Capability = "no-progress"
// IncludeTag capability is about sending annotated tags if we are
// sending objects they point to. If we pack an object to the client, and
// a tag object points exactly at that object, we pack the tag object too.
// In general this allows a client to get all new annotated tags when it
// fetches a branch, in a single network connection.
//
// Clients MAY always send include-tag, hardcoding it into a request when
// the server advertises this capability. The decision for a client to
// request include-tag only has to do with the client's desires for tag
// data, whether or not a server had advertised objects in the
// refs/tags/* namespace.
//
// Servers MUST pack the tags if their referrant is packed and the client
// has requested include-tags.
//
// Clients MUST be prepared for the case where a server has ignored
// include-tag and has not actually sent tags in the pack. In such
// cases the client SHOULD issue a subsequent fetch to acquire the tags
// that include-tag would have otherwise given the client.
//
// The server SHOULD send include-tag, if it supports it, regardless
// of whether or not there are tags available.
IncludeTag Capability = "include-tag"
// ReportStatus the receive-pack process can receive a 'report-status'
// capability, which tells it that the client wants a report of what
// happened after a packfile upload and reference update. If the pushing
// client requests this capability, after unpacking and updating references
// the server will respond with whether the packfile unpacked successfully
// and if each reference was updated successfully. If any of those were not
// successful, it will send back an error message. See pack-protocol.txt
// for example messages.
ReportStatus Capability = "report-status"
// ReportStatusV2 extends capability report-status by adding new "option"
// directives in order to support reference rewritten by the "proc-receive"
// hook. The "proc-receive" hook may handle a command for a
// pseudo-reference which may create or update a reference with different
// name, new-oid, and old-oid. While the capability report-status cannot
// report for such case. See gitprotocol-pack[5] for details.
ReportStatusV2 Capability = "report-status-v2"
// DeleteRefs If the server sends back this capability, it means that
// it is capable of accepting a zero-id value as the target
// value of a reference update. It is not sent back by the client, it
// simply informs the client that it can be sent zero-id values
// to delete references
DeleteRefs Capability = "delete-refs"
// Quiet If the receive-pack server advertises this capability, it is
// capable of silencing human-readable progress output which otherwise may
// be shown when processing the received pack. A send-pack client should
// respond with the 'quiet' capability to suppress server-side progress
// reporting if the local progress reporting is also being suppressed
// (e.g., via `push -q`, or if stderr does not go to a tty).
Quiet Capability = "quiet"
// Atomic If the server sends this capability it is capable of accepting
// atomic pushes. If the pushing client requests this capability, the server
// will update the refs in one atomic transaction. Either all refs are
// updated or none.
Atomic Capability = "atomic"
// PushOptions If the server sends this capability it is able to accept
// push options after the update commands have been sent, but before the
// packfile is streamed. If the pushing client requests this capability,
// the server will pass the options to the pre- and post- receive hooks
// that process this push request.
PushOptions Capability = "push-options"
// AllowTipSHA1InWant if the upload-pack server advertises this capability,
// fetch-pack may send "want" lines with SHA-1s that exist at the server but
// are not advertised by upload-pack.
AllowTipSHA1InWant Capability = "allow-tip-sha1-in-want"
// AllowReachableSHA1InWant if the upload-pack server advertises this
// capability, fetch-pack may send "want" lines with SHA-1s that exist at
// the server but are not advertised by upload-pack.
AllowReachableSHA1InWant Capability = "allow-reachable-sha1-in-want"
// PushCert the receive-pack server that advertises this capability is
// willing to accept a signed push certificate, and asks the <nonce> to be
// included in the push certificate. A send-pack client MUST NOT
// send a push-cert packet unless the receive-pack server advertises
// this capability.
PushCert Capability = "push-cert"
// SymRef symbolic reference support for better negotiation.
SymRef Capability = "symref"
// ObjectFormat takes a hash algorithm as an argument, indicates that the
// server supports the given hash algorithms.
ObjectFormat Capability = "object-format"
// Filter if present, fetch-pack may send "filter" commands to request a
// partial clone or partial fetch and request that the server omit various objects from the packfile
Filter Capability = "filter"
)
// V2 command capabilities are advertised by the server in protocol v2
// capability advertisements.
const (
// LsRefs is a v2 command capability indicating the server supports
// the ls-refs command for reference discovery.
LsRefs Capability = "ls-refs"
// FetchCmd is a v2 command capability indicating the server supports
// the fetch command. Named FetchCmd to avoid collision with the
// transport-level Fetch method.
FetchCmd Capability = "fetch"
// ObjectInfo is a v2 command capability indicating the server supports
// the object-info command.
ObjectInfo Capability = "object-info"
// BundleURI is a v2 command capability indicating the server supports
// the bundle-uri command.
BundleURI Capability = "bundle-uri"
// V2 non-command capabilities.
// ServerOption indicates the server supports the server-option capability.
ServerOption Capability = "server-option"
// SessionID indicates the server supports session-id for correlating
// requests and responses in stateless RPC (HTTP).
SessionID Capability = "session-id"
// WaitForDone is a v2 fetch sub-feature indicating the server supports
// the wait-for-done argument in the fetch command.
WaitForDone Capability = "wait-for-done"
)
const userAgent = "go-git/6.x"
// DefaultAgent provides the user agent string.
func DefaultAgent() string {
if envUserAgent, ok := os.LookupEnv("GO_GIT_USER_AGENT_EXTRA"); ok && strings.TrimSpace(envUserAgent) != "" {
return fmt.Sprintf("%s %s", userAgent, envUserAgent)
}
return userAgent
}
// Validate validates that all capabilities in the list are valid v0/v1
// capabilities and that they have proper arguments.
func Validate(l *List) error {
for _, c := range l.All() {
values := l.Get(c)
if err := validateCapability(c, values); err != nil {
return err
}
}
return nil
}
func validateCapability(c Capability, values []string) error {
if err := validateNoEmptyArgs(values); err != nil {
return err
}
if !isKnown(c) {
return fmt.Errorf("unknown capability: %s", c)
}
if requiresArgument(c) && len(values) == 0 {
return ErrArgumentsRequired
}
if !requiresArgument(c) && len(values) != 0 {
return ErrArguments
}
if !allowsMultipleArguments(c) && len(values) > 1 {
return ErrMultipleArguments
}
if c == SessionID && len(values) == 1 {
if err := validateSessionID(values[0]); err != nil {
return err
}
}
return nil
}
// isKnown reports whether the capability is a known v0/v1 capability.
func isKnown(c Capability) bool {
switch c {
case MultiACK, MultiACKDetailed, NoDone, ThinPack, NoThin, Sideband,
Sideband64k, OFSDelta, Agent, Shallow, DeepenSince, DeepenNot,
DeepenRelative, NoProgress, IncludeTag, ReportStatus, ReportStatusV2,
DeleteRefs, Quiet, Atomic, PushOptions, SymRef, AllowTipSHA1InWant,
AllowReachableSHA1InWant, PushCert, Filter, ObjectFormat, SessionID:
return true
default:
return false
}
}
// requiresArgument reports whether the capability requires an argument.
func requiresArgument(c Capability) bool {
switch c {
case Agent, PushCert, SymRef, ObjectFormat, SessionID:
return true
default:
return false
}
}
// allowsMultipleArguments reports whether the capability accepts multiple arguments.
func allowsMultipleArguments(c Capability) bool {
switch c {
case SymRef:
return true
default:
return false
}
}
// validateNoEmptyArgs validates that no argument is empty.
func validateNoEmptyArgs(values []string) error {
if slices.Contains(values, "") {
return ErrEmptyArgument
}
return nil
}
// validateSessionID validates that the session ID is not empty and only
// contains printable ASCII characters except space.
func validateSessionID(sessionID string) error {
if sessionID == "" {
return ErrEmptyArgument
}
if len(sessionID) > pktline.MaxPayloadSize {
return fmt.Errorf("session ID is too long: %d bytes", len(sessionID))
}
if strings.ContainsFunc(sessionID, func(r rune) bool {
return r <= 32 || r >= 127 // Non-printable ASCII characters and space
}) {
return fmt.Errorf("session ID contains invalid characters: %q", sessionID)
}
return nil
}
package capability
import (
"bytes"
)
// List represents a list of capabilities. The zero value is safe to use;
// the internal map is lazily initialized on first write. List is not safe for
// concurrent use.
type List struct {
m map[string]*entry
sort []string
}
type entry struct {
Name string
Values []string
}
// IsEmpty returns true if the List is empty
func (l *List) IsEmpty() bool {
if l == nil {
return true
}
return len(l.sort) == 0
}
// DecodeList decodes a v0/v1 space-separated capability string into the
// List. This is the format used in advertise-refs, upload-request, and
// update-request messages.
func DecodeList(raw []byte, l *List) {
if l == nil {
return
}
raw = bytes.TrimSpace(raw)
if len(raw) == 0 {
return
}
for len(raw) > 0 {
var chunk []byte
if i := bytes.IndexByte(raw, ' '); i >= 0 {
chunk = raw[:i]
raw = raw[i+1:]
} else {
chunk = raw
raw = nil
}
if len(chunk) == 0 {
continue
}
if before, after, ok := bytes.Cut(chunk, []byte{'='}); ok {
l.Add(string(before), string(after))
} else {
l.Add(string(chunk))
}
}
}
// EncodeList encodes the List into a v0/v1 space-separated capability string.
// This is the format used in advertise-refs, upload-request, and
// update-request messages.
func EncodeList(l *List) []byte {
if l == nil {
return nil
}
b, _ := l.MarshalText()
return b
}
// Get returns the values for a capability
func (l *List) Get(capability string) []string {
if l.m == nil {
return nil
}
if _, ok := l.m[capability]; !ok {
return nil
}
return l.m[capability].Values
}
// Set sets a capability removing the previous values
func (l *List) Set(capability string, values ...string) {
if _, ok := l.m[capability]; ok {
l.m[capability].Values = l.m[capability].Values[:0]
}
l.Add(capability, values...)
}
func (l *List) init() {
if l.m == nil {
l.m = make(map[string]*entry)
}
}
// Add adds a capability, values are optional
func (l *List) Add(c string, values ...string) {
l.init()
if !l.Supports(c) {
l.m[c] = &entry{Name: c}
l.sort = append(l.sort, c)
}
if len(values) == 0 {
return
}
l.m[c].Values = append(l.m[c].Values, values...)
}
// Supports returns true if capability is present
func (l *List) Supports(capability string) bool {
if l.m == nil {
return false
}
_, ok := l.m[capability]
return ok
}
// Delete deletes a capability from the List
func (l *List) Delete(capability string) {
if !l.Supports(capability) {
return
}
delete(l.m, capability)
for i, c := range l.sort {
if c != capability {
continue
}
l.sort = append(l.sort[:i], l.sort[i+1:]...)
return
}
}
// All returns a slice with all defined capabilities.
func (l *List) All() []string {
if len(l.sort) == 0 {
return nil
}
cs := make([]string, len(l.sort))
copy(cs, l.sort)
return cs
}
// MarshalText implements encoding.TextMarshaler.
func (l *List) MarshalText() ([]byte, error) {
return l.AppendText(nil)
}
// AppendText implements encoding.TextAppender.
func (l *List) AppendText(b []byte) ([]byte, error) {
first := true
for _, key := range l.sort {
if l.m == nil {
continue
}
c := l.m[key]
if len(c.Values) == 0 {
if !first {
b = append(b, ' ')
}
first = false
b = append(b, key...)
continue
}
for _, value := range c.Values {
if !first {
b = append(b, ' ')
}
first = false
b = append(b, key...)
b = append(b, '=')
b = append(b, value...)
}
}
return b, nil
}
// UnmarshalText implements encoding.TextUnmarshaler.
func (l *List) UnmarshalText(text []byte) error {
DecodeList(text, l)
return nil
}
// String generates the capabilities strings, the capabilities are sorted in
// insertion order.
func (l *List) String() string {
b, _ := l.MarshalText()
return string(b)
}
// Package packp implements encoding and decoding of the Git packfile protocol messages.
package packp
import (
"fmt"
"sort"
"strings"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/protocol"
"github.com/go-git/go-git/v6/plumbing/protocol/capability"
)
// AdvRefs values represent the information transmitted on an
// advertised-refs message. The zero value is safe to use; References
// and Shallows can be populated via append.
type AdvRefs struct {
// Version is the protocol version of the advertisement. The only acceptable
// values are V0 and V1; any other value is invalid. Decode parses it from
// the leading "version" pkt-line (absent for V0) and Encode emits that line
// from it. Within the transport it is set from the version the handshake
// negotiated (DiscoverVersion), which is the single source of truth.
Version protocol.Version
// Capabilities are the capabilities.
Capabilities capability.List
// References are the hash references, including HEAD and peeled refs
// (whose names end in ^{}). They are stored in wire order.
References []*plumbing.Reference
// Shallows are the shallow object ids.
Shallows []plumbing.Hash
}
// Head returns the HEAD reference. It checks the first reference in
// References (HEAD is always first on the wire) before scanning the rest.
func (a *AdvRefs) Head() (*plumbing.Reference, error) {
if len(a.References) > 0 && a.References[0].Name() == plumbing.HEAD {
return a.References[0], nil
}
for _, ref := range a.References {
if ref.Name() == plumbing.HEAD {
return ref, nil
}
}
return nil, plumbing.ErrReferenceNotFound
}
// ResolvedHead returns HEAD as a SymbolicReference when possible. If the
// symref capability is present it is used; otherwise the heuristic
// described in resolvedHeadFromHeuristic is applied. If HEAD cannot be
// resolved it is returned as-is (a HashReference). Returns
// ErrReferenceNotFound if HEAD is not present in References.
func (a *AdvRefs) ResolvedHead() (*plumbing.Reference, error) {
head, err := a.Head()
if err != nil {
return nil, err
}
if head.Type() == plumbing.SymbolicReference {
return head, nil
}
if a.supportSymrefs() {
return a.resolvedHeadFromSymref(head)
}
return a.resolvedHeadFromHeuristic(head), nil
}
// resolvedHeadFromSymref resolves HEAD using the symref capability.
// Returns head unchanged if no HEAD entry exists in the symref map.
func (a *AdvRefs) resolvedHeadFromSymref(head *plumbing.Reference) (*plumbing.Reference, error) {
symrefs, err := a.symRefMap()
if err != nil {
return nil, err
}
if target, ok := symrefs[plumbing.HEAD]; ok {
return plumbing.NewSymbolicReference(plumbing.HEAD, target), nil
}
return head, nil
}
// ResolvedReferences returns all references with HEAD resolved to a
// SymbolicReference when possible, and symref capabilities applied to
// other references. The result is sorted by reference name.
func (a *AdvRefs) ResolvedReferences() ([]*plumbing.Reference, error) {
refs := make([]*plumbing.Reference, len(a.References))
copy(refs, a.References)
symrefs, err := a.symRefMap()
if err != nil {
return nil, err
}
if a.supportSymrefs() {
for name, target := range symrefs {
symRef := plumbing.NewSymbolicReference(name, target)
found := false
for i, ref := range refs {
if ref.Name() == name {
refs[i] = symRef
found = true
break
}
}
if !found {
refs = append(refs, symRef)
}
}
} else {
for i, ref := range refs {
if ref.Name() == plumbing.HEAD && ref.Type() == plumbing.HashReference {
refs[i] = a.resolvedHeadFromHeuristic(ref)
break
}
}
}
sort.Slice(refs, func(i, j int) bool {
return refs[i].Name() < refs[j].Name()
})
return refs, nil
}
// symRefMap parses the symref capability values into a name→target map.
func (a *AdvRefs) symRefMap() (map[plumbing.ReferenceName]plumbing.ReferenceName, error) {
symrefs := a.Capabilities.Get(capability.SymRef)
m := make(map[plumbing.ReferenceName]plumbing.ReferenceName, len(symrefs))
for _, symref := range symrefs {
chunks := strings.Split(symref, ":")
if len(chunks) != 2 {
return nil, fmt.Errorf("bad number of `:` in symref value (%q)", symref)
}
m[plumbing.ReferenceName(chunks[0])] = plumbing.ReferenceName(chunks[1])
}
return m, nil
}
// resolvedHeadFromHeuristic tries to convert HEAD from a HashReference to
// a SymbolicReference pointing to the branch that shares its hash.
//
// If the server does not support the symref capability, git versions
// prior to 1.8.4.3 used this heuristic:
// - Check if master exists and has the same hash as HEAD.
// - If not, scan references in alphabetical order for a matching hash.
// - If no match is found, HEAD is returned unchanged.
func (a *AdvRefs) resolvedHeadFromHeuristic(head *plumbing.Reference) *plumbing.Reference {
return ResolveHeadFromHashHeuristic(head, a.References)
}
// ResolveHeadFromHashHeuristic converts a detached HEAD (a HashReference) into a
// SymbolicReference pointing to the branch that shares its hash, scanning refs.
// It is shared by the v0/v1 advertisement resolution and the Protocol v2 ls-refs
// path, so a detached remote HEAD still yields a symbolic local HEAD on clone,
// matching reference git's pre-symref heuristic:
// - Prefer refs/heads/master when it has the same hash as HEAD.
// - Otherwise pick the alphabetically-first non-peeled ref with that hash.
// - If nothing matches, HEAD is returned unchanged.
func ResolveHeadFromHashHeuristic(head *plumbing.Reference, refs []*plumbing.Reference) *plumbing.Reference {
headHash := head.Hash()
for _, ref := range refs {
if ref.Name() == plumbing.Master && ref.Type() == plumbing.HashReference && ref.Hash() == headHash {
return plumbing.NewSymbolicReference(plumbing.HEAD, plumbing.Master)
}
}
candidates := make([]*plumbing.Reference, 0, len(refs))
for _, ref := range refs {
if ref.Name() == plumbing.HEAD || ref.Name().IsPeeled() {
continue
}
if ref.Type() == plumbing.HashReference && ref.Hash() == headHash {
candidates = append(candidates, ref)
}
}
if len(candidates) > 0 {
sort.Slice(candidates, func(i, j int) bool {
return candidates[i].Name() < candidates[j].Name()
})
return plumbing.NewSymbolicReference(plumbing.HEAD, candidates[0].Name())
}
return head
}
// IsEmpty returns true if doesn't contain any reference.
func (a *AdvRefs) IsEmpty() bool {
return len(a.References) == 0 &&
len(a.Shallows) == 0
}
func (a *AdvRefs) supportSymrefs() bool {
return a.Capabilities.Supports(capability.SymRef)
}
package packp
import (
"bytes"
"errors"
"fmt"
"io"
"strings"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/format/pktline"
"github.com/go-git/go-git/v6/plumbing/protocol"
"github.com/go-git/go-git/v6/plumbing/protocol/capability"
)
var (
// ErrEmptyAdvRefs is returned by Decode if it gets an empty advertised
// references message.
ErrEmptyAdvRefs = errors.New("empty advertised-ref message")
// ErrEmptyInput is returned by Decode if the input is empty.
ErrEmptyInput = errors.New("empty input")
)
// Decode reads the next advertised-refs message form its input and
// stores it in the AdvRefs.
func (a *AdvRefs) Decode(r io.Reader) error {
var (
nLine int
line []byte
err error
)
s := pktline.NewScanner(r)
nextLine := func() bool {
nLine++
if !s.Scan() {
if s.Err() == nil {
if nLine == 1 {
err = ErrEmptyInput
} else {
err = NewErrUnexpectedData(fmt.Sprintf("pkt-line %d: unexpected EOF", nLine), bytes.Clone(line))
}
} else {
err = s.Err()
}
return false
}
if s.Len() == pktline.Flush {
line = nil
return true
}
line = bytes.TrimSuffix(s.Bytes(), eol)
return true
}
decodeError := func(format string, a ...any) error {
msg := fmt.Sprintf("pkt-line %d: %s", nLine, fmt.Sprintf(format, a...))
return NewErrUnexpectedData(msg, bytes.Clone(line))
}
if !nextLine() {
return err
}
// Check for empty repository (flush packet)
if isFlush(line) {
return ErrEmptyAdvRefs
}
if line := string(line); strings.HasPrefix(line, "version ") {
v, perr := protocol.Parse(line[len("version "):])
if perr != nil {
return perr
}
a.Version = v
if !nextLine() {
return err
}
}
if a.Version != protocol.V0 && a.Version != protocol.V1 {
return decodeError("unsupported protocol version: %d", a.Version)
}
// Check for empty repository (flush packet), which may appear
// either as the first line or after the version line.
if isFlush(line) {
return ErrEmptyAdvRefs
}
// Must have at least a hash
if len(line) < sha1HexSize {
return decodeError("line too short for hash")
}
hash, e := hashFrom(line)
if e != nil {
return decodeError("cannot read hash: %s", e)
}
remain := line[hash.HexSize():]
if hash.IsZero() {
// Empty repo: skip SP "capabilities^{}" NUL
if len(remain) < len(noHeadMark) {
return decodeError("too short zero-id ref")
}
if !bytes.HasPrefix(remain, noHeadMark) {
return decodeError("malformed zero-id ref")
}
remain = remain[len(noHeadMark):]
} else {
// Normal ref: SP refname NUL
if len(remain) < 3 {
return decodeError("line too short after hash")
}
if remain[0] != ' ' {
return decodeError("no space after hash")
}
remain = remain[1:]
chunks := bytes.SplitN(remain, null, 2)
if len(chunks) < 2 {
return decodeError("NULL not found")
}
a.References = append(a.References, plumbing.NewHashReference(
plumbing.ReferenceName(chunks[0]), hash,
))
remain = chunks[1]
}
// Decode capabilities
capability.DecodeList(remain, &a.Capabilities)
// Decode remaining refs and shallows
inShallows := false
for nextLine() {
if len(line) == 0 {
return nil // flush packet
}
if bytes.HasPrefix(line, shallow) {
inShallows = true
data := bytes.TrimPrefix(line, shallow)
if len(data) != sha1HexSize && len(data) != sha256HexSize {
return decodeError("malformed shallow hash: wrong length")
}
h, ok := plumbing.FromHex(string(data))
if !ok {
return decodeError("invalid hash text: %s", string(data))
}
a.Shallows = append(a.Shallows, h)
continue
}
// Once we see shallows, refs cannot follow
if inShallows {
return decodeError("malformed shallow prefix, found ref after shallow")
}
// parse ref line: hash SP refname
name, hash, e := parseRef(line)
if e != nil {
return decodeError("%s", e)
}
a.References = append(a.References, plumbing.NewHashReference(
plumbing.ReferenceName(name), hash,
))
}
return err
}
func hashFrom(line []byte) (plumbing.Hash, error) {
hashSize := bytes.IndexByte(line, ' ')
if hashSize == -1 {
hashSize = len(line)
}
if hashSize != sha1HexSize && hashSize != sha256HexSize {
return plumbing.ZeroHash, fmt.Errorf("cannot read hash, invalid size: %d", hashSize)
}
h, ok := plumbing.FromHex(string(line[:hashSize]))
if !ok {
return plumbing.ZeroHash, fmt.Errorf("invalid hash text: %s", line[:hashSize])
}
return h, nil
}
func parseRef(data []byte) (string, plumbing.Hash, error) {
before, after, ok := bytes.Cut(data, []byte{' '})
if !ok {
return "", plumbing.ZeroHash, fmt.Errorf("malformed ref data: no space")
}
if bytes.IndexByte(after, ' ') != -1 {
return "", plumbing.ZeroHash, fmt.Errorf("malformed ref data: multiple spaces")
}
return string(after), plumbing.NewHash(string(before)), nil
}
package packp
import (
"fmt"
"io"
"sort"
"strings"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/format/pktline"
"github.com/go-git/go-git/v6/plumbing/protocol"
)
// Encode writes the AdvRefs encoding to a writer.
//
// All the payloads will end with a newline character. Capabilities,
// references and shallows are written in alphabetical order, except for
// peeled references that always follow their corresponding references.
func (a *AdvRefs) Encode(w io.Writer) error {
switch a.Version {
case protocol.V0:
case protocol.V1:
if _, err := pktline.Writef(w, "version %d\n", a.Version); err != nil {
return err
}
default:
return fmt.Errorf("unsupported protocol version: %d", a.Version)
}
// Find HEAD or use first ref
firstName, firstHash := a.firstRef()
// Write first line: hash SP refname NUL capabilities
caps := a.Capabilities.String()
if firstName == "" {
// No refs: zero-id capabilities^{}
firstLine := fmt.Sprintf("%s %s\x00%s\n",
plumbing.ZeroHash.String(), "capabilities^{}", caps)
if _, err := pktline.WriteString(w, firstLine); err != nil {
return err
}
} else {
firstLine := fmt.Sprintf("%s %s\x00%s\n",
firstHash.String(), firstName, caps)
if _, err := pktline.WriteString(w, firstLine); err != nil {
return err
}
}
// Build peeled map
peeled := make(map[string]plumbing.Hash)
for _, ref := range a.References {
name := ref.Name().String()
if base, ok := strings.CutSuffix(name, "^{}"); ok {
peeled[base] = ref.Hash()
}
}
// Sort non-peeled refs (excluding HEAD which was already written)
sorted := make([]*plumbing.Reference, 0, len(a.References))
for _, ref := range a.References {
if ref.Name().IsPeeled() || ref.Name().String() == firstName {
continue
}
sorted = append(sorted, ref)
}
sort.Slice(sorted, func(i, j int) bool {
return sorted[i].Name() < sorted[j].Name()
})
// Write refs and their peeled versions
for _, ref := range sorted {
name := ref.Name().String()
if _, err := pktline.Writef(w, "%s %s\n", ref.Hash().String(), name); err != nil {
return err
}
if hash, ok := peeled[name]; ok {
if _, err := pktline.Writef(w, "%s %s^{}\n", hash.String(), name); err != nil {
return err
}
}
}
// Write shallows
if len(a.Shallows) > 0 {
shallowStrs := make([]string, len(a.Shallows))
for i, h := range a.Shallows {
shallowStrs[i] = h.String()
}
sort.Strings(shallowStrs)
for _, h := range shallowStrs {
if _, err := pktline.Writef(w, "shallow %s\n", h); err != nil {
return err
}
}
}
return pktline.WriteFlush(w)
}
// firstRef returns the reference to use as the first line (HEAD or first available).
func (a *AdvRefs) firstRef() (string, plumbing.Hash) {
for _, ref := range a.References {
if ref.Name().IsPeeled() {
continue
}
if ref.Name() == plumbing.HEAD {
return ref.Name().String(), ref.Hash()
}
}
for _, ref := range a.References {
if ref.Name().IsPeeled() {
continue
}
return ref.Name().String(), ref.Hash()
}
return "", plumbing.ZeroHash
}
package packp
import (
"bytes"
"errors"
"fmt"
"io"
"github.com/go-git/go-git/v6/plumbing/format/pktline"
"github.com/go-git/go-git/v6/plumbing/protocol"
"github.com/go-git/go-git/v6/plumbing/protocol/capability"
)
// CapabilityAdv represents a protocol v2 server capability advertisement.
// It includes the version line and the capability lines that follow it.
//
// In protocol v2, the server sends:
//
// version 2\n
// agent=git/2.45.0\n
// ls-refs=unborn\n
// fetch=shallow wait-for-done filter\n
// 0000
//
// Capabilities are one per line in "key" or "key=value" format,
// terminated by a flush packet. This differs from v0/v1 where
// capabilities are space-separated after a NUL byte on the first ref line.
type CapabilityAdv struct {
// Version is the protocol version. Decode sets this to V2.
// Encode writes the version line when Version is V2.
Version protocol.Version
// Capabilities is the parsed list of server capabilities.
Capabilities capability.List
}
// Decode reads a v2 capability advertisement from a pkt-line stream.
// It expects the stream to start with the "version 2\n" line,
// followed by capability lines (one per line), terminated by a flush packet.
func (ca *CapabilityAdv) Decode(r io.Reader) error {
// Read version line first.
l, line, err := pktline.ReadLine(r)
if err != nil {
return err
}
if l < 4 || line == nil {
return errInvalidVersionLine
}
line = bytes.TrimSuffix(line, []byte("\n"))
if !bytes.HasPrefix(line, []byte("version ")) {
return errInvalidVersionLine
}
v, err := protocol.Parse(string(line[8:]))
if err != nil {
return err
}
if v != protocol.V2 {
return fmt.Errorf("unsupported protocol version in capability advertisement: %s", v)
}
ca.Version = v
// Read capability lines until flush.
length, err := DecodeListV2(r, &ca.Capabilities)
if err != nil {
return fmt.Errorf("decoding capability list: %w", err)
}
if length != pktline.Flush {
return fmt.Errorf("expected flush-pkt after capability list, got %04x", length)
}
return nil
}
// Encode writes a v2 capability advertisement to a pkt-line stream.
// It writes the "version N\n" line (where N is ca.Version), then each
// capability on its own line, and terminates with a flush packet.
// Encode returns an error if ca.Version is not V2.
func (ca *CapabilityAdv) Encode(w io.Writer) error {
if ca.Version != protocol.V2 {
return fmt.Errorf("unsupported protocol version for capability advertisement: %s", ca.Version)
}
if _, err := pktline.Writef(w, "version %d\n", ca.Version); err != nil {
return err
}
if err := EncodeListV2(w, &ca.Capabilities); err != nil {
return err
}
return pktline.WriteFlush(w)
}
var errInvalidVersionLine = errors.New("capability advertisement must start with version line")
package packp
import (
"bytes"
"errors"
"fmt"
"io"
"github.com/go-git/go-git/v6/plumbing/format/pktline"
"github.com/go-git/go-git/v6/plumbing/protocol/capability"
)
// CommandArgs is the interface for v2 command-specific arguments.
type CommandArgs interface {
Encoder
Decoder
}
// CommandRequest represents a v2 command request.
//
// Wire format:
//
// request = empty-request | command-request
// empty-request = flush-pkt
// command-request = command
// capability-list
// delim-pkt
// command-args
// flush-pkt
// command = PKT-LINE("command=" key LF)
// command-args = *command-specific-arg
//
// An empty Command encodes as an empty request (a single flush-pkt).
// On decode, a flush-pkt as the first packet leaves Command empty.
type CommandRequest struct {
Command string
Capabilities capability.List
Args CommandArgs
}
// Encode writes the command request to w.
// If Command is empty, it writes a single flush-pkt (empty request).
func (c *CommandRequest) Encode(w io.Writer) error {
if c.Command == "" {
return pktline.WriteFlush(w)
}
if _, err := pktline.Writef(w, "command=%s\n", c.Command); err != nil {
return err
}
if err := EncodeListV2(w, &c.Capabilities); err != nil {
return err
}
if err := pktline.WriteDelim(w); err != nil {
return err
}
if c.Args != nil {
if err := c.Args.Encode(w); err != nil {
return err
}
}
return pktline.WriteFlush(w)
}
// Decode reads a command request from r.
// If the first packet is a flush-pkt, Command is left empty (empty request).
func (c *CommandRequest) Decode(r io.Reader) error {
c.Command = ""
c.Capabilities = capability.List{}
length, line, err := pktline.ReadLine(r)
if err != nil {
if errors.Is(err, io.EOF) {
return nil
}
return err
}
if length == pktline.Flush {
return nil
}
line = bytes.TrimSuffix(line, []byte("\n"))
const prefix = "command="
if !bytes.HasPrefix(line, []byte(prefix)) {
return fmt.Errorf("expected command line, got %q", string(line))
}
c.Command = string(line[len(prefix):])
// Read capabilities until delim-pkt.
length, err = DecodeListV2(r, &c.Capabilities)
if err != nil {
return err
}
if length != pktline.Delim {
return fmt.Errorf("expected delim-pkt after capabilities, got %04x", length)
}
// Read command args until flush-pkt.
if c.Args != nil {
return c.Args.Decode(r)
}
// No args decoder — consume the flush-pkt.
length, _, err = pktline.ReadLine(r)
if err != nil {
return err
}
if length != pktline.Flush {
return fmt.Errorf("expected flush-pkt after empty args, got %04x", length)
}
return nil
}
package packp
import (
"fmt"
)
const (
sha1HexSize = 40
sha256HexSize = 64
)
var (
// common
sp = []byte(" ")
eol = []byte("\n")
// advertised-refs
null = []byte("\x00")
noHeadMark = []byte(" capabilities^{}\x00")
// upload-request
want = []byte("want ")
shallow = []byte("shallow ")
deepen = []byte("deepen")
deepenCommits = []byte("deepen ")
deepenSince = []byte("deepen-since ")
deepenReference = []byte("deepen-not ")
// shallow-update
unshallow = []byte("unshallow ")
// server-response
ack = []byte("ACK")
nak = []byte("NAK")
// updreq
shallowNoSp = []byte("shallow")
)
func isFlush(payload []byte) bool {
return len(payload) == 0
}
// ErrNilWriter is returned when a nil writer is passed to the encoder.
var ErrNilWriter = fmt.Errorf("nil writer")
// ErrUnexpectedData represents an unexpected data decoding a message
type ErrUnexpectedData struct {
Msg string
Data []byte
}
// NewErrUnexpectedData returns a new ErrUnexpectedData containing the data and
// the message given
func NewErrUnexpectedData(msg string, data []byte) error {
return &ErrUnexpectedData{Msg: msg, Data: data}
}
func (err *ErrUnexpectedData) Error() string {
if len(err.Data) == 0 {
return err.Msg
}
return fmt.Sprintf("%s (%s)", err.Msg, err.Data)
}
package packp
import (
"errors"
"fmt"
"io"
"strconv"
"strings"
"time"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/format/pktline"
)
// maxSectionLines bounds how many entries a single fetch section (want, have,
// shallow, ACK, wanted-ref, ...) may contribute on decode. It is a defensive
// backstop against a hostile peer streaming unbounded lines into an in-memory
// slice; it sits far above any legitimate request or response (a single
// negotiation round carries at most a flush-batch of haves, and real repos have
// far fewer than four million refs). It is a var only so tests can lower it.
var maxSectionLines = 1 << 22
// MalformedResponseError reports a server response that violates the
// gitprotocol-v2 grammar: a malformed pkt-line, an unrecognized line within a
// section, an unexpected/repeated/out-of-order section, or a section terminator
// that contradicts the response shape. It mirrors the situations where upstream
// fetch-pack.c calls die() on the response.
type MalformedResponseError struct {
Reason string
}
func (e *MalformedResponseError) Error() string {
return "malformed v2 fetch response: " + e.Reason
}
// FetchArgs represents the arguments for the v2 fetch command.
type FetchArgs struct {
// Wants is the list of object IDs the client wants.
Wants []plumbing.Hash
// Haves is the list of object IDs the client already has.
Haves []plumbing.Hash
// Done indicates the client is done sending wants and haves.
// If false, the client may send additional want/have lines
// in subsequent request rounds (stateful transport only).
Done bool
// ThinPack requests a thin pack if the server supports it.
ThinPack bool
// NoProgress requests that the server suppress progress messages.
NoProgress bool
// IncludeTag requests that the server include tag objects.
IncludeTag bool
// OFSDelta requests that the server use OFS_DELTA objects.
OFSDelta bool
// Shallows is the list of shallow object IDs the client has.
Shallows []plumbing.Hash
// Deepen specifies the number of depth commits to fetch.
Deepen int
// DeepenRelative indicates that deepen is relative to the shallow boundary.
DeepenRelative bool
// DeepenSince specifies a time-based depth constraint.
DeepenSince time.Time
// DeepenNot specifies references to exclude from the shallow boundary.
DeepenNot []string
// Filter specifies a partial clone filter.
Filter Filter
// WaitForDone indicates that the client will wait for the server to send a
// done acknowledgment before sending additional want/have lines.
WaitForDone bool
}
// Encode writes the v2 fetch command arguments to a writer.
// Each argument is written as a separate pkt-line.
// The caller is responsible for writing the delim-pkt before and
// the flush-pkt after these arguments.
func (r *FetchArgs) Encode(w io.Writer) error {
if len(r.Wants) == 0 {
return fmt.Errorf("empty wants provided")
}
wants := append([]plumbing.Hash(nil), r.Wants...)
plumbing.HashesSort(wants)
for _, h := range wants {
if _, err := pktline.Writef(w, "want %s\n", h); err != nil {
return fmt.Errorf("encoding want %q: %w", h, err)
}
}
haves := append([]plumbing.Hash(nil), r.Haves...)
plumbing.HashesSort(haves)
for _, h := range haves {
if _, err := pktline.Writef(w, "have %s\n", h); err != nil {
return fmt.Errorf("encoding have %q: %w", h, err)
}
}
if r.Done {
if _, err := pktline.WriteString(w, "done\n"); err != nil {
return fmt.Errorf("encoding done: %w", err)
}
}
if r.ThinPack {
if _, err := pktline.WriteString(w, "thin-pack\n"); err != nil {
return fmt.Errorf("encoding thin-pack: %w", err)
}
}
if r.NoProgress {
if _, err := pktline.WriteString(w, "no-progress\n"); err != nil {
return fmt.Errorf("encoding no-progress: %w", err)
}
}
if r.IncludeTag {
if _, err := pktline.WriteString(w, "include-tag\n"); err != nil {
return fmt.Errorf("encoding include-tag: %w", err)
}
}
if r.OFSDelta {
if _, err := pktline.WriteString(w, "ofs-delta\n"); err != nil {
return fmt.Errorf("encoding ofs-delta: %w", err)
}
}
shallows := append([]plumbing.Hash(nil), r.Shallows...)
plumbing.HashesSort(shallows)
for _, h := range shallows {
if _, err := pktline.Writef(w, "shallow %s\n", h); err != nil {
return fmt.Errorf("encoding shallow %q: %w", h, err)
}
}
if r.Deepen > 0 {
if _, err := pktline.Writef(w, "deepen %d\n", r.Deepen); err != nil {
return fmt.Errorf("encoding deepen %d: %w", r.Deepen, err)
}
}
if r.DeepenRelative {
// deepen-relative is a flag: the depth is carried by the "deepen <n>"
// line above. Matches git's fetch-pack.c (packet "deepen-relative\n").
if _, err := pktline.WriteString(w, "deepen-relative\n"); err != nil {
return fmt.Errorf("encoding deepen-relative: %w", err)
}
}
if !r.DeepenSince.IsZero() {
if _, err := pktline.Writef(w, "deepen-since %d\n", r.DeepenSince.UTC().Unix()); err != nil {
return fmt.Errorf("encoding deepen-since %s: %w", r.DeepenSince, err)
}
}
for _, ref := range r.DeepenNot {
if _, err := pktline.Writef(w, "deepen-not %s\n", ref); err != nil {
return fmt.Errorf("encoding deepen-not %s: %w", ref, err)
}
}
if r.Filter != "" {
if _, err := pktline.Writef(w, "filter %s\n", r.Filter); err != nil {
return fmt.Errorf("encoding filter %s: %w", r.Filter, err)
}
}
if r.WaitForDone {
if _, err := pktline.WriteString(w, "wait-for-done\n"); err != nil {
return fmt.Errorf("encoding wait-for-done: %w", err)
}
}
return nil
}
// Decode reads v2 fetch command arguments from a reader until a flush-pkt
// is encountered. The caller is responsible for reading the delim-pkt
// and command header before calling Decode.
func (r *FetchArgs) Decode(rd io.Reader) error {
for {
l, pkt, err := pktline.ReadLine(rd)
if err != nil {
if errors.Is(err, io.EOF) {
return nil
}
return err
}
if l == pktline.Flush || l == pktline.Delim {
return nil
}
line := strings.TrimSpace(string(pkt))
if len(line) == 0 {
return nil
}
switch {
case strings.HasPrefix(line, "want "):
h, ok := parseFullHash(line[5:])
if !ok {
return fmt.Errorf("malformed want hash: %q", line[5:])
}
if len(r.Wants) >= maxSectionLines {
return fmt.Errorf("too many want lines (limit %d)", maxSectionLines)
}
r.Wants = append(r.Wants, h)
case strings.HasPrefix(line, "have "):
h, ok := parseFullHash(line[5:])
if !ok {
return fmt.Errorf("malformed have hash: %q", line[5:])
}
if len(r.Haves) >= maxSectionLines {
return fmt.Errorf("too many have lines (limit %d)", maxSectionLines)
}
r.Haves = append(r.Haves, h)
case line == "done":
r.Done = true
case line == "thin-pack":
r.ThinPack = true
case line == "no-progress":
r.NoProgress = true
case line == "include-tag":
r.IncludeTag = true
case line == "ofs-delta":
r.OFSDelta = true
case strings.HasPrefix(line, "shallow "):
h, ok := parseFullHash(line[8:])
if !ok {
return fmt.Errorf("malformed shallow hash: %q", line[8:])
}
if len(r.Shallows) >= maxSectionLines {
return fmt.Errorf("too many shallow lines (limit %d)", maxSectionLines)
}
r.Shallows = append(r.Shallows, h)
case line == "deepen-relative":
r.DeepenRelative = true
case strings.HasPrefix(line, "deepen-relative "):
// Legacy/lenient: the depth belongs to "deepen <n>"; the argument
// here is ignored. git only ever sends the bare flag.
r.DeepenRelative = true
case strings.HasPrefix(line, "deepen-since "):
secs, e := strconv.ParseInt(line[13:], 10, 64)
if e != nil {
return fmt.Errorf("malformed deepen-since: %q", line)
}
r.DeepenSince = time.Unix(secs, 0).UTC()
case strings.HasPrefix(line, "deepen-not "):
if len(r.DeepenNot) >= maxSectionLines {
return fmt.Errorf("too many deepen-not lines (limit %d)", maxSectionLines)
}
r.DeepenNot = append(r.DeepenNot, line[11:])
case strings.HasPrefix(line, "deepen "):
n, e := strconv.Atoi(line[7:])
if e != nil {
return fmt.Errorf("malformed deepen: %q", line)
}
r.Deepen = n
case strings.HasPrefix(line, "filter "):
r.Filter = Filter(line[7:])
case line == "wait-for-done":
r.WaitForDone = true
}
}
}
// Acknowledgments represents the server response to a v2 fetch command's
// acknowledgments section. It is used by the transport layer to determine
// which objects the server has in common with the client.
type Acknowledgments struct {
// ACKs is the list of common object IDs acknowledged by the server.
// Empty list means the server found no common objects (NAK).
ACKs []plumbing.Hash
// Ready indicates the server is ready to send a packfile after the
// acknowledgments section. For stream transports, ready is implied and
// this field is always true.
Ready bool
}
// ShallowInfo represents the server response to a v2 fetch command's
// shallow-info section. It is used by the transport layer to update the
// client's shallow boundary after a fetch.
type ShallowInfo struct {
// Shallows is the list of shallow object IDs sent by the server.
Shallows []plumbing.Hash
// Unshallows is the list of object IDs that are no longer shallow.
Unshallows []plumbing.Hash
}
// WantedRefs represents the server response to a v2 fetch command's
// wanted-refs section. It is used by the transport layer to determine which
// references the server wants the client to have.
type WantedRefs struct {
// Refs is the list of references sent by the server.
Refs []*plumbing.Reference
}
// PackfileURIs represents the server response to a v2 fetch command's
// packfile-uris section. It is used by the transport layer to determine which
// alternate URIs the server suggests for fetching the packfile.
type PackfileURIs struct {
// URIs is the list of alternate URIs the server suggests for fetching the
// packfile.
URIs []string
}
// FetchOutput represents the server response to a v2 fetch command.
//
// The response has explicit sections separated by delim-pkt:
//
// acknowledgments\n
// ACK <oid>\n
// ready\n
// 0001
// shallow-info\n
// shallow <oid>\n
// 0001
// packfile\n
// <sideband packfile data>
// 0000
//
// For HTTP, the transport layer consumes response-end (0002) after Decode returns.
type FetchOutput struct {
// Acknowledgments indicates the server sent an acknowledgments section.
Acknowledgments *Acknowledgments
// ShallowInfo indicates the server sent a shallow-info section.
ShallowInfo *ShallowInfo
// WantedRefs indicates the server sent a wanted-refs section.
WantedRefs *WantedRefs
// PackfileURIs indicates the server sent a packfile-uris section.
PackfileURIs *PackfileURIs
// Packfile reports whether a packfile section follows the metadata
// sections. When true, Decode leaves the reader positioned at the first
// packfile pkt-line so the caller can stream it, and Encode writes the
// "packfile" section header so the caller can write the packfile data.
// When false, the response is a negotiation round
// (acknowledgments flush-pkt) that carries no packfile.
Packfile bool
}
// Decode reads the v2 fetch response from a reader. The response has
// explicit sections separated by delim-pkt:
//
// acknowledgments\n
// ACK <oid>\n
// ready\n
// 0001
// shallow-info\n
// shallow <oid>\n
// 0001
// packfile\n
// <sideband packfile data>
// 0000
//
// A response is one of two shapes (gitprotocol-v2):
//
// output = acknowledgments flush-pkt |
// [acknowledgments delim-pkt] [shallow-info delim-pkt]
// [wanted-refs delim-pkt] [packfile-uris delim-pkt]
// packfile flush-pkt
//
// When a metadata section ends with a flush-pkt (the first shape) the
// response is a negotiation round that carries no packfile, and Decode
// returns with Packfile set to false. When Decode reaches the "packfile"
// section header it sets Packfile to true and returns with the reader
// positioned at the first packfile pkt-line; Decode does not read the
// packfile data, leaving the caller to stream it (demultiplexing the
// sideband as needed).
//
// For HTTP, the transport layer consumes response-end (0002) after
// Decode returns.
func (r *FetchOutput) Decode(rd io.Reader) error {
// Sections appear at most once and in the fixed grammar order
// (acknowledgments < shallow-info < wanted-refs < packfile-uris <
// packfile). lastRank enforces both: a header whose rank is not strictly
// greater than the previous one is a repeat or out-of-order, which upstream
// fetch-pack.c rejects via die(). expectPackfile records that a metadata
// section committed the response to the packfile shape, so a premature
// terminator is also rejected.
lastRank := 0
expectPackfile := false
for {
l, pkt, err := pktline.ReadLine(rd)
if err != nil {
if errors.Is(err, io.EOF) {
// A premature EOF after a metadata section committed the
// response to the packfile shape is a truncated response, not
// a clean end. Match the flush/response-end handling below.
if expectPackfile {
return &MalformedResponseError{Reason: "expected packfile section"}
}
return nil
}
return err
}
// A flush-pkt at the top level ends the response. It is only valid
// before the packfile shape was committed to (a negotiation round, an
// empty response, or a clone that turned out to need nothing).
if l == pktline.Flush || l == pktline.ResponseEnd {
if expectPackfile {
return &MalformedResponseError{Reason: "expected packfile section"}
}
return nil
}
header := strings.TrimSpace(string(pkt))
rank := fetchSectionRank(header)
if rank == 0 {
return &MalformedResponseError{Reason: fmt.Sprintf("unexpected section %q", header)}
}
if rank <= lastRank {
return &MalformedResponseError{Reason: fmt.Sprintf("section %q is repeated or out of order", header)}
}
lastRank = rank
switch header {
case "packfile":
// Leave the reader positioned at the packfile data and let
// the caller stream it. Decode never reads packfile bytes.
r.Packfile = true
return nil
case "acknowledgments":
r.Acknowledgments = &Acknowledgments{}
term, err := r.decodeAcknowledgments(rd)
if err != nil {
return err
}
// ready commits to the packfile shape and must be followed by a
// delim-pkt; otherwise the section is a negotiation round and must
// end the response with a flush-pkt (upstream process_ack).
if r.Acknowledgments.Ready {
if term != pktline.Delim {
return &MalformedResponseError{Reason: "ready acknowledgment must be followed by a delim-pkt"}
}
expectPackfile = true
} else {
if term == pktline.Delim {
return &MalformedResponseError{Reason: "acknowledgments without ready must end the response"}
}
return nil
}
case "shallow-info":
r.ShallowInfo = &ShallowInfo{}
if err := r.decodeMetadataSection(rd, r.decodeShallowInfo); err != nil {
return err
}
expectPackfile = true
case "wanted-refs":
r.WantedRefs = &WantedRefs{}
if err := r.decodeMetadataSection(rd, r.decodeWantedRefs); err != nil {
return err
}
expectPackfile = true
case "packfile-uris":
r.PackfileURIs = &PackfileURIs{}
if err := r.decodeMetadataSection(rd, r.decodePackfileURIs); err != nil {
return err
}
expectPackfile = true
}
}
}
// fetchSectionRank maps a fetch response section header to its position in the
// gitprotocol-v2 grammar, or 0 for an unrecognized header.
func fetchSectionRank(header string) int {
switch header {
case "acknowledgments":
return 1
case "shallow-info":
return 2
case "wanted-refs":
return 3
case "packfile-uris":
return 4
case "packfile":
return 5
default:
return 0
}
}
// decodeMetadataSection runs a section decoder and enforces that the section is
// terminated by a delim-pkt, since every metadata section (shallow-info,
// wanted-refs, packfile-uris) precedes the packfile and is delimited from it.
func (r *FetchOutput) decodeMetadataSection(rd io.Reader, decode func(io.Reader) (int, error)) error {
term, err := decode(rd)
if err != nil {
return err
}
if term != pktline.Delim {
return &MalformedResponseError{Reason: "metadata section must be followed by a delim-pkt"}
}
return nil
}
// Encode writes the v2 fetch response to a writer.
//
// When Packfile is true, Encode writes the present metadata sections
// (acknowledgments, shallow-info, wanted-refs, packfile-uris), each
// terminated by a delim-pkt, followed by the "packfile" section header.
// The caller then streams the packfile data and writes the final
// flush-pkt.
//
// When Packfile is false, the response is a negotiation round: Encode
// writes the acknowledgments section terminated by a flush-pkt and writes
// nothing else. In that case the acknowledgments section must be present
// and must not be ready, and no other metadata sections may be set.
func (r *FetchOutput) Encode(w io.Writer) error {
if !r.Packfile {
if r.Acknowledgments == nil {
return fmt.Errorf("fetch response without a packfile must carry acknowledgments")
}
if r.Acknowledgments.Ready {
return fmt.Errorf("fetch response with ready must carry a packfile")
}
if r.ShallowInfo != nil || r.WantedRefs != nil || r.PackfileURIs != nil {
return fmt.Errorf("fetch response without a packfile cannot carry metadata sections")
}
if _, err := pktline.WriteString(w, "acknowledgments\n"); err != nil {
return err
}
if err := r.encodeAcknowledgments(w); err != nil {
return err
}
return pktline.WriteFlush(w)
}
if r.Acknowledgments != nil {
if _, err := pktline.WriteString(w, "acknowledgments\n"); err != nil {
return err
}
if err := r.encodeAcknowledgments(w); err != nil {
return err
}
if err := pktline.WriteDelim(w); err != nil {
return err
}
}
if r.ShallowInfo != nil {
if _, err := pktline.WriteString(w, "shallow-info\n"); err != nil {
return err
}
if err := r.encodeShallowInfo(w); err != nil {
return err
}
if err := pktline.WriteDelim(w); err != nil {
return err
}
}
if r.WantedRefs != nil {
if _, err := pktline.WriteString(w, "wanted-refs\n"); err != nil {
return err
}
if err := r.encodeWantedRefs(w); err != nil {
return err
}
if err := pktline.WriteDelim(w); err != nil {
return err
}
}
if r.PackfileURIs != nil {
if _, err := pktline.WriteString(w, "packfile-uris\n"); err != nil {
return err
}
if err := r.encodePackfileURIs(w); err != nil {
return err
}
if err := pktline.WriteDelim(w); err != nil {
return err
}
}
// Packfile section header. The caller writes the packfile data and the
// final flush-pkt after this.
if _, err := pktline.WriteString(w, "packfile\n"); err != nil {
return err
}
return nil
}
func (r *FetchOutput) decodeAcknowledgments(rd io.Reader) (int, error) {
for {
l, pkt, err := pktline.ReadLine(rd)
if err != nil {
return 0, err
}
if l == pktline.Delim || l == pktline.Flush || l == pktline.ResponseEnd {
return l, nil
}
line := strings.TrimSpace(string(pkt))
switch {
case strings.HasPrefix(line, "ACK "):
parts := strings.SplitN(line, " ", 2)
if len(parts) < 2 {
return 0, &MalformedResponseError{Reason: fmt.Sprintf("malformed ACK line: %q", line)}
}
h, ok := parseFullHash(strings.TrimSpace(parts[1]))
if !ok {
return 0, &MalformedResponseError{Reason: fmt.Sprintf("malformed ACK hash: %q", parts[1])}
}
if len(r.Acknowledgments.ACKs) >= maxSectionLines {
return 0, &MalformedResponseError{Reason: fmt.Sprintf("too many ACK lines (limit %d)", maxSectionLines)}
}
r.Acknowledgments.ACKs = append(r.Acknowledgments.ACKs, h)
case line == "NAK":
// NAK: no common objects
case line == "ready":
r.Acknowledgments.Ready = true
default:
return 0, &MalformedResponseError{Reason: fmt.Sprintf("unexpected acknowledgments line: %q", line)}
}
}
}
func (r *FetchOutput) decodeShallowInfo(rd io.Reader) (int, error) {
for {
l, pkt, err := pktline.ReadLine(rd)
if err != nil {
return 0, err
}
if l == pktline.Delim || l == pktline.Flush || l == pktline.ResponseEnd {
return l, nil
}
line := strings.TrimSpace(string(pkt))
switch {
case strings.HasPrefix(line, "shallow "):
h, ok := parseFullHash(line[8:])
if !ok {
return 0, &MalformedResponseError{Reason: fmt.Sprintf("malformed shallow hash: %q", line)}
}
if len(r.ShallowInfo.Shallows) >= maxSectionLines {
return 0, &MalformedResponseError{Reason: fmt.Sprintf("too many shallow lines (limit %d)", maxSectionLines)}
}
r.ShallowInfo.Shallows = append(r.ShallowInfo.Shallows, h)
case strings.HasPrefix(line, "unshallow "):
h, ok := parseFullHash(line[10:])
if !ok {
return 0, &MalformedResponseError{Reason: fmt.Sprintf("malformed unshallow hash: %q", line)}
}
if len(r.ShallowInfo.Unshallows) >= maxSectionLines {
return 0, &MalformedResponseError{Reason: fmt.Sprintf("too many unshallow lines (limit %d)", maxSectionLines)}
}
r.ShallowInfo.Unshallows = append(r.ShallowInfo.Unshallows, h)
default:
return 0, &MalformedResponseError{Reason: fmt.Sprintf("expected shallow/unshallow, got: %q", line)}
}
}
}
func (r *FetchOutput) decodeWantedRefs(rd io.Reader) (int, error) {
for {
l, pkt, err := pktline.ReadLine(rd)
if err != nil {
return 0, err
}
if l == pktline.Delim || l == pktline.Flush || l == pktline.ResponseEnd {
return l, nil
}
line := strings.TrimSpace(string(pkt))
parts := strings.SplitN(line, " ", 2)
if len(parts) < 2 {
return 0, &MalformedResponseError{Reason: fmt.Sprintf("malformed wanted-refs line: %q", line)}
}
h, ok := parseFullHash(parts[0])
if !ok {
return 0, &MalformedResponseError{Reason: fmt.Sprintf("malformed wanted-refs hash: %q", parts[0])}
}
if len(r.WantedRefs.Refs) >= maxSectionLines {
return 0, &MalformedResponseError{Reason: fmt.Sprintf("too many wanted-refs lines (limit %d)", maxSectionLines)}
}
r.WantedRefs.Refs = append(r.WantedRefs.Refs,
plumbing.NewHashReference(plumbing.ReferenceName(parts[1]), h),
)
}
}
func (r *FetchOutput) decodePackfileURIs(rd io.Reader) (int, error) {
for {
l, pkt, err := pktline.ReadLine(rd)
if err != nil {
return 0, err
}
if l == pktline.Delim || l == pktline.Flush || l == pktline.ResponseEnd {
return l, nil
}
line := strings.TrimSuffix(string(pkt), "\n")
if len(r.PackfileURIs.URIs) >= maxSectionLines {
return 0, &MalformedResponseError{Reason: fmt.Sprintf("too many packfile-uris lines (limit %d)", maxSectionLines)}
}
r.PackfileURIs.URIs = append(r.PackfileURIs.URIs, line)
}
}
// encodeAcknowledgments writes the acknowledgments body following upstream
// send_acks (upload-pack.c): the ACK lines first, then a single "ready" when
// the server is ready to send a packfile (and nothing after it), otherwise a
// lone "NAK" when there were no common objects. The grammar is
// (nak | *ack) (ready): NAK is mutually exclusive with ACKs and is suppressed
// once ready is sent, and ready always comes last.
func (r *FetchOutput) encodeAcknowledgments(w io.Writer) error {
for _, h := range r.Acknowledgments.ACKs {
if _, err := pktline.Writef(w, "ACK %s\n", h); err != nil {
return err
}
}
if r.Acknowledgments.Ready {
if _, err := pktline.WriteString(w, "ready\n"); err != nil {
return err
}
return nil
}
if len(r.Acknowledgments.ACKs) == 0 {
if _, err := pktline.WriteString(w, "NAK\n"); err != nil {
return err
}
}
return nil
}
func (r *FetchOutput) encodeShallowInfo(w io.Writer) error {
for _, h := range r.ShallowInfo.Shallows {
if _, err := pktline.Writef(w, "shallow %s\n", h); err != nil {
return err
}
}
for _, h := range r.ShallowInfo.Unshallows {
if _, err := pktline.Writef(w, "unshallow %s\n", h); err != nil {
return err
}
}
return nil
}
func (r *FetchOutput) encodeWantedRefs(w io.Writer) error {
for _, ref := range r.WantedRefs.Refs {
if _, err := pktline.Writef(w, "%s %s\n", ref.Hash(), ref.Name()); err != nil {
return err
}
}
return nil
}
func (r *FetchOutput) encodePackfileURIs(w io.Writer) error {
for _, uri := range r.PackfileURIs.URIs {
if _, err := pktline.WriteString(w, uri+"\n"); err != nil {
return err
}
}
return nil
}
package packp
import (
"errors"
"fmt"
"net/url"
"strings"
"github.com/go-git/go-git/v6/plumbing"
)
// ErrUnsupportedObjectFilterType is returned when the filter type is not supported.
var ErrUnsupportedObjectFilterType = errors.New("unsupported object filter type")
// Filter values enable the partial clone capability which causes
// the server to omit objects that match the filter.
//
// See [Git's documentation] for more details.
//
// [Git's documentation]: https://github.com/git/git/blob/e02ecfcc534e2021aae29077a958dd11c3897e4c/Documentation/rev-list-options.txt#L948
type Filter string
// BlobLimitPrefix specifies the unit prefix for blob size limits.
type BlobLimitPrefix string
// Blob limit prefix values.
const (
BlobLimitPrefixNone BlobLimitPrefix = ""
BlobLimitPrefixKibi BlobLimitPrefix = "k"
BlobLimitPrefixMebi BlobLimitPrefix = "m"
BlobLimitPrefixGibi BlobLimitPrefix = "g"
)
// FilterBlobNone omits all blobs.
func FilterBlobNone() Filter {
return "blob:none"
}
// FilterBlobLimit omits blobs of size at least n bytes (when prefix is
// BlobLimitPrefixNone), n kibibytes (when prefix is BlobLimitPrefixKibi),
// n mebibytes (when prefix is BlobLimitPrefixMebi) or n gibibytes (when
// prefix is BlobLimitPrefixGibi). n can be zero, in which case all blobs
// will be omitted.
func FilterBlobLimit(n uint64, prefix BlobLimitPrefix) Filter {
return Filter(fmt.Sprintf("blob:limit=%d%s", n, prefix))
}
// FilterTreeDepth omits all blobs and trees whose depth from the root tree
// is larger or equal to depth.
func FilterTreeDepth(depth uint64) Filter {
return Filter(fmt.Sprintf("tree:%d", depth))
}
// FilterObjectType omits all objects which are not of the requested type t.
// Supported types are TagObject, CommitObject, TreeObject and BlobObject.
func FilterObjectType(t plumbing.ObjectType) (Filter, error) {
switch t {
case plumbing.TagObject:
fallthrough
case plumbing.CommitObject:
fallthrough
case plumbing.TreeObject:
fallthrough
case plumbing.BlobObject:
return Filter(fmt.Sprintf("object:type=%s", t.String())), nil
default:
return "", fmt.Errorf("%w: %s", ErrUnsupportedObjectFilterType, t.String())
}
}
// FilterCombine combines multiple Filter values together.
func FilterCombine(filters ...Filter) Filter {
escapedFilters := make([]string, 0, len(filters))
for _, filter := range filters {
escapedFilters = append(escapedFilters, url.QueryEscape(string(filter)))
}
return Filter(fmt.Sprintf("combine:%s", strings.Join(escapedFilters, "+")))
}
package packp
import (
"fmt"
"io"
"strings"
"github.com/go-git/go-git/v6/plumbing/format/pktline"
)
// ErrInvalidGitProtoRequest is returned by Decode if the input is not a
// valid git protocol request.
var ErrInvalidGitProtoRequest = fmt.Errorf("invalid git protocol request")
// GitProtoRequest is a command request for the git protocol.
// It is used to send the command, endpoint, and extra parameters to the
// remote.
// See https://git-scm.com/docs/pack-protocol#_git_transport
type GitProtoRequest struct {
RequestCommand string
Pathname string
// Optional
Host string
// Optional
ExtraParams []string
}
// validate validates the request.
func (g *GitProtoRequest) validate() error {
if g.RequestCommand == "" {
return fmt.Errorf("%w: empty request command", ErrInvalidGitProtoRequest)
}
// The request is a single pkt-line whose fields are separated by NUL
// bytes ("<command> <pathname>\x00host=<host>\x00\x00<params>..."). A
// control byte in any field breaks that framing or splices in extra
// NUL-delimited fields (a second host=, additional parameters) that the
// caller never set. A git:// URL with a percent-encoded NUL, for example
// "git://host/repo%00host=evil", decodes into exactly such a Pathname, so
// refuse control bytes in every field.
if err := validateGitProtoField("request command", g.RequestCommand); err != nil {
return err
}
if err := validateGitProtoField("pathname", g.Pathname); err != nil {
return err
}
if err := validateGitProtoField("host", g.Host); err != nil {
return err
}
for _, p := range g.ExtraParams {
if err := validateGitProtoField("extra parameter", p); err != nil {
return err
}
}
return nil
}
// validateGitProtoField rejects a request field containing an ASCII control
// byte (0x00-0x1f or 0x7f). Such a byte would break the NUL-framed pkt-line or
// splice in additional fields. No valid command, path, host, or parameter
// contains one. This matches upstream git, which forbids newlines in the host
// and path of a git:// request (git.git a02ea577, CVE-2021-40330), and extends
// it to the full control range including NUL, which a Go string can carry
// through where a C string cannot.
func validateGitProtoField(name, value string) error {
for i := 0; i < len(value); i++ {
if value[i] < 0x20 || value[i] == 0x7f {
return fmt.Errorf("%w: %s contains control byte %#02x",
ErrInvalidGitProtoRequest, name, value[i])
}
}
return nil
}
// Encode encodes the request into the writer.
func (g *GitProtoRequest) Encode(w io.Writer) error {
if w == nil {
return ErrNilWriter
}
if err := g.validate(); err != nil {
return err
}
var req strings.Builder
fmt.Fprintf(&req, "%s %s\x00", g.RequestCommand, g.Pathname)
if host := g.Host; host != "" {
fmt.Fprintf(&req, "host=%s\x00", host)
}
if len(g.ExtraParams) > 0 {
req.WriteString("\x00")
for _, param := range g.ExtraParams {
req.WriteString(param)
req.WriteString("\x00")
}
}
if _, err := pktline.Write(w, []byte(req.String())); err != nil {
return err
}
return nil
}
// Decode decodes the request from the reader.
func (g *GitProtoRequest) Decode(r io.Reader) error {
s := pktline.NewScanner(r)
if !s.Scan() {
if s.Err() == nil {
return ErrInvalidGitProtoRequest
}
return s.Err()
}
if s.Len() == pktline.Flush {
return io.EOF
}
line := s.Text()
if len(line) == 0 {
return io.EOF
}
if line[len(line)-1] != 0 {
return fmt.Errorf("%w: missing null terminator", ErrInvalidGitProtoRequest)
}
parts := strings.SplitN(line, " ", 2)
if len(parts) != 2 {
return fmt.Errorf("%w: short request", ErrInvalidGitProtoRequest)
}
g.RequestCommand = parts[0]
params := strings.Split(parts[1], string(null))
if len(params) < 1 {
return fmt.Errorf("%w: missing pathname", ErrInvalidGitProtoRequest)
}
g.Pathname = params[0]
if len(params) > 1 {
g.Host = strings.TrimPrefix(params[1], "host=")
}
if len(params) > 2 {
for _, param := range params[2:] {
if param != "" {
g.ExtraParams = append(g.ExtraParams, param)
}
}
}
// A decoded request comes straight off the wire from an untrusted peer.
// NUL cannot survive here (it delimits the fields split above), but other
// control bytes such as newline or ESC can, and the server forwards these
// fields into URL construction and log lines. Reject them symmetrically
// with Encode.
return g.validate()
}
package packp
import (
"bufio"
"errors"
"fmt"
"io"
"strings"
"github.com/go-git/go-git/v6/plumbing"
format "github.com/go-git/go-git/v6/plumbing/format/config"
)
// ErrInvalidInfoRefs is returned when an info/refs advertisement holds a line
// that is not an object ID, a tab and a reference name.
var ErrInvalidInfoRefs = errors.New("invalid info/refs")
// InfoRefs represents the information of the references advertised by an
// HTTP dumb server.
type InfoRefs struct {
// References are the hash references, including peeled refs (whose
// names end in ^{}). They are stored in the order received from the
// server.
References []*plumbing.Reference
}
// Decode decodes an InfoRefs from reader.
//
// Every non-empty line must be an object ID in hexadecimal, a tab, and a
// reference name, as git update-server-info writes it. A line that is not
// fails the whole advertisement with ErrInvalidInfoRefs rather than being
// skipped, which mirrors canonical git: parse_info_refs dies on the first line
// whose hash field is not exactly the hash size in hex digits.
//
// Skipping is not a safe alternative here, because a body that is not an
// info/refs at all decodes to something plausible. An HTML page yields no
// references when it holds no tabs, and references named after fragments of
// its own markup when it does — indented markup regularly puts hex-looking
// text before a tab. Either way the caller cannot tell that apart from a
// repository with nothing to advertise.
//
// The hash length is checked before parsing because plumbing.FromHex pads a
// short hex string rather than rejecting it, so "deadbeef" would otherwise
// decode to a reference at a hash the server never sent.
//
// A line too long to scan — over bufio.MaxScanTokenSize — is malformed on the
// same grounds: no advertisement holds one, and a page minified onto a single
// line arrives this way. A failure to read the body is returned unchanged,
// because what did arrive may have been a valid advertisement.
//
// A rejected advertisement leaves i as it was. The references ahead of the
// offending line are not a shorter ref list; they are part of a body that
// turned out not to be a ref list at all.
//
// Errors describe the offending line by position only. The bytes are supplied
// by the server, and the caller is better placed to decide whether any of them
// can be shown: the HTTP transport, for one, quotes a rejected body back only
// when it is plain text.
func (i *InfoRefs) Decode(r io.Reader) error {
var refs []*plumbing.Reference
s := bufio.NewScanner(r)
line := 0
for s.Scan() {
line++
text := s.Text()
if text == "" {
continue
}
hash, name, ok := strings.Cut(text, "\t")
if !ok {
return fmt.Errorf("%w: line %d has no tab", ErrInvalidInfoRefs, line)
}
if len(hash) != format.SHA1HexSize && len(hash) != format.SHA256HexSize {
return fmt.Errorf("%w: line %d has a %d-digit hash", ErrInvalidInfoRefs, line, len(hash))
}
id, valid := plumbing.FromHex(hash)
if !valid {
return fmt.Errorf("%w: line %d has a non-hexadecimal hash", ErrInvalidInfoRefs, line)
}
if name == "" {
return fmt.Errorf("%w: line %d has no reference name", ErrInvalidInfoRefs, line)
}
refs = append(refs, plumbing.NewHashReference(
plumbing.ReferenceName(name), id,
))
}
if err := s.Err(); err != nil {
// A line the scanner cannot hold fails as a malformed line, not as a
// scanner error the caller has no reason to match on. A read failure
// is returned unchanged.
if errors.Is(err, bufio.ErrTooLong) {
return fmt.Errorf("%w: line %d is longer than %d bytes",
ErrInvalidInfoRefs, line+1, bufio.MaxScanTokenSize)
}
return err
}
i.References = append(i.References, refs...)
return nil
}
// Encode encodes an InfoRefs to writer.
func (i *InfoRefs) Encode(w io.Writer) error {
for _, ref := range i.References {
if _, err := fmt.Fprintf(w, "%s\t%s\n", ref.Hash().String(), ref.Name().String()); err != nil {
return err
}
}
return nil
}
package packp
import (
"bytes"
"errors"
"io"
"strings"
"github.com/go-git/go-git/v6/plumbing/format/pktline"
"github.com/go-git/go-git/v6/plumbing/protocol/capability"
)
// EncodeListV2 writes capabilities in v2 format: one capability per pkt-line.
// Each capability is written as "key\n" or "key=value\n" or "key=v1 v2\n". The
// caller is responsible for writing the terminating packet (flush-pkt or
// delim-pkt) after the last capability.
func EncodeListV2(w io.Writer, l *capability.List) error {
for _, key := range l.All() {
values := l.Get(key)
if len(values) == 0 {
if _, err := pktline.Writef(w, "%s\n", key); err != nil {
return err
}
} else {
if _, err := pktline.Writef(w, "%s=%s\n", key, strings.Join(values, " ")); err != nil {
return err
}
}
}
return nil
}
// DecodeListV2 reads capabilities in v2 format from a pkt-line stream. It
// reads pkt-lines until flush-pkt, delim-pkt, or EOF, appending each parsed
// capability to the list. It returns the terminating packet length
// (pktline.Flush, pktline.Delim, or pktline.ResponseEnd) so the caller knows
// what terminated the capability list.
func DecodeListV2(r io.Reader, l *capability.List) (int, error) {
for {
length, line, err := pktline.ReadLine(r)
if err != nil {
if errors.Is(err, io.EOF) {
return pktline.Flush, nil
}
return 0, err
}
if length == pktline.Flush || length == pktline.Delim || length == pktline.ResponseEnd {
return length, nil
}
line = bytes.TrimSuffix(line, []byte("\n"))
if len(line) == 0 {
continue
}
key, value, hasValue := strings.Cut(string(line), "=")
if hasValue {
for v := range strings.SplitSeq(value, " ") {
if v != "" {
l.Add(key, v)
}
}
} else {
l.Add(key)
}
}
}
package packp
import (
"errors"
"fmt"
"io"
"strings"
"unicode"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/format/pktline"
)
// LsRefsArgs represents the arguments for the v2 ls-refs command.
// It is encoded as the command-specific arguments and a flush-pkt in a v2
// command request.
type LsRefsArgs struct {
Peel bool
Symrefs bool
Unborn bool
RefPrefixes []string
}
// Encode writes the ls-refs arguments to a writer. Each argument is
// written as a separate pkt-line. The caller is responsible for writing
// the delim-pkt before and the flush-pkt after these arguments.
func (r *LsRefsArgs) Encode(w io.Writer) error {
// Validate every ref-prefix before writing anything, so an invalid prefix
// can never leave a partially-written arguments section on the stream
// (Encode is all-or-nothing on a validation error).
for _, p := range r.RefPrefixes {
if err := validateRefPrefix(p); err != nil {
return err
}
}
if r.Peel {
if _, err := pktline.WriteString(w, "peel\n"); err != nil {
return err
}
}
if r.Symrefs {
if _, err := pktline.WriteString(w, "symrefs\n"); err != nil {
return err
}
}
if r.Unborn {
if _, err := pktline.WriteString(w, "unborn\n"); err != nil {
return err
}
}
for _, p := range r.RefPrefixes {
if _, err := pktline.Writef(w, "ref-prefix %s\n", p); err != nil {
return err
}
}
return nil
}
// validateRefPrefix rejects a ref-prefix that cannot be safely framed as a
// "ref-prefix <p>" pkt-line. An empty prefix would emit a stray "ref-prefix "
// argument, and whitespace or control bytes (notably LF and NUL) would break
// the pkt-line framing or let a caller inject extra lines. No valid Git
// reference contains such characters, so this only rejects malformed input.
func validateRefPrefix(p string) error {
if p == "" {
return fmt.Errorf("invalid ref-prefix: empty")
}
for _, c := range p {
if c == 0 || unicode.IsControl(c) || unicode.IsSpace(c) {
return fmt.Errorf("invalid ref-prefix %q: contains whitespace or control character", p)
}
}
return nil
}
// tooManyRefPrefixes mirrors ls-refs.c TOO_MANY_PREFIXES: past this many
// ref-prefix arguments, upstream clears the list and advertises every ref, both
// to bound memory and because prefix filtering stops paying off.
const tooManyRefPrefixes = 65536
// Decode reads ls-refs arguments from a reader until a flush-pkt is encountered.
func (r *LsRefsArgs) Decode(rd io.Reader) error {
tooMany := false
for {
l, pkt, err := pktline.ReadLine(rd)
if err != nil {
if errors.Is(err, io.EOF) {
return nil
}
return err
}
if l == pktline.Flush {
return nil
}
line := strings.TrimSuffix(string(pkt), "\n")
if len(line) == 0 {
continue
}
switch {
case line == "peel":
r.Peel = true
case line == "symrefs":
r.Symrefs = true
case line == "unborn":
r.Unborn = true
case strings.HasPrefix(line, "ref-prefix "):
if tooMany {
continue
}
r.RefPrefixes = append(r.RefPrefixes, line[len("ref-prefix "):])
if len(r.RefPrefixes) >= tooManyRefPrefixes {
// Too many prefixes: drop them and advertise every ref, as
// upstream ls-refs.c does, instead of growing without bound.
r.RefPrefixes = nil
tooMany = true
}
}
}
}
// LsRefsOutput represents the server response to an ls-refs command.
//
// Each ref line has the format:
//
// <oid> SP <refname> [SP symref-target:<target>] [SP peeled:<oid>]
//
// or for unborn refs:
//
// unborn SP <refname> SP symref-target:<target>
//
// The response ends with a flush-pkt. For HTTP, response-end (0002) is
// consumed by the transport layer and not seen by Decode.
type LsRefsOutput struct {
References []*plumbing.Reference
}
// Encode writes the ls-refs response lines as pkt-lines following the v2
// grammar: "<oid> SP <refname> [SP symref-target:<target>] [SP peeled:<oid>]",
// or "unborn SP <refname> SP symref-target:<target>" for an unborn HEAD. Peeled
// "^{}" entries are folded into their base ref's line as a peeled attribute, and
// a symbolic ref carries the resolved oid of its target when present. The caller
// is responsible for writing the flush-pkt after these lines.
func (r *LsRefsOutput) Encode(w io.Writer) error {
hashByName := make(map[string]plumbing.Hash, len(r.References))
for _, ref := range r.References {
if ref.Type() == plumbing.HashReference {
hashByName[ref.Name().String()] = ref.Hash()
}
}
for _, ref := range r.References {
name := ref.Name().String()
// Peeled entries are folded into their base ref's line below.
if ref.Name().IsPeeled() {
continue
}
if ref.Type() == plumbing.SymbolicReference {
oid := "unborn"
if h, ok := hashByName[ref.Target().String()]; ok && !h.IsZero() {
oid = h.String()
}
if _, err := pktline.Writef(w, "%s %s symref-target:%s\n", oid, name, ref.Target()); err != nil {
return err
}
continue
}
line := fmt.Sprintf("%s %s", ref.Hash(), name)
if peeled, ok := hashByName[name+"^{}"]; ok {
line += " peeled:" + peeled.String()
}
if _, err := pktline.Writef(w, "%s\n", line); err != nil {
return err
}
}
return nil
}
// Decode reads ref lines until a flush-pkt.
func (r *LsRefsOutput) Decode(rd io.Reader) error {
for {
l, pkt, err := pktline.ReadLine(rd)
if err != nil {
if errors.Is(err, io.EOF) {
return nil
}
return err
}
if l == pktline.Flush {
return nil
}
line := strings.TrimSuffix(string(pkt), "\n")
if len(line) == 0 {
continue
}
refs, err := parseLsRefsLine(line)
if err != nil {
return err
}
r.References = append(r.References, refs...)
}
}
// parseLsRefsLine parses a single ref line from ls-refs output.
// Format: <oid-or-unborn> SP <refname> [SP <attr>...] LF
// Returns one or two references (base + peeled if the peeled attribute is present).
func parseLsRefsLine(line string) ([]*plumbing.Reference, error) {
// Fields tolerates the SP-separated grammar without producing empty tokens
// on repeated spaces: [oid-or-unborn, refname, attr1, attr2, ...].
parts := strings.Fields(line)
if len(parts) < 2 {
return nil, fmt.Errorf("malformed ref line: %q", line)
}
oidStr := parts[0]
refName := plumbing.ReferenceName(parts[1])
var symrefTarget plumbing.ReferenceName
var peeledHash plumbing.Hash
hasPeeled := false
for _, attr := range parts[2:] {
if strings.HasPrefix(attr, "symref-target:") {
symrefTarget = plumbing.ReferenceName(attr[len("symref-target:"):])
} else if strings.HasPrefix(attr, "peeled:") {
h, ok := parseFullHash(attr[len("peeled:"):])
if !ok {
return nil, fmt.Errorf("malformed peeled hash: %q", attr)
}
peeledHash = h
hasPeeled = true
}
}
var refs []*plumbing.Reference
// Handle unborn refs
if oidStr == "unborn" {
if symrefTarget == "" {
return nil, fmt.Errorf("malformed unborn ref line, missing symref-target: %q", line)
}
refs = append(refs, plumbing.NewSymbolicReference(refName, symrefTarget))
return refs, nil
}
// Regular hash ref
hash, ok := parseFullHash(oidStr)
if !ok {
return nil, fmt.Errorf("malformed object id: %q", oidStr)
}
if symrefTarget != "" {
refs = append(refs, plumbing.NewSymbolicReference(refName, symrefTarget))
} else {
refs = append(refs, plumbing.NewHashReference(refName, hash))
}
// If "peeled:" attribute is present, add the peeled ref as a separate entry
if hasPeeled {
refs = append(refs, plumbing.NewHashReference(
plumbing.ReferenceName(refName.String()+"^{}"),
peeledHash,
))
}
return refs, nil
}
// parseFullHash strictly parses a full-length SHA-1 or SHA-256 object id in hex
// form. Object ids on the wire are always full length, so unlike
// plumbing.FromHex (which zero-pads shorter input as a partial SHA-1) it rejects
// anything that is not exactly an object-id length, refusing malformed input.
func parseFullHash(s string) (plumbing.Hash, bool) {
if !plumbing.IsHash(s) {
return plumbing.ZeroHash, false
}
return plumbing.FromHex(s)
}
package packp
import (
"errors"
"fmt"
"io"
"strings"
"unicode"
"github.com/go-git/go-git/v6/plumbing/format/pktline"
)
// ErrInvalidPushOption is returned when a push option contains invalid
// characters.
var ErrInvalidPushOption = errors.New("invalid push option")
// PushOptions represents a list of update request push-options.
//
// See https://git-scm.com/docs/gitprotocol-pack#_reference_update_request_and_packfile_transfer
type PushOptions struct {
Options []string
}
// Encode encodes the push options into the given writer.
func (opts *PushOptions) Encode(w io.Writer) error {
for _, opt := range opts.Options {
if strings.ContainsFunc(opt, isNotGraphic) {
return fmt.Errorf("%w: contains invalid character", ErrInvalidPushOption)
}
if len(opt) > pktline.MaxPayloadSize {
return fmt.Errorf("%w: %w", ErrInvalidPushOption, pktline.ErrPayloadTooLong)
}
}
for _, opt := range opts.Options {
if _, err := pktline.Writef(w, "%s", opt); err != nil {
return err
}
}
return pktline.WriteFlush(w)
}
// Decode decodes the push options from the given reader.
func (opts *PushOptions) Decode(r io.Reader) error {
if opts.Options == nil {
opts.Options = make([]string, 0)
}
s := pktline.NewScanner(r)
flushed := false
for s.Scan() {
if s.Len() == pktline.Flush {
flushed = true
break
}
opt := s.Text()
if strings.ContainsFunc(opt, isNotGraphic) {
return fmt.Errorf("%w: contains invalid character", ErrInvalidPushOption)
}
opts.Options = append(opts.Options, opt)
}
if err := s.Err(); err != nil {
return err
}
if !flushed {
return io.ErrUnexpectedEOF
}
return nil
}
func isNotGraphic(r rune) bool {
return !unicode.IsGraphic(r)
}
package packp
import (
"bytes"
"fmt"
"io"
"strings"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/format/pktline"
)
const (
ok = "ok"
)
// UnpackStatusErr is the error returned when the report status is not ok.
type UnpackStatusErr struct {
Status string
}
// Error implements the error interface.
func (e UnpackStatusErr) Error() string {
return fmt.Sprintf("unpack error: %s", e.Status)
}
// CommandStatusErr is the error returned when the command status is not ok.
type CommandStatusErr struct {
ReferenceName plumbing.ReferenceName
Status string
}
// Error implements the error interface.
func (e CommandStatusErr) Error() string {
return fmt.Sprintf("command error on %s: %s", e.ReferenceName.String(), e.Status)
}
// ReportStatus is a report status message, as used in the git-receive-pack
// process whenever the 'report-status' capability is negotiated.
// The zero value is safe to use.
type ReportStatus struct {
UnpackStatus string
CommandStatuses []*CommandStatus
}
// Error returns the first error if any.
func (s *ReportStatus) Error() error {
if s.UnpackStatus != ok {
return UnpackStatusErr{s.UnpackStatus}
}
for _, cs := range s.CommandStatuses {
if err := cs.Error(); err != nil {
// XXX: Here, we only return the first error following canonical
// Git behavior.
return err
}
}
return nil
}
// Encode writes the report status to a writer.
func (s *ReportStatus) Encode(w io.Writer) error {
if _, err := pktline.Writef(w, "unpack %s\n", s.UnpackStatus); err != nil {
return err
}
for _, cs := range s.CommandStatuses {
if err := cs.encode(w); err != nil {
return err
}
}
return pktline.WriteFlush(w)
}
// Decode reads from the given reader and decodes a report-status message. It
// does not read more input than what is needed to fill the report status.
func (s *ReportStatus) Decode(r io.Reader) error {
sc := pktline.NewScanner(r)
b, err := s.scanFirstLine(sc)
if err != nil {
return err
}
if err := s.decodeReportStatus(b); err != nil {
return err
}
flushed := false
for sc.Scan() {
if sc.Len() == pktline.Flush {
flushed = true
break
}
if err := s.decodeCommandStatus(sc.Bytes()); err != nil {
return err
}
}
if !flushed {
if err := sc.Err(); err != nil {
return fmt.Errorf("missing flush: %w", err)
}
return fmt.Errorf("missing flush: %w", io.ErrUnexpectedEOF)
}
return nil
}
func (s *ReportStatus) scanFirstLine(sc *pktline.Scanner) ([]byte, error) {
if !sc.Scan() {
if sc.Err() == nil {
return nil, io.ErrUnexpectedEOF
}
return nil, sc.Err()
}
return sc.Bytes(), nil
}
func (s *ReportStatus) decodeReportStatus(b []byte) error {
if isFlush(b) {
return fmt.Errorf("premature flush")
}
b = bytes.TrimSuffix(b, eol)
line := string(b)
fields := strings.SplitN(line, " ", 2)
if len(fields) != 2 || fields[0] != "unpack" {
return fmt.Errorf("malformed unpack status: %s", line)
}
s.UnpackStatus = fields[1]
return nil
}
func (s *ReportStatus) decodeCommandStatus(b []byte) error {
b = bytes.TrimSuffix(b, eol)
line := string(b)
fields := strings.SplitN(line, " ", 3)
status := ok
if len(fields) == 3 && fields[0] == "ng" {
status = fields[2]
} else if len(fields) != 2 || fields[0] != "ok" {
return fmt.Errorf("malformed command status: %s", line)
}
cs := &CommandStatus{
ReferenceName: plumbing.ReferenceName(fields[1]),
Status: status,
}
s.CommandStatuses = append(s.CommandStatuses, cs)
return nil
}
// CommandStatus is the status of a reference in a report status.
// See ReportStatus struct.
type CommandStatus struct {
ReferenceName plumbing.ReferenceName
Status string
}
// Error returns the error, if any.
func (s *CommandStatus) Error() error {
if s.Status == ok {
return nil
}
return CommandStatusErr{
ReferenceName: s.ReferenceName,
Status: s.Status,
}
}
func (s *CommandStatus) encode(w io.Writer) error {
if s.Error() == nil {
_, err := pktline.Writef(w, "ok %s\n", s.ReferenceName.String())
return err
}
_, err := pktline.Writef(w, "ng %s %s\n", s.ReferenceName.String(), s.Status)
return err
}
package packp
import (
"bytes"
"fmt"
"io"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/format/pktline"
)
const (
shallowLineLen = 48
unshallowLineLen = 50
)
// ShallowUpdate represents shallow/unshallow updates during fetch.
type ShallowUpdate struct {
Shallows []plumbing.Hash
Unshallows []plumbing.Hash
}
// Decode parses shallow update information from the reader.
func (r *ShallowUpdate) Decode(reader io.Reader) error {
s := pktline.NewScanner(reader)
for s.Scan() {
if s.Len() == pktline.Flush {
return nil
}
line := bytes.TrimSpace(s.Bytes())
var err error
switch {
case bytes.HasPrefix(line, shallow):
err = r.decodeShallowLine(line)
case bytes.HasPrefix(line, unshallow):
err = r.decodeUnshallowLine(line)
default:
err = fmt.Errorf("unexpected shallow line: %q", line)
}
if err != nil {
return err
}
}
return s.Err()
}
func (r *ShallowUpdate) decodeShallowLine(line []byte) error {
hash, err := r.decodeLine(line, shallow, shallowLineLen)
if err != nil {
return err
}
r.Shallows = append(r.Shallows, hash)
return nil
}
func (r *ShallowUpdate) decodeUnshallowLine(line []byte) error {
hash, err := r.decodeLine(line, unshallow, unshallowLineLen)
if err != nil {
return err
}
r.Unshallows = append(r.Unshallows, hash)
return nil
}
func (r *ShallowUpdate) decodeLine(line, prefix []byte, expLen int) (plumbing.Hash, error) {
if len(line) != expLen {
return plumbing.ZeroHash, fmt.Errorf("malformed %s%q", prefix, line)
}
raw := string(line[expLen-40 : expLen])
return plumbing.NewHash(raw), nil
}
// Encode writes the shallow update to the writer.
func (r *ShallowUpdate) Encode(w io.Writer) error {
for _, h := range r.Shallows {
if _, err := pktline.Writef(w, "%s%s\n", shallow, h.String()); err != nil {
return err
}
}
for _, h := range r.Unshallows {
if _, err := pktline.Writef(w, "%s%s\n", unshallow, h.String()); err != nil {
return err
}
}
return pktline.WriteFlush(w)
}
package packp
import (
"bytes"
"errors"
"fmt"
"io"
"strings"
"github.com/go-git/go-git/v6/plumbing/format/pktline"
)
// ErrInvalidSmartReply is returned when a SmartReply is invalid.
var ErrInvalidSmartReply = errors.New("invalid smart reply")
// SmartReply represents Git HTTP smart protocol service payload.
//
// When sending a message over (smart) HTTP, you have to add a pktline before
// the whole thing with the following payload:
//
// '# service=$servicename" LF
//
// Moreover, some if not all, git HTTP smart servers will send a flush-pkt just
// after the first pkt-line.
type SmartReply struct {
Service string
}
// Decode decodes a SmartReply from reader.
func (s *SmartReply) Decode(r io.Reader) error {
sc := pktline.NewScanner(r)
if !sc.Scan() {
if sc.Err() != nil {
return sc.Err()
}
return fmt.Errorf("%w: empty input", ErrInvalidSmartReply)
}
p := sc.Bytes()
if len(p) == 0 || !bytes.HasPrefix(p, []byte("# service=")) {
return fmt.Errorf("%w: %q", ErrInvalidSmartReply, p)
}
s.Service = strings.TrimSpace(string(p[10:]))
if !sc.Scan() {
if sc.Err() != nil {
return sc.Err()
}
return fmt.Errorf("%w: expected flush-pkt", ErrInvalidSmartReply)
}
if sc.Len() != pktline.Flush {
return fmt.Errorf("%w: expected flush-pkt", ErrInvalidSmartReply)
}
return nil
}
// Encode encodes a SmartReply to writer.
func (s *SmartReply) Encode(w io.Writer) error {
if _, err := pktline.Writef(w, "# service=%s\n", s.Service); err != nil {
return err
}
return pktline.WriteFlush(w)
}
package packp
import (
"bytes"
"errors"
"fmt"
"io"
"strings"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/format/pktline"
)
const ackLineLen = 44
// ServerResponse object acknowledgement from upload-pack service
type ServerResponse struct {
ACKs []ACK
}
// ACKStatus represents the status of an object acknowledgement.
type ACKStatus byte
// String returns the string representation of the ACKStatus.
func (s ACKStatus) String() string {
switch s {
case ACKContinue:
return "continue"
case ACKCommon:
return "common"
case ACKReady:
return "ready"
}
return ""
}
// ACKStatus values
const (
ACKContinue ACKStatus = iota + 1
ACKCommon
ACKReady
)
// ACK represents an object acknowledgement. A status can be zero when the
// response doesn't support multi_ack and multi_ack_detailed capabilities.
type ACK struct {
Hash plumbing.Hash
Status ACKStatus
}
// Decode decodes the response into the struct.
func (r *ServerResponse) Decode(reader io.Reader) error {
s := pktline.NewScanner(reader)
for s.Scan() {
if err := r.decodeLine(s.Bytes()); err != nil {
if errors.Is(err, io.EOF) {
return nil
}
return err
}
}
return s.Err()
}
func (r *ServerResponse) decodeLine(line []byte) error {
if len(line) == 0 {
return fmt.Errorf("unexpected flush")
}
if len(line) >= 3 {
if bytes.Equal(line[0:3], ack) {
return r.decodeACKLine(line)
}
if bytes.Equal(line[0:3], nak) {
return io.EOF
}
}
return fmt.Errorf("unexpected content %q", string(line))
}
func (r *ServerResponse) decodeACKLine(line []byte) (err error) {
parts := bytes.Split(line, []byte(" "))
if len(line) < ackLineLen || len(parts) < 2 {
return fmt.Errorf("malformed ACK %q", line)
}
var ack ACK
// TODO: Dynamic hash size and sha256 support
ack.Hash = plumbing.NewHash(string(bytes.TrimSuffix(parts[1], []byte("\n"))))
err = io.EOF
if len(parts) > 2 {
err = nil
switch status := strings.TrimSpace(string(parts[2])); status {
case "continue":
ack.Status = ACKContinue
case "common":
ack.Status = ACKCommon
case "ready":
ack.Status = ACKReady
}
}
r.ACKs = append(r.ACKs, ack)
return err
}
// Encode encodes the ServerResponse into a writer.
func (r *ServerResponse) Encode(w io.Writer) error {
return encodeServerResponse(w, r.ACKs)
}
// encodeServerResponse encodes the ServerResponse into a writer.
func encodeServerResponse(w io.Writer, acks []ACK) error {
if len(acks) == 0 {
_, err := pktline.WriteString(w, string(nak)+"\n")
return err
}
var multiAck bool
for _, a := range acks {
var err error
if a.Status > 0 {
_, err = pktline.Writef(w, "%s %s %s\n", ack, a.Hash, a.Status)
if !multiAck {
multiAck = true
}
} else {
_, err = pktline.Writef(w, "%s %s\n", ack, acks[0].Hash)
}
if err != nil {
return err
}
if !multiAck {
break
}
}
return nil
}
package packp
import (
"time"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/protocol/capability"
)
// UploadRequest values represent the information transmitted on a
// upload-request message. The zero value is safe to use; Wants, Shallows
// and Capabilities can be populated via append.
type UploadRequest struct {
Capabilities capability.List
Wants []plumbing.Hash
Shallows []plumbing.Hash
Depth DepthRequest
Filter Filter
}
// DepthRequest specifies the depth constraints for a fetch request.
// The zero value means no depth constraint (infinite depth).
//
// Commits cannot be combined with Since or NotRefs (git rejects it).
// Since and NotRefs may be combined to further refine the shallow boundary.
type DepthRequest struct {
// Deepen limits the fetch to the given number of commits from the tip.
// Zero means no commit-based depth limit.
// Corresponds to "deepen <n>" in the protocol.
Deepen int
// DeepenSince limits the fetch to commits newer than the given time.
// Zero value means no time-based limit.
// Corresponds to "deepen-since <timestamp>" in the protocol.
DeepenSince time.Time
// DeepenNot excludes commits reachable from the named references.
// Multiple refs may be specified. Each emits a "deepen-not <ref>" line.
DeepenNot []string
}
// IsZero returns true when no depth constraints are set.
func (d DepthRequest) IsZero() bool {
return d.Deepen == 0 && d.DeepenSince.IsZero() && len(d.DeepenNot) == 0
}
package packp
import (
"bytes"
"errors"
"fmt"
"io"
"strconv"
"time"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/format/pktline"
"github.com/go-git/go-git/v6/plumbing/protocol/capability"
)
// ErrDeepenMutuallyExclusive is returned when a request contains both deepen
// and deepen-since/deepen-not specifications.
var ErrDeepenMutuallyExclusive = errors.New("deepen and deepen-since (or deepen-not) cannot be used together")
// Decode reads the next upload-request from its input and
// stores it in the UploadRequest.
func (req *UploadRequest) Decode(r io.Reader) error {
var (
nLine int
line []byte
deepenRevList bool
)
s := pktline.NewScanner(r)
nextLine := func() (hasData bool, err error) {
nLine++
if !s.Scan() {
if s.Err() == nil {
return false, NewErrUnexpectedData(fmt.Sprintf("pkt-line %d: EOF", nLine), bytes.Clone(line))
}
return false, s.Err()
}
if s.Len() == pktline.Flush {
return false, nil
}
line = bytes.TrimSuffix(s.Bytes(), eol)
return true, nil
}
decodeError := func(format string, a ...any) error {
msg := fmt.Sprintf("pkt-line %d: %s", nLine, fmt.Sprintf(format, a...))
return NewErrUnexpectedData(msg, bytes.Clone(line))
}
readHash := func() (plumbing.Hash, error) {
h, err := hashFrom(line)
if err != nil {
return plumbing.ZeroHash, fmt.Errorf("malformed hash: %v", line)
}
line = line[h.HexSize():]
return h, nil
}
// First want line: want <hash>[ capabilities]
ok, err := nextLine()
if err != nil {
return err
}
if !ok {
return fmt.Errorf("empty input")
}
if !bytes.HasPrefix(line, want) {
return decodeError("missing 'want ' prefix")
}
line = bytes.TrimPrefix(line, want)
hash, err := readHash()
if err != nil {
return err
}
req.Wants = append(req.Wants, hash)
// Capabilities (if present after SP)
line = bytes.TrimPrefix(line, sp)
capability.DecodeList(line, &req.Capabilities)
// Additional want lines
for {
ok, err := nextLine()
if err != nil {
return err
}
if !ok || len(line) == 0 {
return nil
}
if !bytes.HasPrefix(line, want) {
break
}
line = bytes.TrimPrefix(line, want)
h, err := readHash()
if err != nil {
return err
}
req.Wants = append(req.Wants, h)
}
for bytes.HasPrefix(line, shallow) {
line = bytes.TrimPrefix(line, shallow)
h, err := readHash()
if err != nil {
return err
}
req.Shallows = append(req.Shallows, h)
ok, err := nextLine()
if err != nil {
return err
}
if !ok || len(line) == 0 {
return nil
}
}
for bytes.HasPrefix(line, deepen) {
switch {
case bytes.HasPrefix(line, deepenCommits):
if deepenRevList {
return ErrDeepenMutuallyExclusive
}
line = bytes.TrimPrefix(line, deepenCommits)
n, err := strconv.Atoi(string(line))
if err != nil {
return err
}
if n < 0 {
return fmt.Errorf("negative depth")
}
req.Depth = DepthRequest{Deepen: n}
case bytes.HasPrefix(line, deepenSince):
if req.Depth.Deepen > 0 {
return ErrDeepenMutuallyExclusive
}
line = bytes.TrimPrefix(line, deepenSince)
secs, err := strconv.ParseInt(string(line), 10, 64)
if err != nil {
return err
}
req.Depth.DeepenSince = time.Unix(secs, 0).UTC()
deepenRevList = true
case bytes.HasPrefix(line, deepenReference):
if req.Depth.Deepen > 0 {
return ErrDeepenMutuallyExclusive
}
line = bytes.TrimPrefix(line, deepenReference)
req.Depth.DeepenNot = append(req.Depth.DeepenNot, string(line))
deepenRevList = true
default:
return decodeError("unexpected deepen specification: %q", line)
}
ok, err := nextLine()
if err != nil {
return err
}
if !ok || len(line) == 0 {
return nil
}
// After deepen <n>, only flush-pkt is valid
if req.Depth.Deepen > 0 {
if bytes.HasPrefix(line, deepenSince) || bytes.HasPrefix(line, deepenReference) {
return ErrDeepenMutuallyExclusive
}
return decodeError("unexpected payload while expecting a flush-pkt: %q", line)
}
// After deepen-since/deepen-not, only deepen-since/deepen-not or flush is valid
if deepenRevList && bytes.HasPrefix(line, deepen) && !bytes.HasPrefix(line, deepenSince) && !bytes.HasPrefix(line, deepenReference) {
return ErrDeepenMutuallyExclusive
}
}
// Unexpected payload after shallows or wants
if len(line) != 0 {
return decodeError("unexpected payload while expecting a flush-pkt: %q", line)
}
return nil
}
package packp
import (
"fmt"
"io"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/format/pktline"
)
// Encode writes the UlReq encoding of u to the stream.
//
// All the payloads will end with a newline character. Wants and
// shallows are sorted alphabetically. A depth of 0 means no depth
// request is sent.
func (req *UploadRequest) Encode(w io.Writer) error {
if len(req.Wants) == 0 {
return fmt.Errorf("empty wants provided")
}
plumbing.HashesSort(req.Wants)
// First want line (with optional capabilities)
if req.Capabilities.IsEmpty() {
if _, err := pktline.Writef(w, "want %s\n", req.Wants[0]); err != nil {
return fmt.Errorf("encoding first want line: %s", err)
}
} else {
if _, err := pktline.Writef(w, "want %s %s\n",
req.Wants[0],
req.Capabilities.String(),
); err != nil {
return fmt.Errorf("encoding first want line: %s", err)
}
}
// Additional wants (deduplicated)
last := req.Wants[0]
for _, h := range req.Wants[1:] {
if last.Compare(h.Bytes()) == 0 {
continue
}
if _, err := pktline.Writef(w, "want %s\n", h); err != nil {
return fmt.Errorf("encoding want %q: %s", h, err)
}
last = h
}
// Shallows (sorted, deduplicated)
plumbing.HashesSort(req.Shallows)
var lastShallow plumbing.Hash
for _, s := range req.Shallows {
if lastShallow.Compare(s.Bytes()) == 0 {
continue
}
if _, err := pktline.Writef(w, "shallow %s\n", s); err != nil {
return fmt.Errorf("encoding shallow %q: %s", s, err)
}
lastShallow = s
}
// Depth
depth := req.Depth
if depth.Deepen > 0 && (!depth.DeepenSince.IsZero() || len(depth.DeepenNot) > 0) {
return ErrDeepenMutuallyExclusive
}
if depth.Deepen > 0 {
if _, err := pktline.Writef(w, "deepen %d\n", depth.Deepen); err != nil {
return fmt.Errorf("encoding depth %d: %s", depth.Deepen, err)
}
}
if !depth.DeepenSince.IsZero() {
when := depth.DeepenSince.UTC()
if _, err := pktline.Writef(w, "deepen-since %d\n", when.Unix()); err != nil {
return fmt.Errorf("encoding depth %s: %s", when, err)
}
}
for _, ref := range depth.DeepenNot {
if _, err := pktline.Writef(w, "deepen-not %s\n", ref); err != nil {
return fmt.Errorf("encoding depth %s: %s", ref, err)
}
}
// Filter
if filter := req.Filter; filter != "" {
if _, err := pktline.Writef(w, "filter %s\n", filter); err != nil {
return fmt.Errorf("encoding filter %s: %s", filter, err)
}
}
return pktline.WriteFlush(w)
}
package packp
import (
"errors"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/protocol/capability"
)
// Errors returned by the updreq package.
var (
ErrEmptyCommands = errors.New("commands cannot be empty")
ErrMalformedCommand = errors.New("malformed command")
)
// UpdateRequests values represent reference upload requests.
// The zero value is safe to use; Commands and Shallows can be populated
// via append.
type UpdateRequests struct {
Capabilities capability.List
Commands []*Command
Shallows []plumbing.Hash
// TODO: Support push-cert
}
func validateUpdateRequests(req *UpdateRequests) error {
if len(req.Commands) == 0 {
return ErrEmptyCommands
}
for _, c := range req.Commands {
if err := c.validate(); err != nil {
return err
}
}
return nil
}
// Action represents the action type of a command.
type Action string
// Action types.
const (
Create Action = "create"
Update Action = "update"
Delete Action = "delete"
Invalid Action = "invalid"
)
// Command represents a command to be executed on a reference.
type Command struct {
Name plumbing.ReferenceName
Old plumbing.Hash
New plumbing.Hash
}
// Action returns the action type of the command.
func (c *Command) Action() Action {
// Compare with IsZero rather than == plumbing.ZeroHash: the latter also
// matches on the object-format field, and a zero object id decoded from
// the wire carries the negotiated format (e.g. sha256 for a 64-hex id)
// while plumbing.ZeroHash is format-unset. IsZero looks only at the
// bytes, mirroring Git's is_null_oid.
if c.Old.IsZero() && c.New.IsZero() {
return Invalid
}
if c.Old.IsZero() {
return Create
}
if c.New.IsZero() {
return Delete
}
return Update
}
func (c *Command) validate() error {
if c.Action() == Invalid {
return ErrMalformedCommand
}
return nil
}
package packp
import (
"bytes"
"errors"
"fmt"
"io"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/format/pktline"
"github.com/go-git/go-git/v6/plumbing/protocol/capability"
)
var (
minCommandLength = sha1HexSize*2 + 2 + 1
minCommandAndCapsLength = minCommandLength + 1
)
// Decode errors.
var (
ErrEmpty = errors.New("empty update-request message")
errNoCommands = errors.New("unexpected EOF before any command")
errMissingCapabilitiesDelimiter = errors.New("capabilities delimiter not found")
errNoFlush = errors.New("unexpected EOF before flush line")
)
func errMalformedRequest(reason string) error {
return fmt.Errorf("malformed request: %s", reason)
}
func errInvalidHash(hash string) error {
return fmt.Errorf("invalid hash: %s", hash)
}
func errInvalidShallowLineLength(got int) error {
return errMalformedRequest(fmt.Sprintf(
"invalid shallow line length: expected %d or %d, got %d",
len(shallow)+sha1HexSize, len(shallow)+sha256HexSize, got,
))
}
func errInvalidCommandCapabilitiesLineLength(got int) error {
return errMalformedRequest(fmt.Sprintf(
"invalid command and capabilities line length: expected at least %d, got %d",
minCommandAndCapsLength, got,
))
}
func errInvalidCommandLineLength(got int) error {
return errMalformedRequest(fmt.Sprintf(
"invalid command line length: expected at least %d, got %d",
minCommandLength, got,
))
}
func errInvalidShallowObjID(err error) error {
return errMalformedRequest(
fmt.Sprintf("invalid shallow object id: %s", err.Error()),
)
}
func errInvalidOldObjID(err error) error {
return errMalformedRequest(
fmt.Sprintf("invalid old object id: %s", err.Error()),
)
}
func errInvalidNewObjID(err error) error {
return errMalformedRequest(
fmt.Sprintf("invalid new object id: %s", err.Error()),
)
}
func errMalformedCommand(err error) error {
return errMalformedRequest(fmt.Sprintf(
"malformed command: %s", err.Error(),
))
}
// Decode reads the next update-request message from the reader.
//
// https://github.com/git/git/blob/1630431f326e15fcde608827b5ff38422528eb59/builtin/receive-pack.c#L2562-L2566
// https://github.com/git/git/blob/1630431f326e15fcde608827b5ff38422528eb59/pkt-line.c#L466-L493
func (req *UpdateRequests) Decode(r io.Reader) error {
var (
payload []byte
length int
)
s := pktline.NewScanner(r)
readLine := func(eofErr error) error {
if !s.Scan() {
if s.Err() == nil {
return eofErr
}
return s.Err()
}
length = s.Len()
if length == pktline.Flush {
payload = nil
} else {
payload = s.Bytes()
}
return nil
}
// Scan first line
if err := readLine(ErrEmpty); err != nil {
return err
}
// Process all consecutive shallow lines
for {
b := bytes.TrimSuffix(payload, eol)
if !bytes.HasPrefix(b, shallowNoSp) {
break
}
hashLen := len(b) - len(shallow)
if hashLen != sha1HexSize && hashLen != sha256HexSize {
return errInvalidShallowLineLength(len(b))
}
h, err := parseHash(string(b[len(shallow):]))
if err != nil {
return errInvalidShallowObjID(err)
}
req.Shallows = append(req.Shallows, h)
if err := readLine(errNoCommands); err != nil {
return err
}
}
// A shallow-only no-op push (shallow lines followed immediately by a
// flush, with no commands) is a valid empty request, e.g. from a shallow
// clone with nothing to push. A bare flush with no shallows is still
// treated as malformed.
if length == pktline.Flush && len(req.Shallows) > 0 {
return nil
}
// The first command line must contain capabilities separated by a null byte
before, after, ok := bytes.Cut(payload, []byte{0})
if !ok {
return errMissingCapabilitiesDelimiter
}
if len(payload) < minCommandAndCapsLength {
return errInvalidCommandCapabilitiesLineLength(len(payload))
}
// Extract and decode capabilities (everything after the null byte)
capability.DecodeList(after, &req.Capabilities)
// Extract the command (everything before the null byte)
cmd, err := parseCommand(before)
if err != nil {
return err
}
req.Commands = append(req.Commands, cmd)
// Read and process remaining commands
for {
if err := readLine(errNoFlush); err != nil {
return err
}
// Stop reading once we reach the flush line
if length == pktline.Flush {
break
}
// Match receive-pack's PACKET_READ_CHOMP_NEWLINE without stripping
// whitespace that belongs to the reference name.
cmd, err := parseCommand(bytes.TrimSuffix(payload, eol))
if err != nil {
return err
}
req.Commands = append(req.Commands, cmd)
}
// We should always have a flush line at the end of the request.
if len(payload) != 0 || length != pktline.Flush {
return errMalformedRequest("unexpected data after flush")
}
return validateUpdateRequests(req)
}
// parseCommand preserves the complete reference name after the two object IDs.
// See https://github.com/git/git/blob/1630431f326e15fcde608827b5ff38422528eb59/builtin/receive-pack.c#L2144-L2152.
func parseCommand(b []byte) (*Command, error) {
if len(b) < minCommandLength {
return nil, errInvalidCommandLineLength(len(b))
}
oldHex, rest, ok := bytes.Cut(b, []byte{' '})
if !ok {
return nil, errMalformedCommand(io.EOF)
}
newHex, name, ok := bytes.Cut(rest, []byte{' '})
if !ok || len(name) == 0 {
return nil, errMalformedCommand(io.EOF)
}
oh, err := parseHash(string(oldHex))
if err != nil {
return nil, errInvalidOldObjID(err)
}
nh, err := parseHash(string(newHex))
if err != nil {
return nil, errInvalidNewObjID(err)
}
// Git's queue_command (builtin/receive-pack.c) keeps the entire remainder
// after the two object IDs. The receive-pack name gate must see whitespace
// in that remainder rather than update a different, truncated reference.
return &Command{Old: oh, New: nh, Name: plumbing.ReferenceName(name)}, nil
}
func parseHash(s string) (plumbing.Hash, error) {
if len(s) != sha1HexSize && len(s) != sha256HexSize {
return plumbing.ZeroHash, errInvalidHash(s)
}
h, ok := plumbing.FromHex(s)
if !ok {
return plumbing.ZeroHash, errInvalidHash(s)
}
return h, nil
}
package packp
import (
"fmt"
"io"
"github.com/go-git/go-git/v6/plumbing/format/pktline"
"github.com/go-git/go-git/v6/plumbing/protocol/capability"
)
// Encode writes the ReferenceUpdateRequest encoding to the stream.
func (req *UpdateRequests) Encode(w io.Writer) error {
if err := validateUpdateRequests(req); err != nil {
return err
}
if err := req.encodeShallow(w); err != nil {
return err
}
if err := req.encodeCommands(w, req.Commands, &req.Capabilities); err != nil {
return err
}
return nil
}
func (req *UpdateRequests) encodeShallow(w io.Writer) error {
for _, h := range req.Shallows {
objID := []byte(h.String())
_, err := pktline.Writef(w, "%s%s", shallow, objID)
if err != nil {
return err
}
}
return nil
}
func (req *UpdateRequests) encodeCommands(w io.Writer,
cmds []*Command, caps *capability.List,
) error {
capStr := caps.String()
if len(capStr) > 0 {
// Canonical Git adds a space before the capabilities.
// See https://github.com/git/git/blob/57da342c786f59eaeb436c18635cc1c7597733d9/send-pack.c#L594
capStr = " " + capStr
}
if _, err := pktline.Writef(w, "%s\x00%s",
formatCommand(cmds[0]), capStr); err != nil {
return err
}
for _, cmd := range cmds[1:] {
if _, err := pktline.Write(w, []byte(formatCommand(cmd))); err != nil {
return err
}
}
return pktline.WriteFlush(w)
}
func formatCommand(cmd *Command) string {
o := cmd.Old.String()
n := cmd.New.String()
return fmt.Sprintf("%s %s %s", o, n, cmd.Name)
}
package packp
import (
"bytes"
"fmt"
"io"
"strings"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/format/pktline"
)
// UploadHaves is a message to signal the references that a client has in a
// upload-pack. Done is true when the client has sent a "done" message.
// Otherwise, it means that the client has more haves to send and this request
// was completed with a flush.
type UploadHaves struct {
Haves []plumbing.Hash
Done bool
}
// Encode encodes the UploadHaves into the Writer.
func (u *UploadHaves) Encode(w io.Writer) error {
plumbing.HashesSort(u.Haves)
var last plumbing.Hash
for _, have := range u.Haves {
if last.Compare(have.Bytes()) == 0 {
continue
}
if _, err := pktline.Writef(w, "have %s\n", have); err != nil {
return fmt.Errorf("sending haves for %q: %w", have, err)
}
last = have
}
if u.Done {
if _, err := pktline.Writeln(w, "done"); err != nil {
return fmt.Errorf("sending done: %w", err)
}
} else {
if err := pktline.WriteFlush(w); err != nil {
return fmt.Errorf("sending flush-pkt: %w", err)
}
}
return nil
}
// Decode decodes the UploadHaves from the Reader.
func (u *UploadHaves) Decode(r io.Reader) error {
u.Haves = make([]plumbing.Hash, 0)
s := pktline.NewScanner(r)
for s.Scan() {
if s.Len() == pktline.Flush {
break
}
line := s.Bytes()
if bytes.HasPrefix(line, []byte("done")) {
u.Done = true
break
}
if !bytes.HasPrefix(line, []byte("have ")) {
return fmt.Errorf("invalid have line: %q", line)
}
have := plumbing.NewHash(strings.TrimSpace(string(line[5:])))
u.Haves = append(u.Haves, have)
}
if err := s.Err(); err != nil {
return fmt.Errorf("decoding haves: %w", err)
}
return nil
}
package transport
import (
"bufio"
"context"
"fmt"
"io"
"strings"
"github.com/go-git/go-git/v6/plumbing/format/pktline"
"github.com/go-git/go-git/v6/plumbing/protocol/packp/sideband"
"github.com/go-git/go-git/v6/utils/ioutil"
)
// ArchiveRequest describes a git-upload-archive request.
type ArchiveRequest struct {
// Args is the list of arguments sent as "argument <arg>\n" pkt-lines.
// These are the same arguments accepted by git-archive:
// e.g. []string{"--format=tar.gz", "--prefix=project/", "HEAD", "src/"}
Args []string
// Progress receives human-readable status from the server (sideband channel 2).
Progress sideband.Progress
}
// Archiver is implemented by Sessions that support git-upload-archive.
// Callers should type-assert their Session to Archiver at the call
// site, following the io.WriterTo / io.ReaderFrom pattern.
type Archiver interface {
Archive(ctx context.Context, req *ArchiveRequest) (io.ReadCloser, error)
}
// Archive speaks the git-upload-archive client wire protocol.
//
// It sends argument pkt-lines to w, closes w, then reads the ACK/NACK
// response and sideband-encoded archive stream from r. The returned
// io.ReadCloser yields archive data (sideband channel 1); closing it
// closes r.
//
// Wire protocol:
//
// Client → Server: "argument <arg>\n" pkt-lines + flush
// Server → Client: "ACK\n" pkt-line + flush
// Server → Client: sideband packets (band 1 = data, band 2 = progress)
func Archive(ctx context.Context, w io.WriteCloser, r io.ReadCloser, req *ArchiveRequest) (io.ReadCloser, error) {
w = ioutil.NewContextWriteCloser(ctx, w)
for _, arg := range req.Args {
if _, err := pktline.WriteString(w, fmt.Sprintf("argument %s\n", arg)); err != nil {
return nil, fmt.Errorf("archive: writing argument: %w", err)
}
}
if err := pktline.WriteFlush(w); err != nil {
return nil, fmt.Errorf("archive: writing flush: %w", err)
}
if err := w.Close(); err != nil {
return nil, fmt.Errorf("archive: closing writer: %w", err)
}
rd := bufio.NewReader(r)
sc := pktline.NewScanner(rd)
if !sc.Scan() {
if sc.Err() != nil {
return nil, fmt.Errorf("archive: reading ACK/NACK: %w", sc.Err())
}
return nil, fmt.Errorf("archive: expected ACK/NACK, got EOF")
}
if sc.Len() == pktline.Flush {
return nil, fmt.Errorf("archive: expected ACK/NACK, got flush")
}
resp := strings.TrimSuffix(sc.Text(), "\n")
switch {
case resp == "ACK":
case strings.HasPrefix(resp, "NACK "):
return nil, fmt.Errorf("archive: NACK %s", resp[5:])
default:
return nil, fmt.Errorf("archive: protocol error: %s", resp)
}
if !sc.Scan() {
if sc.Err() != nil {
return nil, fmt.Errorf("archive: reading flush after ACK: %w", sc.Err())
}
return nil, fmt.Errorf("archive: expected flush after ACK, got EOF")
}
if sc.Len() != pktline.Flush {
return nil, fmt.Errorf("archive: expected flush after ACK, got data")
}
demuxer := sideband.NewDemuxer(sideband.Sideband64k, rd)
if req.Progress != nil {
demuxer.Progress = req.Progress
}
return ioutil.NewReadCloser(demuxer, r), nil
}
package transport
import (
"context"
"io"
"net"
internal "github.com/go-git/go-git/v6/internal/transport"
"github.com/go-git/go-git/v6/plumbing/protocol/packp"
"github.com/go-git/go-git/v6/plumbing/protocol/packp/sideband"
)
// DialContextFunc is the function signature for dialing network connections.
// It also implements proxy.Dialer and proxy.ContextDialer so it can be
// passed directly to proxy.FromURL without an adapter.
type DialContextFunc func(ctx context.Context, network, address string) (net.Conn, error)
// Dial implements proxy.Dialer.
func (f DialContextFunc) Dial(network, addr string) (net.Conn, error) {
return f(context.Background(), network, addr)
}
// DialContext implements proxy.ContextDialer.
func (f DialContextFunc) DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
return f(ctx, network, addr)
}
// RemoteError represents an error returned by the remote.
// TODO: embed error
type RemoteError struct {
Reason string
}
// Error implements the error interface.
func (e *RemoteError) Error() string {
return e.Reason
}
// NewRemoteError creates a new RemoteError.
func NewRemoteError(reason string) error {
return &RemoteError{Reason: reason}
}
// FetchRequest contains the parameters for a fetch-pack request.
// This is used during the pack negotiation phase of the fetch operation.
// See https://git-scm.com/docs/pack-protocol#_packfile_negotiation
// FetchRequest is the request sent to the remote to fetch objects. It is an
// alias of the shared internal type so the v0/v1 and v2 fetch paths use the
// exact same request.
type FetchRequest = internal.FetchRequest
// PushRequest contains the parameters for a push request.
type PushRequest struct {
// Packfile is the packfile reader.
Packfile io.ReadCloser
// Commands is the list of ref update commands to send to the server.
// The caller builds these from refspec matching against local and
// remote refs, including force-push validation and fast-forward
// checks. This matches canonical git's send-pack, which also
// receives pre-built commands from the caller.
Commands []*packp.Command
// Progress is the progress sideband.
Progress sideband.Progress
// Options is a set of push-options to be sent to the server during push.
Options []string
// Atomic indicates an atomic push.
// If the server supports atomic push, it will update the refs in one
// atomic transaction. Either all refs are updated or none.
Atomic bool
// Quiet indicates whether the server should suppress human-readable
// output.
Quiet bool
}
package transport
import (
"errors"
"fmt"
"net/url"
internal "github.com/go-git/go-git/v6/internal/transport"
)
// Transport errors.
var (
ErrRepositoryNotFound = errors.New("repository not found")
ErrEmptyRemoteRepository = errors.New("remote repository is empty")
ErrNoChange = internal.ErrNoChange
ErrAuthenticationRequired = errors.New("authentication required")
ErrAuthorizationFailed = errors.New("authorization failed")
ErrEmptyUploadPackRequest = errors.New("empty git-upload-pack given")
ErrInvalidAuthMethod = errors.New("invalid auth method")
ErrAlreadyConnected = errors.New("session already established")
ErrInvalidRequest = errors.New("invalid request")
)
// Transport capability and support errors.
var (
ErrConnectUnsupported = errors.New("transport does not support raw connections")
ErrArchiveUnsupported = errors.New("transport does not support archive")
ErrCommandUnsupported = errors.New("command is not supported by transport")
ErrProtocolUnsupported = errors.New("protocol version is not supported")
ErrUnsupportedVersion = errors.New("unsupported protocol version")
ErrUnsupportedService = errors.New("unsupported service")
ErrInvalidResponse = errors.New("invalid response")
ErrTimeoutExceeded = errors.New("timeout exceeded")
ErrPackedObjectsNotSupported = errors.New("packed objects not supported")
)
// Negotiation errors.
var (
ErrFilterNotSupported = errors.New("server does not support filters")
ErrShallowNotSupported = errors.New("server does not support shallow clients")
)
// CredentialsDroppedError reports that a redirect chain left the origin
// credentials were issued for, so they were not sent to the origin the chain
// ended at. Withholding is sticky: a chain that leaves the origin and returns
// has still left it.
//
// It never appears alone — a redirect target that serves the repository
// anonymously is a success — and only where a credential existed to withhold.
// It annotates ErrAuthorizationFailed as well as ErrAuthenticationRequired, so
// matching only on the latter misses the failures a 403 produces. The
// annotated error wraps two errors, so errors.Unwrap returns nil on it;
// errors.Is and errors.As are the way in.
//
// var dropped *transport.CredentialsDroppedError
// if errors.As(err, &dropped) {
// // dropped.To is the origin that needs a credential
// }
type CredentialsDroppedError struct {
// From is the repository origin the withheld credential belonged to. It is
// an independent copy carrying scheme and host only.
From *url.URL
// To is the final origin that refused the unauthenticated request, an
// independent copy carrying scheme and host only. It can equal From when a
// chain leaves that origin and returns.
To *url.URL
}
// Error implements the error interface. A nil From or To renders as "<nil>".
func (e *CredentialsDroppedError) Error() string {
return fmt.Sprintf(
"credentials for %s were not sent to %s because a redirect crossed an origin boundary",
e.From, e.To,
)
}
package transport
import (
"context"
"io"
"github.com/go-git/go-git/v6/plumbing/format/packfile"
"github.com/go-git/go-git/v6/plumbing/protocol/capability"
"github.com/go-git/go-git/v6/plumbing/protocol/packp"
"github.com/go-git/go-git/v6/plumbing/protocol/packp/sideband"
"github.com/go-git/go-git/v6/storage"
"github.com/go-git/go-git/v6/utils/ioutil"
)
// FetchPack fetches a packfile from the remote into the given storage.
func FetchPack(
ctx context.Context,
st storage.Storer,
caps capability.List,
packf io.ReadCloser,
shallowInfo *packp.ShallowUpdate,
req *FetchRequest,
) error {
packf = ioutil.NewContextReadCloser(ctx, packf)
var demuxer *sideband.Demuxer
var reader io.Reader = packf
if caps.Supports(capability.Sideband64k) {
demuxer = sideband.NewDemuxer(sideband.Sideband64k, reader)
} else if caps.Supports(capability.Sideband) {
demuxer = sideband.NewDemuxer(sideband.Sideband, reader)
}
if demuxer != nil && req.Progress != nil {
demuxer.Progress = req.Progress
reader = demuxer
}
// A filtered fetch deliberately leaves out objects, so the pack has to be
// recorded as coming from a promisor remote. Git otherwise reads those
// absences as corruption: fsck reports broken links to them and gc fails
// with "unable to read".
//
// The marker is left empty. Git fills it with the refs it sought on this
// path (fetch-pack.c create_promisor_file) and leaves it empty when
// repacking (repack-promisor.c), and accepts either, because only the
// file's presence is ever consulted — packfile.c tests it with access(2)
// and never opens it.
if req.Filter != "" {
if err := packfile.UpdatePromisorObjectStorage(st, reader, ""); err != nil {
return err
}
} else if err := packfile.UpdateObjectStorage(st, reader); err != nil {
return err
}
if err := packf.Close(); err != nil {
return err
}
if shallowInfo != nil {
if err := updateShallow(st, shallowInfo); err != nil {
return err
}
}
return nil
}
func updateShallow(st storage.Storer, shallowInfo *packp.ShallowUpdate) error {
shallows, err := st.Shallow()
if err != nil {
return err
}
outer:
for _, s := range shallowInfo.Shallows {
for _, oldS := range shallows {
if s == oldS {
continue outer
}
}
shallows = append(shallows, s)
}
for _, s := range shallowInfo.Unshallows {
for i, oldS := range shallows {
if s == oldS {
shallows = append(shallows[:i], shallows[i+1:]...)
break
}
}
}
return st.SetShallow(shallows)
}
package http
import "net/http"
// BasicAuth implements HTTP basic authentication.
type BasicAuth struct {
Username, Password string
}
// Authorizer sets basic auth on the HTTP request.
func (a *BasicAuth) Authorizer(r *http.Request) error {
r.SetBasicAuth(a.Username, a.Password)
return nil
}
// TokenAuth implements HTTP bearer token authentication.
type TokenAuth struct {
Token string
}
// Authorizer sets the bearer token on the HTTP request.
func (a *TokenAuth) Authorizer(r *http.Request) error {
r.Header.Set("Authorization", "Bearer "+a.Token)
return nil
}
package http
import (
"bytes"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"unicode"
transport "github.com/go-git/go-git/v6/plumbing/transport"
"github.com/go-git/go-git/v6/utils/trace"
)
// Err represents an HTTP error response.
type Err struct {
// URL is the URL the failing request was made against, an independent copy
// redacted the way Error renders it. Reading the field is as safe as
// reading the message.
URL *url.URL
// Status is the status code of the response.
Status int
// Reason is the message the server sent with the status, empty unless it
// arrived as text/plain. See checkError. It is truncated to a bounded size
// and has every character that would not print as itself replaced by a
// space, so reading the field is as safe as reading the message.
Reason string
}
// StatusCode returns the HTTP status code of the error.
func (e *Err) StatusCode() int { return e.Status }
func (e *Err) Error() string {
format := "unexpected requesting %q status code: %d"
if e.Reason != "" {
// Quoted, because the text is the server's, not this package's. git
// marks the same distinction by prefixing each line with "remote: ";
// an error has no such channel, and quoting also stops a multi-line
// message from forging records in a caller's log.
return fmt.Sprintf(format+": %q", redactedURL(e.URL), e.Status, e.Reason)
}
return fmt.Sprintf(format, redactedURL(e.URL), e.Status)
}
// maxErrorBodySize caps how much of an error response body is read into the
// returned error. The body may come from a server the caller never named — a
// redirect target — so it is not read to EOF.
const maxErrorBodySize = 8 << 10
// maxRedactedComponent caps how long a part of a URL — host, path, query,
// username — may be and still be rendered. Anything longer is replaced whole.
//
// A URL printed here is often a redirect target, so its length is the server's
// choice, and net/http accepts 10 MB of response headers by default. Redacting
// a query costs several times its length, and checkRedirect renders refusals
// eagerly, so an uncapped Location becomes a message the caller then holds.
//
// A kilobyte is far past any real repository URL.
const maxRedactedComponent = 1 << 10
// maxDrainSize caps how much of a spent response body is discarded to keep its
// connection reusable. Sized against what it buys: discarding the tail saves
// one handshake, so spending more transfer than a handshake costs is a bad
// trade. net/http draws the same line tighter still (maxBodySlurpSize, 2 KiB).
const maxDrainSize = 64 << 10
// drainAndClose releases a response body nobody will read again.
//
// What is left of it is discarded before the close, because net/http returns a
// connection to the pool only once its body has reached EOF: closing with
// bytes outstanding drops the connection instead. Those bytes are usually the
// tail of a body that was read for something else — a message taken up to a
// byte cap, a response a decoder left at its flush-pkt — and often just the
// terminating chunk, for which the next request would pay a whole handshake.
//
// The discard stops at maxDrainSize, which bounds a server that keeps sending.
// A server that stops sending without closing is bounded by the request's
// context instead, since net/http ends the read when that context does.
func drainAndClose(body io.ReadCloser) {
_, _ = io.Copy(io.Discard, io.LimitReader(body, maxDrainSize))
_ = body.Close()
}
// contentMediaType returns the media type of a Content-Type header, without
// parameters and lower-cased, mirroring git's extract_content_type().
//
// Not mime.ParseMediaType: it reports ErrInvalidMediaParameter alongside a
// usable media type, which would force a choice between rejecting a smart
// server over a broken charset and ignoring parse errors wholesale.
func contentMediaType(header string) string {
mediaType, _, _ := strings.Cut(header, ";")
return strings.ToLower(strings.TrimSpace(mediaType))
}
// smartContentType reports whether a response advertises the smart protocol
// for service.
func smartContentType(header, service string) bool {
return contentMediaType(header) == "application/x-"+service+"-advertisement"
}
// checkError maps HTTP response status codes to typed transport errors.
//
// The server's message is kept only when it arrives as text/plain, the rule
// git applies in show_http_message. Anything else is markup meant for a
// browser — an SSO interstitial, a CDN error page — and hosting providers send
// the messages they do want a git client to show as text/plain because of that
// rule. Canonical git's own http-backend sends no body at all here, writing
// its reason to the server's stderr instead.
//
// The declared charset is ignored, where git reencodes to its log output
// encoding. A library has no such setting, and its caller is free to reencode
// what it is handed.
//
// git shows a message on the /info/refs GET alone, and discards an RPC
// response body as soon as the status reaches 300. That asymmetry follows from
// streaming the RPC body into the pack parser rather than from a decision to
// hide it, so the rule is applied to every request here: a text/plain refusal
// of a receive-pack POST is what a caller most needs to be told.
//
// The body is consumed and closed whichever it is: the message it yields is
// capped at maxErrorBodySize, the rest is discarded, and then it is closed.
// Discarding it is what keeps the connection — a 404 is ordinary control flow
// for the dumb walk, which asks for every object as a loose file before
// falling back to the packs, so a connection dropped here costs a handshake
// per object.
func checkError(r *http.Response) error {
if r.StatusCode >= http.StatusOK && r.StatusCode < http.StatusMultipleChoices {
return nil
}
var reason string
if r.Body != nil {
if contentMediaType(r.Header.Get("Content-Type")) == "text/plain" {
var message bytes.Buffer
_, _ = message.ReadFrom(io.LimitReader(r.Body, maxErrorBodySize))
reason = strings.TrimSpace(sanitizeReason(message.String()))
}
drainAndClose(r.Body)
}
err := &Err{
// Redacted here rather than in Error, so the field carries the same
// guarantee the message does and does not alias the live request URL.
URL: redactURL(r.Request.URL),
Status: r.StatusCode,
Reason: reason,
}
switch r.StatusCode {
case http.StatusUnauthorized:
return fmt.Errorf("%w: %w", transport.ErrAuthenticationRequired, err)
case http.StatusForbidden:
return fmt.Errorf("%w: %w", transport.ErrAuthorizationFailed, err)
case http.StatusNotFound:
return fmt.Errorf("%w: %w", transport.ErrRepositoryNotFound, err)
}
return err
}
const infoRefsPath = "/info/refs"
// effectiveBase returns base with the path in the spelling the requests built
// from it actually carry.
//
// discovery.request assembles its URL with JoinPath, which cleans the path, and
// applyRedirect recovers the base from the request URL that came back, so a
// caller path of "/repo.git/" is requested as "/repo.git/info/refs" and
// recovered as "/repo.git". Comparing that against the caller's own spelling
// would read an ordinary clone as a repository the server moved:
// Options.Credentials is consulted a second time with Redirected set, and both
// documented ways of scoping a credential to a path decline that call, leaving
// the session anonymous. Deriving the base through the same round trip keeps
// every later comparison between like and like.
//
// Cleaning changes only the spelling, never which resource is named: it
// collapses "//", "/./" and a trailing "/" and leaves %2F alone.
func effectiveBase(base *url.URL) (*url.URL, error) {
// Built exactly as discovery.request builds it, or the two could disagree
// about the spelling this exists to agree on.
origin := &url.URL{Scheme: base.Scheme, Host: base.Host, Path: base.Path, RawPath: base.RawPath}
joined := origin.JoinPath("info/refs").EscapedPath()
// JoinPath leaves a relative path relative, so a repository at the root of
// an origin joins to "info/refs", not "/info/refs". URL.String inserts the
// separator when there is a host, so discovery.request never sends it that
// way; insert it on the same condition to match the path the request carries.
if origin.Host != "" && !strings.HasPrefix(joined, "/") {
joined = "/" + joined
}
if !strings.HasSuffix(joined, infoRefsPath) {
return nil, fmt.Errorf(
"http transport: repository path %q leaves no base to request",
base.EscapedPath(),
)
}
out := *base
// Defensive, and uncoverable: no input reaches this error. EscapedPath
// always returns a valid encoding, and cutting the literal /info/refs tail
// cannot split a %XX sequence because the byte at the cut is "/". Kept so a
// future caller passing a hand-built path cannot slip a broken encoding
// through, and recorded so the missing test is a decision, not an oversight.
if err := setEscapedPath(&out, joined[:len(joined)-len(infoRefsPath)]); err != nil {
return nil, fmt.Errorf(
"http transport: repository path %q is unusable: %w",
base.EscapedPath(), err,
)
}
return &out, nil
}
// applyRedirect derives a new base URL from the final request URL after the
// HTTP client followed any redirects during the /info/refs GET.
//
// It mirrors canonical git's update_url_from_redirect(): strip the
// request-specific "/info/refs" tail to recover the new base. A missing tail
// is an error — git die()s here, because a mismatch could let a server rewrite
// the base to an unrelated repository. The scheme is checked for the same
// reason, keeping a redirect to file:// or gopher:// out of the session;
// cross-scheme redirects permit only the upgrade schemeUpgrade describes.
//
// The path is carried in the spelling the target is written in, never in the
// one it decodes to: on a forge with nested groups "/a%2Fb.git" and "/a/b.git"
// are two repositories, so decoding the escaping away — or letting the base's
// own outlive the path it described — would address a repository neither the
// caller nor the redirect named.
func applyRedirect(resp *http.Response, baseURL *url.URL) (*url.URL, error) {
if resp.Request == nil {
return baseURL, nil
}
final := resp.Request.URL
// Matched against the escaped path: a target ending in "/info%2Frefs" has
// one last segment spelled "info/refs", which is not the discovery request
// coming back.
finalPath := final.EscapedPath()
if !strings.HasSuffix(finalPath, infoRefsPath) {
// Azure DevOps answers an unauthenticated request for a private
// repository with a redirect to /_signin rather than a 401. Report it as
// an authentication challenge, so a caller sees one instead of a
// redirect target that leaves no base to recover.
if strings.HasSuffix(finalPath, "/_signin") {
return nil, fmt.Errorf("%w: redirect to %q", transport.ErrAuthenticationRequired, finalPath)
}
return nil, fmt.Errorf(
"http transport: redirect target %q does not end with %s",
finalPath, infoRefsPath,
)
}
// Cut from the escaped spelling: an index taken there does not fall in the
// same place in the decoded one.
targetPath := finalPath[:len(finalPath)-len(infoRefsPath)]
if final.Host == baseURL.Host &&
final.Scheme == baseURL.Scheme &&
targetPath == baseURL.EscapedPath() {
return baseURL, nil
}
if final.Scheme != "http" && final.Scheme != "https" {
return nil, fmt.Errorf("http transport: redirect to unsupported scheme %q", final.Scheme)
}
if final.Scheme != baseURL.Scheme && !schemeUpgrade(baseURL.Scheme, final.Scheme) {
return nil, fmt.Errorf(
"http transport: redirect changes scheme from %q to %q",
baseURL.Scheme, final.Scheme,
)
}
redirected := *baseURL
redirected.Host = final.Host
redirected.Scheme = final.Scheme
// Uncoverable for the same reason as the call in effectiveBase: targetPath
// is cut from an EscapedPath on a "/". A server chooses this path, so the
// guard stays even though nothing it can send reaches the error.
if err := setEscapedPath(&redirected, targetPath); err != nil {
return nil, fmt.Errorf(
"http transport: redirect target %q has an unusable path: %w",
finalPath, err,
)
}
// The query is the caller's, not the server's: it comes from the repository
// URL and rides on every later request built from this base. Several forges
// accept a credential there (?private_token=, ?job_token=), so it drops
// exactly where a credential drops — gated on credentialsMayFollow rather
// than on "the origin changed at all", so the two rules cannot drift apart.
// The http-to-https upgrade therefore keeps the query: it never rides the
// discovery GET and first leaves on the pack POST, by then over TLS. The
// target's own query is never picked up here, only dropped.
if !credentialsMayFollow(baseURL, &redirected) {
redirected.RawQuery = ""
redirected.ForceQuery = false
}
return &redirected, nil
}
// setEscapedPath sets Path and RawPath so EscapedPath returns escaped exactly.
// It leaves RawPath empty when the decoded path has the same spelling, which is
// what url.Parse stores and keeps a URL built here comparable with one parsed
// from the same string, and rejects invalid encodings rather than silently
// re-escaping them.
func setEscapedPath(u *url.URL, escaped string) error {
decoded, err := url.PathUnescape(escaped)
if err != nil {
return err
}
u.Path = decoded
u.RawPath = ""
if u.EscapedPath() != escaped {
u.RawPath = escaped
}
return nil
}
// schemeUpgrade reports whether the scheme transition from one URL to another
// is the one cross-scheme change go-git permits: a plain-http origin upgrading
// to https.
//
// Permitting it at all is a deliberate deviation: curl, git and the Fetch
// standard all count scheme as part of host identity and drop credentials on
// the upgrade. Auth is sent pre-emptively here, so an http origin has already
// spent its credential in cleartext on the first request; refusing the upgrade
// would break the clone without unspending it. The host is unchanged, where an
// on-path attacker needs a valid certificate to receive anything.
//
// applyRedirect and credentialsMayFollow are both built on this, so the
// permitted direction is decided in one place, and each adds its own
// condition: applyRedirect asks only about the scheme, while
// credentialsMayFollow also requires the default ports, because a port is part
// of an origin. An upgrade from http on 8080 to https on 8443 therefore moves
// the session and leaves the credential behind — the safe direction for a base
// URL is wider than the safe direction for a secret.
func schemeUpgrade(from, to string) bool {
return strings.EqualFold(from, "http") && strings.EqualFold(to, "https")
}
// effectivePort returns u's port as the connection will use it: the scheme's
// well-known port when the URL does not spell one out, and without leading
// zeroes, so "https://x", "https://x:443" and "https://x:0443" all agree.
func effectivePort(u *url.URL) string {
port := u.Port()
if port == "" {
switch strings.ToLower(u.Scheme) {
case "http":
return "80"
case "https":
return "443"
default:
return ""
}
}
if trimmed := strings.TrimLeft(port, "0"); trimmed != "" {
return trimmed
}
return "0"
}
// credentialsMayFollow reports whether credentials issued for one URL may be
// sent to another.
//
// The relation is deliberately asymmetric: scheme, host and effective port
// must all match, except that http on port 80 may upgrade to https on port
// 443 of the same host (see schemeUpgrade), mirroring applyRedirect. Any
// other port pairing is two origins as usual, and the reverse direction never
// follows.
//
// Host matching is exact. Unlike Go's http.Client, which forwards credentials
// from a host to any subdomain of it, a subdomain is a different origin here —
// matching canonical git and libcurl.
func credentialsMayFollow(from, to *url.URL) bool {
// Hostnames are compared as bytes, folding nothing: not ASCII case, not the
// spellings of one address literal, not a trailing root dot, not a unicode
// name against the punycode that encodes it. Byte equality is finer than any
// fold, so it can only find more origin crossings, never fewer; the cost is a
// credential lost across a redirect that merely respells the host, which the
// caller can supply again for the origin the chain reached.
//
// A fold made here that net/http does not make is the dangerous direction:
// no crossing would be recorded on a hop where net/http had already taken
// Authorization away, so nothing would be re-acquired, no
// transport.CredentialsDroppedError would name the origin that challenged,
// and the headers net/http does not know are credentials would travel on.
//
// Hostname panics on a nil URL, deliberately: stripCredentials treats a URL
// it cannot read as a crossing and never reaches here with one.
if from.Hostname() != to.Hostname() {
return false
}
if strings.EqualFold(from.Scheme, to.Scheme) {
return effectivePort(from) == effectivePort(to)
}
return schemeUpgrade(from.Scheme, to.Scheme) &&
effectivePort(from) == "80" && effectivePort(to) == "443"
}
// safeHeaders lists the headers go-git sets itself, none of which can carry a
// caller credential. It has two consumers: trace.HTTP logs only these, and
// stripCredentials keeps only these when a redirect leaves the credential's
// origin. Adding a name here makes it both loggable and forwardable across an
// origin boundary — do not add anything a caller can put a secret in. This
// narrows rather than eliminates the exposure: an Authorizer that writes a
// credential into one of these names directly — for example
// Header.Set("User-Agent", "token "+secret) — still survives a cross-origin
// redirect and still gets logged.
var safeHeaders = map[string]struct{}{
"User-Agent": {},
"Host": {},
"Accept": {},
"Content-Type": {},
"Content-Length": {},
"Cache-Control": {},
"Git-Protocol": {},
"Transfer-Encoding": {},
"Content-Encoding": {},
}
func filterHeaders(h http.Header) http.Header {
filtered := make(http.Header)
for key, values := range h {
if _, ok := safeHeaders[http.CanonicalHeaderKey(key)]; ok {
filtered[key] = values
}
}
return filtered
}
// safeQueryParams lists the query parameters go-git puts on a URL itself,
// against the exact values it writes for them. It is the query-string
// counterpart of safeHeaders, with one difference that decides its shape.
//
// For a header the name is enough, because a caller cannot choose the name
// go-git sends its own headers under. A query parameter is not like that: the
// only "service" this transport writes is the one it chose, but the name is a
// name a forge is free to spell a token with, and transport.Request.Command is
// an unvalidated string, so ?service=<secret> can arrive from either side.
// Matching the value as well as the name is what keeps such an element out of
// error strings and trace output — everything that is not a value below is
// redacted like any other parameter.
//
// Archive discovery is not a third value: git archive discovers through the
// upload-pack endpoint, so "service=" only ever carries one of the two below.
//
// The value match is also what makes the legacy ";" separator harmless.
// net/url does not recognise it, so "service=git-upload-pack;private_token=x"
// arrives here as one element whose value is that whole tail, which no entry
// matches.
//
// Add nothing whose values a caller can choose.
var safeQueryParams = map[string]map[string]struct{}{
"service": {
transport.UploadPackService: {},
transport.ReceivePackService: {},
},
}
// ownQueryParam reports whether a query element is one this transport wrote
// itself, and so may be rendered as it is. A parameter with no value is never
// one: nothing distinguishes a bare flag from a bare secret.
func ownQueryParam(name, value string, hasValue bool) bool {
if !hasValue {
return false
}
_, ok := safeQueryParams[name][value]
return ok
}
// redactedQuery replaces the value of every query parameter that is not
// go-git's own, because a credential in a query string is a pattern several
// forges support (?private_token=, ?job_token=).
//
// The parameter's name survives, so a message still says what was sent. A
// parameter with no value at all is replaced whole: nothing distinguishes a
// bare flag from a bare secret.
//
// A query past maxRedactedComponent is replaced whole rather than walked:
// trimming it to fit would cut inside an element and print the prefix of its
// value. The cap applies to the result too, since replacing values lengthens
// it — a kilobyte of "&" renders as nine. Bounding both ends is what keeps
// redacting twice a no-op, which an *Err needs: it holds a URL that Error
// renders through here again.
//
// It says "REDACTED" where bounded says "TRUNCATED" because every other bare
// word is a valueless element, which this replaces whole.
func redactedQuery(raw string) string {
if raw == "" {
return raw
}
if len(raw) > maxRedactedComponent {
return "REDACTED"
}
var b strings.Builder
b.Grow(len(raw))
for i, rest := 0, raw; ; i++ {
param, tail, more := strings.Cut(rest, "&")
if i > 0 {
b.WriteByte('&')
}
name, value, hasValue := strings.Cut(param, "=")
switch {
// An element go-git wrote itself is rendered as it is. Both halves are
// matched, so a caller-chosen value under one of those names is not.
case ownQueryParam(name, value, hasValue):
b.WriteString(param)
case !hasValue || value == "":
b.WriteString("REDACTED")
default:
b.WriteString(name)
b.WriteString("=REDACTED")
}
if !more {
break
}
rest = tail
}
if b.Len() > maxRedactedComponent {
return "REDACTED"
}
return b.String()
}
// bounded returns s, or "TRUNCATED" when s is longer than
// maxRedactedComponent.
//
// The two words report different things — a length, and a withheld secret —
// and a reader needs to tell them apart. The query is the one part that says
// "REDACTED" for a length; see redactedQuery.
func bounded(s string) string {
if len(s) > maxRedactedComponent {
return "TRUNCATED"
}
return s
}
// sanitizeReason replaces every character in s that would not print as itself
// with a space, so a server's error body renders as text.
//
// The body arrives as the far end wrote it, and unlike a URL nothing has
// escaped it on the way: url.Parse rejects a control character, so no *url.URL
// in this package can carry one, while a body can carry any byte. Rendered
// unchanged, an escape sequence in it redraws the reader's terminal and a
// newline forges a line that reads as a separate message.
//
// unicode.IsPrint excludes both: the C0 and C1 controls that begin such a
// sequence, and the format characters that reorder what follows without
// printing anything themselves.
//
// A space rather than nothing, because removing a character joins the text on
// either side of it into a word the server did not send. Invalid encoding
// decodes to U+FFFD, which is printable and so survives, leaving one
// unreadable character where the bytes were.
func sanitizeReason(s string) string {
return strings.Map(func(r rune) rune {
if unicode.IsPrint(r) {
return r
}
return ' '
}, s)
}
// redactURL returns a copy of u with anything a caller can have put a secret
// in replaced, and every part short enough to print. Nothing the copy holds is
// shared with u but its immutable strings.
//
// Userinfo without a password is left as it is, matching url.URL.Redacted: a
// bare username is an identity, not a secret, and printing it is how a caller
// tells two clone URLs apart. On the paths that print a redirect target —
// checkRedirect's refusals and redactClientError — that username came out of a
// Location header, so a target of the form https://<token>@host/ would have
// its token printed.
//
// Redacting an already-redacted URL returns it unchanged, so a URL kept on an
// error and rendered again comes out the same.
func redactURL(u *url.URL) *url.URL {
if u == nil {
return nil
}
redacted := *u
redacted.Host = bounded(u.Host)
if path := u.EscapedPath(); len(path) > maxRedactedComponent {
redacted.Path, redacted.RawPath = bounded(path), ""
}
redacted.RawQuery = redactedQuery(u.RawQuery)
// The fragment never reaches the wire — net/http omits it from the request
// URI — but it reaches every message this renders.
if u.Fragment != "" {
redacted.Fragment = "REDACTED"
redacted.RawFragment = ""
}
if u.User != nil {
name := bounded(u.User.Username())
if _, hasPassword := u.User.Password(); hasPassword {
redacted.User = url.UserPassword(name, "REDACTED")
} else if name != u.User.Username() {
redacted.User = url.User(name)
}
}
return &redacted
}
// redactedURL renders u the way redactURL redacts it. Every error string and
// trace line in this package prints a URL through it.
func redactedURL(u *url.URL) string {
if u == nil {
return ""
}
return redactURL(u).String()
}
// redactedClientError reports msg in place of err's own message, and unwraps
// to err so errors.Is and errors.As still reach it.
type redactedClientError struct {
msg string
err error
}
func (e *redactedClientError) Error() string { return e.msg }
func (e *redactedClientError) Unwrap() error { return e.err }
// redactClientError rebuilds the message of a *url.Error from client.Do with
// its URL rendered through redactedURL. Any other error is returned unchanged.
//
// net/http copies the Location header into url.Error.URL verbatim, so that URL
// is the redirect target's own choice of bytes: a secret it planted is printed,
// and a megabyte it sent is retained. Every guard here has already run by then.
//
// The wrapped error is bounded but not redacted. net/http builds it from the
// target too — a DNS failure names the host it looked up — but it is prose
// this package does not parse, and Unwrap leaves the original reachable.
//
// The URL is quoted, as url.Error quotes it, so a URL with nothing to redact
// renders the way url.Error renders it. Only a URL this withholds a part of
// reads differently, which is the difference worth seeing.
func redactClientError(err error) error {
var uerr *url.Error
if !errors.As(err, &uerr) {
return err
}
cause := bounded(uerr.Err.Error())
u, perr := url.Parse(uerr.URL)
if perr != nil {
// Omit a URL that will not parse rather than print what made it so.
return &redactedClientError{msg: fmt.Sprintf("%s: %s", uerr.Op, cause), err: err}
}
return &redactedClientError{
msg: fmt.Sprintf("%s %q: %s", uerr.Op, redactedURL(u), cause),
err: err,
}
}
// doRequest performs an HTTP request and returns a typed error on failure.
//
// Every non-2xx status is turned into an error here, so a caller that saw a nil
// error has a 2xx response and need not check the status again.
//
// A response is returned alongside that error, for what it says about the
// request rather than for its body: checkError has taken what it needs of the
// body and closed it. Closing it again is a no-op.
func doRequest(client *http.Client, req *http.Request) (*http.Response, error) {
traceHTTP := trace.HTTP.Enabled()
if traceHTTP {
trace.HTTP.Printf("requesting %s %s %v", req.Method, redactedURL(req.URL), filterHeaders(req.Header))
}
res, err := client.Do(req)
if err != nil {
// The only client.Do in this package, and so the only place the URL
// net/http embeds in its error can be caught.
return nil, redactClientError(err)
}
if traceHTTP {
trace.HTTP.Printf("response %s %s %s %v", res.Proto, res.Status, redactedURL(res.Request.URL), filterHeaders(res.Header))
}
if res.StatusCode >= http.StatusOK && res.StatusCode < http.StatusMultipleChoices {
return res, nil
}
return res, checkError(res)
}
// basicAuth returns an authorizer setting HTTP Basic credentials from userinfo,
// or nil when there is none to set.
func basicAuth(user *url.Userinfo) Authorizer {
if user == nil {
return nil
}
username := user.Username()
password, _ := user.Password()
return func(req *http.Request) error {
req.SetBasicAuth(username, password)
return nil
}
}
// combine returns an authorizer applying each non-nil fn in order, or nil when
// there is nothing to apply. Order matters and later wins: a credential in the
// repository URL is applied before a caller's callback, which may replace it.
func combine(fns ...Authorizer) Authorizer {
kept := make([]Authorizer, 0, len(fns))
for _, fn := range fns {
if fn != nil {
kept = append(kept, fn)
}
}
switch len(kept) {
case 0:
return nil
case 1:
return kept[0]
}
return func(req *http.Request) error {
for _, fn := range kept {
if err := fn(req); err != nil {
return err
}
}
return nil
}
}
// applyAuth authenticates req. A nil authorizer leaves it unauthenticated.
//
// The one credential that does not come through here is the retry's in
// reauthenticate, which applies the authorizer it just composed.
func applyAuth(req *http.Request, authorizer Authorizer) error {
if authorizer == nil {
return nil
}
return authorizer(req)
}
package http
import (
"context"
"net/http"
"net/url"
)
// CredentialRequest identifies what a credential is wanted for. It is the
// argument of a CredentialsFunc.
//
// It always names a server origin; proxy credentials are configured on
// Options.HTTPProxy or Options.Client instead, not asked for here.
//
// The request and its URLs are non-nil and are copies made for that one call:
// read them, do not modify them.
type CredentialRequest struct {
// TargetOrigin is the scheme and host a credential is wanted for. It
// carries no path, query, fragment or userinfo.
TargetOrigin *url.URL
// TargetPath is the repository path on TargetOrigin the credential is
// wanted for, in its escaped on-wire form. A redirect may change it
// without changing the origin.
//
// It is what a source keyed on the path looks the credential up by, which
// is what credential.useHttpPath asks for. It corresponds to git's path
// credential attribute: git derives that from the repository URL too, and
// re-derives it from the redirect target once a redirect has been adopted,
// so the two agree about which path a credential was stored against.
//
// Git's form is decoded and has no leading slash, so building the attribute
// from this takes both:
//
// path, err := url.PathUnescape(strings.TrimPrefix(req.TargetPath, "/"))
//
// Keep the escaped form for anything that compares rather than looks up.
//
// To restrict a credential to the path the caller named, compare it with
// RepositoryURL.EscapedPath():
//
// if req.TargetPath != req.RepositoryURL.EscapedPath() {
// return nil, nil
// }
//
// Compare the escaped form: distinct repository paths can decode alike.
// Spelling is what is compared, so a server respelling a path — "%2E" for
// "." — reads here as a move. On a same-origin redirect the credential has
// already been sent by the time this can decline; the comparison only keeps
// it out of the session that follows.
TargetPath string
// RepositoryURL is the repository URL the caller named, normalized as this
// transport requests it — dot segments and duplicate separators cleaned, a
// trailing slash removed — and with any userinfo removed. Unlike
// TargetOrigin it keeps its path, so a credential scoped to a path under a
// host can be selected. It is the same value however many times a
// credential is asked for during one handshake, including after a redirect.
RepositoryURL *url.URL
// Redirected reports that what is being asked about was reached by
// following a redirect rather than being what the caller named: the origin
// may have moved, the path, or both. It can be true while TargetOrigin is
// still the repository's own origin, since a chain may leave that origin
// and return, or move only the path.
Redirected bool
}
// IsOrigin reports whether a credential held for u may be supplied for this
// request. It applies the origin comparison described on ForOrigin, so a store
// spanning many origins — with nothing to name to ForOrigin up front — can
// range over what it holds:
//
// for _, held := range store.URLs() {
// if req.IsOrigin(held) {
// return store.CredentialFor(held), nil
// }
// }
// return nil, nil
//
// Only u's scheme and host are considered, so a repository URL can be passed
// whole. The comparison is asymmetric, so argument order matters: u is the
// origin the credential is held for, TargetOrigin where the request is about
// to be made.
//
// Nil inputs report false, as does a URL without a host — url.Parse("github.com")
// yields one, because it is a path. A path-scoped store must also compare
// TargetPath.
func (r *CredentialRequest) IsOrigin(u *url.URL) bool {
if r == nil || u == nil || r.TargetOrigin == nil {
return false
}
if u.Host == "" || r.TargetOrigin.Host == "" {
return false
}
return credentialsMayFollow(u, r.TargetOrigin)
}
// Authorizer authenticates an outgoing request by mutating it, typically by
// setting a header.
//
// It is called for every request the credential it belongs to covers, not only
// the first, so a short-lived token should be refreshed inside an Authorizer
// rather than around it. One Transport serves concurrent operations, so an
// Authorizer must be safe for concurrent use.
//
// Requests net/http builds while following a redirect are not authorized
// again; they carry the headers set on the request they redirect from. An
// Authorizer that binds to a specific request — a path MAC, a per-request
// nonce — is therefore not supported across a redirect.
//
// A nil Authorizer leaves the request unauthenticated.
type Authorizer func(*http.Request) error
// Credential is a credential for the origin that was asked about. It is what a
// CredentialsFunc returns. The zero Credential declines: its Authorizer is nil,
// which leaves the request unauthenticated exactly as returning no credential
// does.
type Credential struct {
// Authorizer authenticates an outgoing request by mutating it. See
// Authorizer for when it is called and what it must be safe for, and
// Options.Credentials for the header filtering applied to it if a later
// redirect leaves the origin.
Authorizer Authorizer
}
// CredentialsFunc supplies a credential for the origin a request is about to be
// made to: usually the repository's own origin, and a redirect target when a
// redirect moved the repository to one. req.RepositoryURL names where the caller
// pointed, so the two can be compared.
//
// Answer only for req.TargetOrigin. A redirecting server chooses that origin, so
// an unconditional credential leaks to arbitrary redirect targets. Use
// ForRepositoryOrigin or ForOrigin instead of comparing origins by hand, and
// CredentialRequest.IsOrigin for a store that spans many origins.
//
// A nil *Credential, or one whose Authorizer is nil, declines; an error aborts
// the operation. One Transport serves concurrent operations, so this may be
// called concurrently.
//
// The number of calls is not part of the contract. A credential serves the
// request it was asked about and the session's later requests until a redirect
// moves the origin or the repository path, at which point it is discarded and
// this is called again, so a moved path re-asks even though the origin is
// unchanged. Later versions may ask in further cases, in particular in response
// to a server challenge, which is how canonical git acquires a credential by
// default. An implementation should be inexpensive to call and safe to call
// repeatedly; one that prompts a person will be reached more than once.
//
// The service being run is not reported here, because a credential belongs to
// an origin rather than to an operation, and the git credential protocol has
// no attribute for it either. A source that has to tell a fetch from a push can
// return a Credential whose Authorizer reads the request it is handed.
//
// The adapters below do not restrict repository paths, because a moved or
// renamed repository is what a same-origin redirect is normally for; see
// CredentialRequest.TargetPath to scope a credential to a path. Withholding a
// credential across an origin boundary matches header names, not values;
// Options.Credentials describes what that leaves exposed.
type CredentialsFunc func(ctx context.Context, req *CredentialRequest) (*Credential, error)
// ForOrigin adapts a single authorizer to a CredentialsFunc, supplying it for
// requests targeting origin and declining every other request.
//
// Origins are compared as this transport compares them everywhere — the same
// rule decides ForRepositoryOrigin, CredentialRequest.IsOrigin, and whether a
// credential may follow a redirect:
//
// - Scheme, host and effective port must match; a default port and the same
// port spelled out are equal.
// - Hosts are compared exactly. No subdomains, no case folding, no unicode
// host against its punycode, no trailing root dot, and one address literal
// written two ways is two origins. Configure the origin with the spelling
// the repository URL uses.
// - http on port 80 may upgrade to https on port 443 of the same host,
// because the first request already spent the credential in cleartext.
// The reverse is never permitted, whatever the ports.
//
// Only origin's scheme and host are read, so a repository URL may be passed
// whole, and the origin is copied, so mutating it afterwards does not move the
// gate. Scoping a credential to a path under a host needs a CredentialsFunc
// reading CredentialRequest.TargetPath.
//
// A nil origin or fn, or an origin without a host, declines every request.
// url.Parse("github.com") yields a hostless URL, so spell out the scheme.
func ForOrigin(origin *url.URL, fn Authorizer) CredentialsFunc {
// Reject an empty host explicitly because two empty hosts compare equal.
if origin == nil || fn == nil || origin.Host == "" {
return func(context.Context, *CredentialRequest) (*Credential, error) {
return nil, nil
}
}
held := originOf(origin)
return func(_ context.Context, req *CredentialRequest) (*Credential, error) {
if req == nil || req.TargetOrigin == nil {
return nil, nil
}
if !credentialsMayFollow(held, req.TargetOrigin) {
return nil, nil
}
return &Credential{Authorizer: fn}, nil
}
}
// ForRepositoryOrigin adapts a single authorizer to a CredentialsFunc that
// supplies it when the origin asked about is the origin of the repository the
// caller named, and declines every other origin. Origin comparison, including
// the permitted upgrade, is described on ForOrigin.
//
// A chain that leaves the repository origin and returns is accepted, because
// the credential was already sent there before the redirect. To reject every
// redirected request, wrap this function and check:
//
// if req.Redirected {
// return nil, nil
// }
//
// Repository paths are not restricted; see CredentialRequest.TargetPath to
// scope a credential to one.
func ForRepositoryOrigin(fn Authorizer) CredentialsFunc {
return func(_ context.Context, req *CredentialRequest) (*Credential, error) {
if fn == nil || req == nil || req.TargetOrigin == nil || req.RepositoryURL == nil {
return nil, nil
}
if !credentialsMayFollow(req.RepositoryURL, req.TargetOrigin) {
return nil, nil
}
return &Credential{Authorizer: fn}, nil
}
}
// Chain consults each function in order and takes the first credential
// supplied, so a credential for the repository can sit alongside one for a
// gateway without either having to know about the other.
//
// A nil *Credential and one whose Authorizer is nil are both declines, matching
// how a credential is consumed. An error stops the chain and is returned: a
// source that failed is not a source that declined, and continuing past it
// would silently downgrade a broken credential store to an anonymous request.
// Nil functions are skipped.
func Chain(fns ...CredentialsFunc) CredentialsFunc {
return func(ctx context.Context, req *CredentialRequest) (*Credential, error) {
for _, fn := range fns {
if fn == nil {
continue
}
cred, err := fn(ctx, req)
if err != nil {
return nil, err
}
if cred != nil && cred.Authorizer != nil {
return cred, nil
}
}
return nil, nil
}
}
package http
import (
"context"
"fmt"
"net/http"
"net/url"
"github.com/go-git/go-git/v6/plumbing/protocol"
"github.com/go-git/go-git/v6/plumbing/protocol/capability"
transport "github.com/go-git/go-git/v6/plumbing/transport"
)
// discovery carries what a discovery request is made of, so the request and any
// re-issue of it are built by the same code from the same values.
type discovery struct {
service string
protocol protocol.Version
forceDumb bool
}
// query returns the discovery query, empty when the server is being treated as
// a dumb one.
func (d discovery) query() string {
if d.forceDumb {
return ""
}
return "service=" + d.service
}
// request builds the discovery GET for base.
//
// The URL is assembled from base's scheme, host and path alone — RawPath with
// it, so an escaped segment such as %2F survives — rather than from
// base.String(), which would carry base's userinfo into the request URL, where
// it reaches trace output and error strings, and would bring the clone URL's
// query and fragment along too. Authentication is applied by the caller, never
// from the URL.
func (d discovery) request(ctx context.Context, base *url.URL) (*http.Request, error) {
origin := &url.URL{Scheme: base.Scheme, Host: base.Host, Path: base.Path, RawPath: base.RawPath}
infoURL := origin.JoinPath("info/refs").String()
if q := d.query(); q != "" {
infoURL += "?" + q
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, infoURL, nil)
if err != nil {
return nil, fmt.Errorf("http transport: %w", err)
}
req.Header.Set("User-Agent", capability.DefaultAgent())
if !d.forceDumb {
if gp := transport.GitProtocolEnv(d.protocol); gp != "" {
req.Header.Set("Git-Protocol", gp)
}
}
return req, nil
}
package http
import (
"bufio"
"context"
"crypto"
"errors"
"fmt"
"io"
"io/fs"
"net/http"
"net/url"
"path"
"path/filepath"
"strings"
"github.com/go-git/go-billy/v6"
"github.com/go-git/go-git/v6/plumbing"
formatcfg "github.com/go-git/go-git/v6/plumbing/format/config"
"github.com/go-git/go-git/v6/plumbing/format/idxfile"
"github.com/go-git/go-git/v6/plumbing/format/objfile"
"github.com/go-git/go-git/v6/plumbing/format/packfile"
"github.com/go-git/go-git/v6/plumbing/hash"
"github.com/go-git/go-git/v6/plumbing/object"
"github.com/go-git/go-git/v6/plumbing/protocol/packp"
transport "github.com/go-git/go-git/v6/plumbing/transport"
"github.com/go-git/go-git/v6/storage"
"github.com/go-git/go-git/v6/utils/ioutil"
)
func (s *dumbPackSession) fetchDumb(ctx context.Context, st storage.Storer, req *transport.FetchRequest) error {
if req.Depth != 0 {
return errors.New("dumb http protocol does not support shallow capabilities")
}
fsi, ok := st.(interface {
Filesystem() billy.Filesystem
})
if !ok {
return errors.New("dumb http protocol requires a filesystem")
}
repoFs := fsi.Filesystem()
r := newFetchWalker(ctx, s, st, repoFs)
if err := r.process(); err != nil {
return err
}
if err := r.fetch(); err != nil {
return fmt.Errorf("error fetching objects: %w", err)
}
return nil
}
type fetchWalker struct {
ctx context.Context
client *http.Client
baseURL *url.URL
authorizer Authorizer
st storage.Storer
refs *packp.AdvRefs
fs billy.Filesystem
queue []plumbing.Hash
packIdx map[plumbing.Hash]string
dropped *redirectRecord
}
func newFetchWalker(ctx context.Context, s *dumbPackSession, st storage.Storer, fs billy.Filesystem) *fetchWalker {
return &fetchWalker{
ctx: ctx,
client: s.client,
baseURL: s.baseURL,
authorizer: s.authorizer,
st: st,
refs: s.refs,
fs: fs,
queue: make([]plumbing.Hash, 0),
packIdx: make(map[plumbing.Hash]string),
dropped: s.dropped,
}
}
func (r *fetchWalker) httpGet(urlPath string) (*http.Response, error) {
u, err := url.JoinPath(r.baseURL.String(), urlPath)
if err != nil {
return nil, err
}
req, err := http.NewRequestWithContext(r.ctx, http.MethodGet, u, nil)
if err != nil {
return nil, err
}
if err := applyAuth(req, r.authorizer); err != nil {
return nil, err
}
resp, err := doRequest(r.client, req)
if err != nil {
if resp != nil {
_ = resp.Body.Close()
}
return nil, wrapDropped(r.dropped, err)
}
return resp, nil
}
func (r *fetchWalker) getInfoPacks() ([]string, error) {
res, err := r.httpGet("objects/info/packs")
if err != nil {
return nil, err
}
defer func() { _ = res.Body.Close() }()
var packs []string
s := bufio.NewScanner(res.Body)
for s.Scan() {
line := s.Text()
h := strings.TrimPrefix(line, "P pack-")
h = strings.TrimSuffix(h, ".pack")
packs = append(packs, h)
}
return packs, s.Err()
}
func (r *fetchWalker) downloadFile(fp string) (rErr error) {
res, err := r.httpGet(fp)
if err != nil {
return err
}
if res.StatusCode != http.StatusOK {
_ = res.Body.Close()
return fmt.Errorf("unexpected status code: %d", res.StatusCode)
}
f, err := r.fs.TempFile(filepath.Dir(fp), filepath.Base(fp)+".temp")
if err != nil {
// The body is an object or pack, so there is nothing to gain from
// draining a file this walk has given up on writing.
_ = res.Body.Close()
return err
}
defer func() {
if err := f.Close(); err != nil {
rErr = err
}
}()
if _, err := ioutil.CopyBufferPool(f, res.Body); err != nil {
// As above: an object or a pack, so a walk that has given up on
// writing it gains nothing from draining what is left.
_ = res.Body.Close()
return err
}
if err := res.Body.Close(); err != nil {
return err
}
return r.fs.Rename(f.Name(), fp)
}
func (r *fetchWalker) getHead() (ref *plumbing.Reference, err error) {
res, err := r.httpGet("HEAD")
if err != nil {
return nil, err
}
defer func() {
if res.Body != nil {
bodyErr := res.Body.Close()
if err == nil {
err = bodyErr
}
}
}()
s := bufio.NewScanner(res.Body)
if !s.Scan() {
if err := s.Err(); err != nil {
return nil, err
}
return nil, transport.ErrRepositoryNotFound
}
line := s.Text()
if target, found := strings.CutPrefix(line, "ref: "); found {
return plumbing.NewSymbolicReference(plumbing.HEAD, plumbing.ReferenceName(target)), nil
}
return plumbing.NewHashReference(plumbing.HEAD, plumbing.NewHash(line)), nil
}
func (r *fetchWalker) process() error {
var head plumbing.Hash
if headRef, err := r.refs.Head(); err != nil {
h, err := r.getHead()
if err != nil {
return err
}
switch h.Type() {
case plumbing.HashReference:
head = h.Hash()
r.refs.References = append([]*plumbing.Reference{h}, r.refs.References...)
case plumbing.SymbolicReference:
for _, ref := range r.refs.References {
if ref.Name().String() == h.Target().String() {
head = ref.Hash()
break
}
}
}
} else {
head = headRef.Hash()
}
if head.IsZero() {
return transport.ErrRepositoryNotFound
}
infoPacks, err := r.getInfoPacks()
if err != nil {
return err
}
for _, h := range infoPacks {
ph := plumbing.NewHash(h)
if ph.IsZero() {
continue
}
packIdx := path.Join("objects", "pack", fmt.Sprintf("pack-%s.idx", h))
if _, err := r.fs.Stat(packIdx); errors.Is(err, fs.ErrExist) {
r.packIdx[ph] = packIdx
} else {
if err := r.downloadFile(packIdx); err != nil {
return err
}
r.packIdx[ph] = packIdx
}
}
r.queue = append(r.queue, head)
for _, ref := range r.refs.References {
if r.st.HasEncodedObject(ref.Hash()) != nil {
r.queue = append(r.queue, ref.Hash())
}
}
r.queue = append(r.queue, head)
return nil
}
func (r *fetchWalker) fetchObject(objHash plumbing.Hash, obj plumbing.EncodedObject) (err error) {
if r.st.HasEncodedObject(objHash) == nil {
return nil
}
h := objHash.String()
res, err := r.httpGet(path.Join("objects", h[:2], h[2:]))
if errors.Is(err, transport.ErrRepositoryNotFound) {
return io.EOF
}
if err != nil {
return err
}
defer func() { _ = res.Body.Close() }()
switch res.StatusCode {
case http.StatusOK:
case http.StatusNotFound:
return io.EOF
default:
return fmt.Errorf("unexpected status code: %d", res.StatusCode)
}
rd, err := objfile.NewReader(res.Body, objectFormatFromHash(objHash))
if err != nil {
return err
}
ioutil.CheckClose(rd, &err)
t, size, err := rd.Header()
if err != nil {
return err
}
obj.SetType(t)
obj.SetSize(size)
w, err := obj.Writer()
if err != nil {
return err
}
ioutil.CheckClose(w, &err)
if _, err := ioutil.CopyBufferPool(w, rd); err != nil {
return err
}
return nil
}
func objectFormatFromHash(h plumbing.Hash) formatcfg.ObjectFormat {
if h.HexSize() == formatcfg.SHA256HexSize {
return formatcfg.SHA256
}
return formatcfg.SHA1
}
func (r *fetchWalker) fetch() error {
packs := map[string]struct{}{}
processed := map[string]struct{}{}
indicies := []*idxfile.MemoryIndex{}
LOOP:
for len(r.queue) > 0 {
objHash := r.queue[0]
r.queue = r.queue[1:]
if _, ok := processed[objHash.String()]; ok {
continue
}
for _, idx := range indicies {
if ok, err := idx.Contains(objHash); err == nil && ok {
continue LOOP
}
}
obj := r.st.NewEncodedObject()
err := r.fetchObject(objHash, obj)
if errors.Is(err, io.EOF) {
for packHash, packIdxPath := range r.packIdx {
idxFile, err := r.fs.Open(packIdxPath)
if err != nil {
return fmt.Errorf("error opening index file: %w", err)
}
var hasher hash.Hash
if packHash.Size() == crypto.SHA256.Size() {
hasher = hash.New(crypto.SHA256)
} else {
hasher = hash.New(crypto.SHA1)
}
idx := idxfile.NewMemoryIndex(packHash.Size())
d := idxfile.NewDecoder(idxFile, hasher)
if err := d.Decode(idx); err != nil {
_ = idxFile.Close()
return fmt.Errorf("error decoding index file: %w", err)
}
indicies = append(indicies, idx)
packPath := path.Join("objects", "pack", fmt.Sprintf("pack-%s.pack", packHash.String()))
if ok, err := idx.Contains(objHash); err == nil && ok {
processed[objHash.String()] = struct{}{}
if _, ok := packs[packPath]; ok {
continue LOOP
}
if _, err := r.fs.Stat(packPath); errors.Is(err, fs.ErrExist) {
packs[packPath] = struct{}{}
continue LOOP
}
if err := r.downloadFile(packPath); err != nil {
return fmt.Errorf("error downloading pack file: %w", err)
}
packs[packPath] = struct{}{}
continue LOOP
}
}
} else if err != nil {
return err
}
switch obj.Type() {
case plumbing.CommitObject:
commit, err := object.DecodeCommit(r.st, obj)
if err != nil {
return err
}
r.queue = append(r.queue, commit.ParentHashes...)
r.queue = append(r.queue, commit.TreeHash)
case plumbing.TreeObject:
tree, err := object.DecodeTree(r.st, obj)
if err != nil {
return err
}
r.queue = append(r.queue, tree.Hash)
for _, e := range tree.Entries {
r.queue = append(r.queue, e.Hash)
}
case plumbing.TagObject:
tag, err := object.DecodeTag(r.st, obj)
if err != nil {
return err
}
r.queue = append(r.queue, tag.Hash)
r.queue = append(r.queue, tag.Target)
case plumbing.BlobObject:
blob, err := object.DecodeBlob(obj)
if err != nil {
return err
}
r.queue = append(r.queue, blob.Hash)
default:
return plumbing.ErrInvalidType
}
if _, err := r.st.SetEncodedObject(obj); err != nil {
return err
}
processed[objHash.String()] = struct{}{}
}
for packPath := range packs {
f, err := r.fs.Open(packPath)
if err != nil {
return err
}
if err := packfile.UpdateObjectStorage(r.st, f); err != nil {
_ = f.Close()
return err
}
if err := f.Close(); err != nil {
return err
}
}
return nil
}
package http
import (
"bufio"
"bytes"
"context"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strings"
internal "github.com/go-git/go-git/v6/internal/transport"
"github.com/go-git/go-git/v6/plumbing/format/pktline"
"github.com/go-git/go-git/v6/plumbing/protocol"
"github.com/go-git/go-git/v6/plumbing/protocol/capability"
"github.com/go-git/go-git/v6/plumbing/protocol/packp"
transport "github.com/go-git/go-git/v6/plumbing/transport"
"github.com/go-git/go-git/v6/storage"
"github.com/go-git/go-git/v6/utils/ioutil"
)
// wrapDropped annotates err with the origin crossing that withheld credentials,
// when there was a crossing, a credential to withhold, and err is an
// authentication failure. The error keeps its type and message.
//
// The credential has to have existed: a caller who configured none and is
// challenged after a redirect would otherwise be told a credential of theirs
// was not sent, naming something they never had.
//
// The origins are copied out of the record, which outlives this call — it is
// stored on the session and read again for every later request.
func wrapDropped(rec *redirectRecord, err error) error {
if err == nil {
return err
}
if !rec.withheld() {
return err
}
from, to, ok := rec.origins()
if !ok {
return err
}
if !errors.Is(err, transport.ErrAuthenticationRequired) &&
!errors.Is(err, transport.ErrAuthorizationFailed) {
return err
}
// originOf again on values that are already origins: it is what makes the
// copies, so a caller mutating the error cannot reach the session's record.
return fmt.Errorf("%w: %w", err, &transport.CredentialsDroppedError{
From: originOf(from),
To: originOf(to),
})
}
// sessionBase is the state every session carries, in one value. Both session
// types embed it, so a field added here reaches both without a signature to
// thread it through.
type sessionBase struct {
client *http.Client
baseURL *url.URL
service string
authorizer Authorizer
dropped *redirectRecord
}
// Handshake implements transport.Transport. GETs /info/refs to discover
// refs and detects smart vs dumb HTTP.
func (t *Transport) Handshake(ctx context.Context, req *transport.Request) (transport.Session, error) {
service := req.Command
// The caller's URL with its path in the spelling the requests will carry;
// everything downstream compares against this base. See effectiveBase.
baseURL, err := effectiveBase(req.URL)
if err != nil {
return nil, err
}
forceDumb := t.opts.ForceDumb
// git archive over HTTP discovers protocol support through the upload-pack
// info/refs endpoint and requires Protocol v2 (remote-curl.c). The archive
// request itself is later POSTed to the git-upload-archive endpoint.
discoverService := service
discoverProtocol := req.Protocol
if service == transport.UploadArchiveService {
discoverService = transport.UploadPackService
discoverProtocol = protocol.V2
}
d := discovery{service: discoverService, protocol: discoverProtocol, forceDumb: forceDumb}
// Only the discovery GET carries the initial-request marker, so only it may
// follow redirects under the default policy.
rec := &redirectRecord{}
httpReq, err := d.request(withRedirectRecord(withInitialRequest(ctx), rec), baseURL)
if err != nil {
return nil, err
}
// One authorizer for every credential this handshake holds — the repository
// URL's userinfo and whatever the caller supplies for the origin it named —
// so there is one thing to withhold rather than two that could disagree.
cred0, err := t.acquire(ctx, baseURL, baseURL, false)
if err != nil {
return nil, fmt.Errorf("http transport: %w", err)
}
var configured Authorizer
if cred0 != nil {
configured = cred0.credential.Authorizer
}
authorizer := combine(basicAuth(baseURL.User), configured)
// Recorded before the request goes out, and read back only on an
// authentication failure. See redirectRecord.held.
rec.holdsCredential(authorizer != nil)
if err := applyAuth(httpReq, authorizer); err != nil {
return nil, fmt.Errorf("http transport: authorize: %w", err)
}
client := t.resolveClient()
resp, err := doRequest(client, httpReq)
// Retry once at the origin a redirect reached, if it challenged. See
// reauthenticate.
reacq, resp, err := t.reauthenticate(ctx, client, baseURL, d, resp, err)
if err != nil {
// doRequest returns a non-nil response with its error for any non-2xx,
// and checkError has already read what it needs of the body.
if resp != nil {
_ = resp.Body.Close()
}
// A credential was minted for the origin this failed at, so the
// caller's own credential being withheld is not what went wrong.
if reacq != nil {
return nil, fmt.Errorf("http transport: %w", err)
}
return nil, fmt.Errorf("http transport: %w", wrapDropped(rec, err))
}
redirectedURL, err := applyRedirect(resp, baseURL)
if err != nil {
_ = resp.Body.Close()
return nil, fmt.Errorf("http transport: %w", wrapDropped(rec, err))
}
// Copy before clearing: applyRedirect returns baseURL itself when the
// redirect changed nothing, and baseURL belongs to the caller. Cleared
// unconditionally, so a credential reaches the wire only through the
// authorizer below and not by a second route no rule governs.
cleared := *redirectedURL
cleared.User = nil
sessURL := &cleared
// Re-acquire after an origin or path move, so the session's one authorizer
// is not reused for a target the server rather than the caller chose. The
// record preserves a sticky crossing the endpoints alone would not show;
// paths are compared escaped, for the reason applyRedirect gives.
if rec.crossed() || redirectedURL.EscapedPath() != baseURL.EscapedPath() {
// Both halves of the hop-0 credential are re-derived below, each under
// the relation deciding whether it may travel to where the chain ended
// up; anything not re-derived stays gone, as in canonical git's
// credential_from_url().
// The credential the retry already spent, when there was one:
// reauthenticate derives it from both sources under these same relations
// against this same target, so reuse it rather than asking the caller a
// question they have answered.
settled := reacq
if settled == nil {
// Nothing was spent, so derive both halves here, under the same
// relations reauthenticate uses.
var fromURL Authorizer
if credentialsMayFollow(baseURL, redirectedURL) {
fromURL = basicAuth(baseURL.User)
}
cred, aerr := t.acquire(ctx, redirectedURL, baseURL, true)
if aerr != nil {
_ = resp.Body.Close()
return nil, fmt.Errorf("http transport: %w", aerr)
}
var fromHook Authorizer
if cred != nil {
fromHook = cred.credential.Authorizer
}
// Hop 0's order, as in reauthenticate.
settled = &originCredential{
origin: originOf(redirectedURL),
credential: &Credential{Authorizer: combine(fromURL, fromHook)},
}
}
// Defense in depth: retain the settled credential only at the origin it
// was acquired for. Unreachable while the retry cannot be redirected,
// which is the other half of the pair — see errRetryRedirected.
authorizer = nil
if credentialsMayFollow(settled.origin, redirectedURL) {
authorizer = settled.credential.Authorizer
}
}
// No gate on the other path: nothing moved, so every hop satisfied the same
// comparison and the credential is already where it is allowed to be.
// The record annotates the session's later authentication failures with the
// crossing, and is only an explanation while the session has no credential:
// one minted for its own origin had nothing withheld on the way there, so
// naming the crossing would tell the caller to supply what they already
// supplied.
dropped := rec
if authorizer != nil {
dropped = nil
}
base := sessionBase{
client: client,
baseURL: sessURL,
service: req.Command,
authorizer: authorizer,
dropped: dropped,
}
return finishHandshake(resp, base, d)
}
// finishHandshake picks the smart or dumb session for a discovery response that
// has already been validated and had its credentials settled, keeping that
// dispatch out of the redirect and credential handling above it.
func finishHandshake(resp *http.Response, base sessionBase, d discovery) (transport.Session, error) {
if d.forceDumb {
return handshakeDumb(resp, base)
}
if smartContentType(resp.Header.Get("Content-Type"), d.service) {
return handshakeSmart(resp, base, d)
}
return handshakeDumb(resp, base)
}
func handshakeSmart(resp *http.Response, base sessionBase, d discovery) (transport.Session, error) {
// The advertisement ends at a flush-pkt, which leaves the rest of the body
// — the terminating chunk, on a chunked response — outstanding. The POST
// that opens the session follows immediately, so the discard is what
// decides whether it reuses this connection.
defer drainAndClose(resp.Body)
rd := bufio.NewReader(resp.Body)
_, prefix, err := pktline.PeekLine(rd)
if err != nil {
return nil, err
}
if bytes.HasPrefix(prefix, []byte("# service=")) {
var reply packp.SmartReply
if err := reply.Decode(rd); err != nil {
return nil, err
}
if reply.Service != d.service {
return nil, fmt.Errorf("unexpected service name: %w", transport.ErrInvalidResponse)
}
}
ver, err := transport.DiscoverVersion(rd)
if err != nil {
return nil, err
}
// git archive over HTTP is only available when the server speaks v2.
if base.service == transport.UploadArchiveService && ver != protocol.V2 {
return nil, transport.ErrArchiveUnsupported
}
if ver == protocol.V2 {
// Protocol v2: the server sends a capability advertisement instead of
// the v0/v1 ref advertisement. References are retrieved lazily via the
// ls-refs command, so refs stays nil here.
adv := &packp.CapabilityAdv{}
if err := adv.Decode(rd); err != nil {
return nil, err
}
// Protocol v2 fetch accepts "want <oid>" without the server
// advertising allow-*-sha1-in-want, so surface the gate as
// satisfied for exact-SHA1 refspecs (isSupportedRefSpec). The
// v2 client only sends agent/object-format on the wire, so these
// never leak into the request (internal.ClientCapabilities).
adv.Capabilities.Set(capability.AllowReachableSHA1InWant)
adv.Capabilities.Set(capability.AllowTipSHA1InWant)
return &smartPackSession{
sessionBase: base,
version: ver,
caps: adv.Capabilities,
}, nil
}
ar := &packp.AdvRefs{}
if err := ar.Decode(rd); err != nil && !errors.Is(err, packp.ErrEmptyAdvRefs) {
return nil, err
}
if err := capability.Validate(&ar.Capabilities); err != nil {
return nil, err
}
// Take the version from DiscoverVersion rather than AdvRefs.Decode's
// independent parse of the same line, so there is one source of truth.
ar.Version = ver
return &smartPackSession{
sessionBase: base,
version: ver,
caps: ar.Capabilities,
refs: ar,
}, nil
}
// maxQuotedBodySize caps how much of a rejected /info/refs body is quoted back
// in the error. Enough to recognise what the server sent, not enough to paste
// a page into a log line, and no larger than a bufio.Reader's buffer, since
// that is what bounds the Peek this size is asked of.
const maxQuotedBodySize = 256
// describeInfoRefsError turns a decode failure into one a caller can act on.
//
// packp reports which line was malformed and nothing about its contents,
// because the bytes belong to the server. What the transport knows, and packp
// does not, is which URL was fetched and what the server said it was serving —
// the difference between "invalid info/refs" and "that host answered your
// clone with an HTML sign-in page".
//
// The body is quoted only when it is plain text, matching git's
// show_http_message: other types are markup meant for a browser, and an
// interstitial can echo the request's own query back inside it.
func describeInfoRefsError(err error, resp *http.Response, base sessionBase, head []byte) error {
// Name the URL actually fetched, which carries the /info/refs tail and the
// service query the session's base does not.
fetched := base.baseURL
if resp.Request != nil && resp.Request.URL != nil {
fetched = resp.Request.URL
}
mediaType := contentMediaType(resp.Header.Get("Content-Type"))
if mediaType == "text/plain" {
return fmt.Errorf("%w: %s served content type %q: %w: %q",
transport.ErrInvalidResponse, redactedURL(fetched), mediaType, err,
strings.TrimSpace(sanitizeReason(string(head))))
}
return fmt.Errorf("%w: %s served content type %q: %w",
transport.ErrInvalidResponse, redactedURL(fetched), mediaType, err)
}
func handshakeDumb(resp *http.Response, base sessionBase) (transport.Session, error) {
defer resp.Body.Close() //nolint:errcheck
// Buffer the head of the body so a rejection can quote it. Peek leaves it
// in place for the decode, and returns what it has on a shorter body. The
// reader keeps bufio's own buffer size, which is what bounds a Peek; how
// much of a body is worth quoting is a separate question from how much of
// it is worth buffering.
rd := bufio.NewReader(resp.Body)
head, _ := rd.Peek(maxQuotedBodySize)
var infoRefs packp.InfoRefs
if err := infoRefs.Decode(rd); err != nil {
return nil, describeInfoRefsError(err, resp, base, head)
}
ar := &packp.AdvRefs{}
ar.References = infoRefs.References
return &dumbPackSession{sessionBase: base, refs: ar}, nil
}
var (
_ transport.Session = (*smartPackSession)(nil)
_ transport.Commander = (*smartPackSession)(nil)
_ transport.Archiver = (*smartPackSession)(nil)
)
type smartPackSession struct {
sessionBase
version protocol.Version
caps capability.List
refs *packp.AdvRefs
}
func (s *smartPackSession) Capabilities() *capability.List { return &s.caps }
func (s *smartPackSession) GetRemoteRefs(ctx context.Context, opts *transport.GetRemoteRefsOptions) (*transport.RemoteRefs, error) {
forPush := s.service == transport.ReceivePackService
if s.version == protocol.V2 {
var prefixes []string
if opts != nil {
prefixes = opts.RefPrefixes
}
refs, err := internal.LsRefs(ctx, s.Command, s.caps, prefixes)
if err != nil {
return nil, err
}
if !forPush && !internal.HasHashRef(refs) {
return nil, transport.ErrEmptyRemoteRepository
}
return transport.NewRemoteRefs(refs), nil
}
if s.refs == nil {
return nil, transport.ErrEmptyRemoteRepository
}
if !forPush && s.refs.IsEmpty() {
return nil, transport.ErrEmptyRemoteRepository
}
refs, err := s.refs.ResolvedReferences()
if err != nil {
return nil, err
}
return transport.NewRemoteRefs(refs), nil
}
// Command implements transport.Commander. It runs a Protocol v2 command as a
// single stateless HTTP POST: the request envelope is buffered and sent, and
// the response is decoded from the response body. Fetch uses its own round
// instead so it can stream the packfile from the body; Command is for
// non-streaming commands such as ls-refs.
func (s *smartPackSession) Command(ctx context.Context, cmd string, req packp.CommandArgs, resp packp.Decoder) error {
if s.version != protocol.V2 {
return transport.ErrUnsupportedVersion
}
r := &httpRequester{session: s, ctx: ctx}
cr := &packp.CommandRequest{
Command: cmd,
Capabilities: internal.ClientCapabilities(s.caps),
Args: req,
}
if err := cr.Encode(r); err != nil {
return err
}
// Command consumes the whole response (it never streams the body out), so
// release it on every path. A bare return on a decode error would otherwise
// leak the response body and its connection. Releasing it includes the
// discard: a decoder stops at the response's flush-pkt, and the request
// that reuses the connection — the fetch POST after an ls-refs — follows
// immediately.
defer func() {
if r.resp != nil {
drainAndClose(r.resp.Body)
}
}()
if resp != nil {
if err := resp.Decode(r); err != nil {
return err
}
}
return nil
}
func (s *smartPackSession) Fetch(ctx context.Context, st storage.Storer, req *transport.FetchRequest) error {
if s.version == protocol.V2 {
return s.fetchV2(ctx, st, req)
}
neg := &httpNegotiator{session: s, ctx: ctx}
shallows, err := transport.NegotiatePack(ctx, st, s.caps, true, neg, neg, req)
if err != nil {
if ioutil.ReadFinished(ctx, err) {
neg.closeResponse()
}
return err
}
if neg.current == nil || neg.current.resp == nil {
neg.current = &httpRequester{session: s, ctx: ctx}
if err := neg.current.doPost(); err != nil {
return err
}
}
err = transport.FetchPack(ctx, st, s.caps, io.NopCloser(neg), shallows, req)
if ioutil.ReadFinished(ctx, err) {
neg.closeResponse()
}
return err
}
// fetchV2 fetches over Protocol v2. Each negotiation round is a fresh stateless
// POST; internal.FetchV2 decodes the metadata via FetchOutput and, once the
// server commits to a packfile, streams it from that round's response body.
func (s *smartPackSession) fetchV2(ctx context.Context, st storage.Storer, req *transport.FetchRequest) error {
if req.Filter != "" && !internal.FetchSupports(s.caps, "filter") {
return transport.ErrFilterNotSupported
}
if req.Depth > 0 && !internal.FetchSupports(s.caps, "shallow") {
return transport.ErrShallowNotSupported
}
if err := transport.ReconcileObjectFormatV2(st, s.caps); err != nil {
return err
}
round := func(args *packp.FetchArgs) (*packp.FetchOutput, io.Reader, error) {
r := &httpRequester{session: s, ctx: ctx}
cr := &packp.CommandRequest{
Command: "fetch",
Capabilities: internal.ClientCapabilities(s.caps),
Args: args,
}
if err := cr.Encode(r); err != nil {
return nil, nil, err
}
out := &packp.FetchOutput{}
if err := out.Decode(r); err != nil {
// The success path hands r.resp.Body to the caller to stream; on a
// decode error nothing downstream will, so release it here.
if r.resp != nil {
_ = r.resp.Body.Close()
}
return nil, nil, err
}
if r.resp == nil {
return nil, nil, fmt.Errorf("http transport: fetch command produced no response")
}
// The response body is positioned at the packfile (when out.Packfile);
// internal.FetchV2 streams it and closes the body via io.Closer.
return out, r.resp.Body, nil
}
return internal.FetchV2(ctx, st, req, round)
}
func (s *smartPackSession) Push(ctx context.Context, st storage.Storer, req *transport.PushRequest) error {
rwc := &httpRequester{session: s, ctx: ctx}
err := transport.SendPack(ctx, st, s.caps, rwc, io.NopCloser(rwc), req)
if ioutil.ReadFinished(ctx, err) && rwc.resp != nil {
_ = rwc.resp.Body.Close()
}
return err
}
func (s *smartPackSession) Close() error { return nil }
// Archive implements transport.Archiver. git archive over HTTP is a v2-only,
// stateless operation (remote-curl.c): the archive request is POSTed to the
// git-upload-archive endpoint and the response carries the ACK/NACK and the
// sideband-encoded archive stream.
func (s *smartPackSession) Archive(ctx context.Context, req *transport.ArchiveRequest) (io.ReadCloser, error) {
if s.version != protocol.V2 {
return nil, transport.ErrArchiveUnsupported
}
rt := &httpRequester{session: s, ctx: ctx}
body := &httpArchiveBody{req: rt}
archive, err := transport.Archive(ctx, rt, body, req)
if err != nil {
_ = body.Close()
return nil, err
}
return archive, nil
}
// httpArchiveBody adapts an httpRequester to the io.ReadCloser the archive
// client reads from: reads come from the POST response body, and Close closes
// that body. The paired httpRequester is passed to transport.Archive as the
// writer, whose Close fires the POST.
type httpArchiveBody struct{ req *httpRequester }
func (b *httpArchiveBody) Read(p []byte) (int, error) { return b.req.Read(p) }
func (b *httpArchiveBody) Close() error {
if b.req.resp != nil {
return b.req.resp.Body.Close()
}
return nil
}
// httpRequester buffers writes and fires a POST on first Read or Close.
type httpRequester struct {
session *smartPackSession
ctx context.Context
buf bytes.Buffer
resp *http.Response
}
func (r *httpRequester) Write(p []byte) (int, error) { return r.buf.Write(p) }
func (r *httpRequester) Read(p []byte) (int, error) {
if r.resp == nil {
if err := r.doPost(); err != nil {
return 0, err
}
}
return r.resp.Body.Read(p)
}
func (r *httpRequester) Close() error {
if r.resp == nil {
return r.doPost()
}
return nil
}
func (r *httpRequester) doPost() error {
serviceURL, err := url.JoinPath(r.session.baseURL.String(), r.session.service)
if err != nil {
return err
}
httpReq, err := http.NewRequestWithContext(r.ctx, http.MethodPost, serviceURL, &r.buf)
if err != nil {
return err
}
httpReq.Header.Set("Content-Type", fmt.Sprintf("application/x-%s-request", r.session.service))
httpReq.Header.Set("Accept", fmt.Sprintf("application/x-%s-result", r.session.service))
httpReq.Header.Set("User-Agent", capability.DefaultAgent())
if gp := transport.GitProtocolEnv(r.session.version); gp != "" {
httpReq.Header.Set("Git-Protocol", gp)
}
if err := applyAuth(httpReq, r.session.authorizer); err != nil {
return err
}
r.resp, err = doRequest(r.session.client, httpReq)
if err != nil {
if r.resp != nil {
_ = r.resp.Body.Close()
}
return fmt.Errorf("http transport: %w", wrapDropped(r.session.dropped, err))
}
// doRequest has already turned any non-2xx into an error, so this catches
// only a 2xx that is not 200 — one the pack protocol cannot parse.
if r.resp.StatusCode != http.StatusOK {
_ = r.resp.Body.Close()
return fmt.Errorf("http transport: POST %s unexpected status %d", redactedURL(r.resp.Request.URL), r.resp.StatusCode)
}
return nil
}
// httpNegotiator supports multi-round stateless RPC negotiation by
// creating a fresh httpRequester for each round. A new round begins
// when Write is called after the previous round's response has arrived.
type httpNegotiator struct {
session *smartPackSession
ctx context.Context
current *httpRequester
}
func (n *httpNegotiator) Write(p []byte) (int, error) {
if n.current != nil && n.current.resp != nil {
// The previous round is complete, and this round is the request that
// reuses its connection.
drainAndClose(n.current.resp.Body)
n.current = nil
}
if n.current == nil {
n.current = &httpRequester{session: n.session, ctx: n.ctx}
}
return n.current.Write(p)
}
func (n *httpNegotiator) Read(p []byte) (int, error) {
if n.current == nil {
return 0, io.ErrClosedPipe
}
return n.current.Read(p)
}
func (n *httpNegotiator) Close() error {
if n.current == nil {
return nil
}
return n.current.Close()
}
// closeResponse closes the current round's response body, without discarding
// what is left of it: Fetch calls this when it is finished with the negotiator
// altogether, so no request follows that the connection could serve.
//
// Whether the body was read to its end is the caller's affair. So is whether
// closing is safe at all — ioutil.ReadFinished answers that, and a caller that
// does not ask races the context reader wrapped around this body.
func (n *httpNegotiator) closeResponse() {
if n.current != nil && n.current.resp != nil {
_ = n.current.resp.Body.Close()
n.current.resp = nil
}
}
var _ transport.Session = (*dumbPackSession)(nil)
type dumbPackSession struct {
sessionBase
refs *packp.AdvRefs
}
func (s *dumbPackSession) Capabilities() *capability.List { return &capability.List{} }
func (s *dumbPackSession) GetRemoteRefs(_ context.Context, _ *transport.GetRemoteRefsOptions) (*transport.RemoteRefs, error) {
if s.refs == nil {
return nil, transport.ErrEmptyRemoteRepository
}
refs, err := s.refs.ResolvedReferences()
if err != nil {
return nil, err
}
return transport.NewRemoteRefs(refs), nil
}
func (s *dumbPackSession) Fetch(ctx context.Context, st storage.Storer, req *transport.FetchRequest) error {
return s.fetchDumb(ctx, st, req)
}
func (s *dumbPackSession) Push(_ context.Context, _ storage.Storer, _ *transport.PushRequest) error {
return fmt.Errorf("dumb HTTP does not support push: %w", transport.ErrCommandUnsupported)
}
func (s *dumbPackSession) Close() error { return nil }
var (
_ transport.Session = (*smartPackSession)(nil)
_ transport.Session = (*dumbPackSession)(nil)
_ transport.Transport = (*Transport)(nil)
)
package http
import (
"context"
"crypto/tls"
"fmt"
"net/http"
"net/url"
"github.com/go-git/go-git/v6/plumbing/transport"
)
// contextKey is an unexported type for context keys in this package.
type contextKey int
const (
initialRequestKey contextKey = iota
redirectRecordKey
)
// originOf returns u's origin as a fresh URL carrying scheme and host only. It
// is built rather than copied so no other field survives into a value the
// transport treats as an origin, and so a receiver cannot reach the transport's
// own URLs through it.
func originOf(u *url.URL) *url.URL {
return &url.URL{Scheme: u.Scheme, Host: u.Host}
}
// withoutUserinfo returns a copy of u with any userinfo removed.
//
// This is how CredentialRequest.RepositoryURL is built, and it is the only
// place a URL that keeps its path is handed to a caller: the copy means a hook
// that logs its request cannot print the caller's password, and one that
// mutates what it was given cannot reach a URL still in use.
//
// Only the userinfo goes. The query and fragment survive, so a hook scoping a
// credential to what the caller wrote can read them — and one that logs the URL
// prints them, which for a clone URL carrying ?private_token= is the caller's
// own secret. The other URLs this package hands out go through originOf, which
// carries no userinfo either.
func withoutUserinfo(u *url.URL) *url.URL {
if u == nil {
return nil
}
c := *u
c.User = nil
return &c
}
// redirectRecord carries what CheckRedirect saw back to Handshake.
//
// It is reached through the request context rather than captured in the
// CheckRedirect closure, because resolveClient's client is stored on the
// session and reused for every later request: a captured variable would leak
// one request's redirect history into the next. net/http propagates the
// original request's context to every hop.
//
// No synchronisation is needed: net/http drives the chain from the goroutine
// that called Do, and Handshake reads the record only after Do returned.
type redirectRecord struct {
didCross bool
// held is whether the request this chain began with carried a credential at
// all. A crossing says one would have been withheld; this says there was one
// to withhold. An explanation naming a credential is only true if one
// existed, while what the session re-derives is the same either way.
held bool
from, to *url.URL
}
func withRedirectRecord(ctx context.Context, rec *redirectRecord) context.Context {
return context.WithValue(ctx, redirectRecordKey, rec)
}
func redirectRecordFrom(req *http.Request) *redirectRecord {
rec, _ := req.Context().Value(redirectRecordKey).(*redirectRecord)
return rec
}
// note records an origin crossing. The two ends come from different crossings
// on purpose: from is the origin the credential was issued for, which every
// crossing of one chain reports identically, so the first to name it settles
// it; to is replaced by each later crossing. Because the strip is sticky every
// hop after the first crossing is noted too, so the last to recorded is the
// origin the request finally reached and therefore the one that challenged it.
// Keeping the first would name an intermediate hop the caller cannot configure
// a credential for.
//
// A crossing is recorded even when an endpoint cannot be read, because
// stripCredentials strips on that path too and the two must not disagree; the
// pair is then incomplete, which origins reports.
func (r *redirectRecord) note(from, to *url.URL) {
if r == nil {
return
}
r.didCross = true
if r.from == nil && from != nil {
r.from = originOf(from)
}
if to != nil {
r.to = originOf(to)
}
}
// crossed reports whether any hop left the origin.
func (r *redirectRecord) crossed() bool { return r != nil && r.didCross }
// holdsCredential records whether the chain began with a credential. Called
// once, before the request goes out, from the only place that knows.
func (r *redirectRecord) holdsCredential(held bool) {
if r == nil {
return
}
r.held = held
}
// withheld reports whether a credential existed and a crossing took it away,
// which is what CredentialsDroppedError says happened.
func (r *redirectRecord) withheld() bool { return r != nil && r.didCross && r.held }
// origins returns the origin the credential was issued for and the origin the
// chain ended at, and whether both are known.
func (r *redirectRecord) origins() (from, to *url.URL, ok bool) {
if r == nil || r.from == nil || r.to == nil {
return nil, nil, false
}
return r.from, r.to, true
}
// RedirectPolicy controls how the HTTP transport follows redirects.
type RedirectPolicy string
const (
// FollowInitialRedirects follows redirects only for the initial
// /info/refs discovery request.
FollowInitialRedirects RedirectPolicy = "initial"
// FollowRedirects follows redirects for all requests.
FollowRedirects RedirectPolicy = "true"
// NoFollowRedirects disables redirects for all requests.
NoFollowRedirects RedirectPolicy = "false"
)
// withInitialRequest marks a context so that checkRedirect allows
// the HTTP client to follow redirects. Only the /info/refs discovery
// request should carry this flag.
func withInitialRequest(ctx context.Context) context.Context {
return context.WithValue(ctx, initialRequestKey, true)
}
func isInitialRequest(req *http.Request) bool {
v, _ := req.Context().Value(initialRequestKey).(bool)
return v
}
// Options configures the HTTP transport.
type Options struct {
// Client is the underlying HTTP client. If nil, a default client is
// created. When Client is set, TLS and HTTPProxy are ignored —
// configure them on the provided Client directly.
//
// A RoundTripper that resolves redirects itself disables every guard in
// this package. The credential stripping, the redirect policy, and the
// re-authentication a crossing triggers all run from CheckRedirect, which
// net/http calls only for redirects it resolves.
//
// Credentials this Client adds are not origin-scoped the way
// Options.Credentials is: a RoundTripper injects after the hop is decided,
// and Client.Jar is consulted after CheckRedirect and keys on host alone,
// so a cookie follows a redirect to a subdomain, or to another port, that
// this transport counts as another origin.
//
// A CheckRedirect set here runs after this transport's own and can refuse a
// hop the policy permits, but any header it adds is dropped on a hop that
// leaves the repository's origin.
Client *http.Client
// FollowRedirects controls redirect handling. The zero value is
// FollowInitialRedirects, matching Git's default: only the /info/refs
// discovery GET, which carries no body, may follow a redirect.
//
// FollowRedirects lets a POST follow one too. A redirect across an origin
// strips the request's credentials but not its body: net/http replays the
// body on a 307 or 308, and Content-Type and Content-Length are preserved,
// so the pack request arrives at the server-chosen origin complete. For
// upload-pack that discloses which objects the caller already has; for
// receive-pack, the packfile being pushed.
//
// Nothing follows up on such a POST: it is not retried, Credentials is not
// consulted for the origin it reached, and its crossing is not recorded, so
// the failure it produces does not name that origin. Those describe the
// transport as it stands and may change. The disclosure above does not
// depend on them, because the body arrives before any of it would apply.
//
// To let the discovery GET follow a cross-origin redirect while refusing
// one for a request with a body, set a CheckRedirect on Client that returns
// an error unless req.Method is GET.
FollowRedirects RedirectPolicy
// HTTPProxy returns the proxy URL for a given HTTP request.
// If nil, the default http.Transport proxy behavior is used.
// Ignored when Client is set.
HTTPProxy func(*http.Request) (*url.URL, error)
// TLS configures TLS for HTTPS connections. Set InsecureSkipVerify
// to skip certificate verification, or set RootCAs for a custom CA
// bundle. Ignored when Client is set.
TLS *tls.Config
// ForceDumb forces the transport to use the dumb HTTP protocol,
// bypassing smart HTTP detection. When true, the transport will
// not send the ?service= query parameter in the info/refs request
// and will always treat the server as a dumb HTTP server.
ForceDumb bool
// Credentials supplies a credential for the origin a request is about to be
// made to. It is called for the repository's origin, and again for a
// redirect target once a redirect has left that origin. The zero value
// leaves the repository URL's userinfo and query as the only credentials.
//
// See CredentialsFunc for the contract, ForRepositoryOrigin and ForOrigin
// for the common adapters, and Chain to combine sources. Userinfo is
// applied first and this credential second, so an Authorizer can replace
// the Authorization header userinfo set, or add another.
//
// Only the headers this transport sets itself survive a cross-origin
// redirect, so an Authorizer's own headers — a trace or tenant header — are
// dropped. The filter matches header names, not values: a secret written
// into a header this transport does set, such as User-Agent, still crosses
// the boundary and appears in trace.HTTP output.
//
// A redirecting remote chooses the origin and repository path this is asked
// about, so a source that prompts a human rather than reading a store is a
// phishing surface reachable from any clone URL.
//
// A token in the repository URL's query (?private_token=, ?job_token=) is
// withheld across an origin boundary like any other credential, but is
// never re-acquired and never rides the /info/refs GET. Supply it here or
// as userinfo instead.
Credentials CredentialsFunc
}
// Transport implements the http:// and https:// transport protocol.
type Transport struct {
opts Options
}
var _ transport.Transport = (*Transport)(nil)
// NewTransport creates an HTTP transport with the given options.
func NewTransport(opts Options) *Transport {
return &Transport{opts: opts}
}
func (t *Transport) resolveClient() *http.Client {
if t.opts.Client != nil {
client := *t.opts.Client
client.CheckRedirect = wrapCheckRedirect(t.opts.redirectPolicy(), t.opts.Client.CheckRedirect)
return &client
}
tr := http.DefaultTransport.(*http.Transport).Clone()
if t.opts.HTTPProxy != nil {
tr.Proxy = t.opts.HTTPProxy
}
if t.opts.TLS != nil {
tr.TLSClientConfig = t.opts.TLS
}
return &http.Client{
Transport: tr,
CheckRedirect: wrapCheckRedirect(t.opts.redirectPolicy(), nil),
}
}
func (o Options) redirectPolicy() RedirectPolicy {
if o.FollowRedirects == "" {
return FollowInitialRedirects
}
return o.FollowRedirects
}
func wrapCheckRedirect(policy RedirectPolicy, next func(*http.Request, []*http.Request) error) func(*http.Request, []*http.Request) error {
return func(req *http.Request, via []*http.Request) error {
if err := checkRedirect(req, via, policy); err != nil {
return err
}
// Strip before the caller's hook so it observes what will be sent, and
// again after so a hook of the common "preserve my headers across
// redirects" shape — copying from via[0], the original unsanitized
// request — cannot reinstate them.
stripCredentials(req, via)
if next != nil {
if err := next(req, via); err != nil {
return err
}
}
stripCredentials(req, via)
return nil
}
}
// stripCredentials removes credentials from req once the redirect chain has
// left the origin of the original, credential-bearing request.
//
// CheckRedirect is the only hook that runs while a redirected request's
// headers are still mutable: http.Client.Do performs the entire chain
// internally, so anything the transport does after Do returns is too late.
//
// Two subtleties:
//
// - net/http rebuilds every redirect request from the original request's
// headers before calling this, so a header removed at one hop reappears
// at the next. The decision is therefore recomputed per hop.
// - The decision is sticky: once the chain has left the origin, credentials
// stay gone even if a later hop returns to it. Stickiness is derived from
// via rather than stored, because this closure is shared across a
// session's requests.
//
// Stripping keeps only safeHeaders. An allowlist is used rather than a list of
// credential header names because caller credentials arrive under names that
// cannot be enumerated — PRIVATE-TOKEN, X-Api-Key, gateway headers — which is
// what net/http's fixed list of sensitive names gets wrong, and because it is
// immune to header-name canonicalisation.
//
// The URL's userinfo goes with the headers: on a redirected request it can only
// have come from the target, via the Location header, and net/http turns
// req.URL.User into an Authorization header on the way out. Emptying the
// headers and leaving the URL alone would let a target plant a credential on
// the very hop this exists to sanitize.
func stripCredentials(req *http.Request, via []*http.Request) {
if len(via) == 0 {
return
}
// req.URL is non-nil by construction — checkRedirect dereferences
// req.URL.Scheme on every path that returns nil, so it runs first or not at
// all. This check and the two in crossedOrigin are defensive against a
// synthetic caller; each treats an unreadable URL as a crossing, because
// removing one panics in credentialsMayFollow rather than leaking.
origin := via[0].URL
if origin != nil && !crossedOrigin(origin, req, via) {
return
}
// Recorded on every path that strips, including the defensive one: a record
// that could disagree with the strip is the divergence it exists to remove.
redirectRecordFrom(req).note(origin, req.URL)
req.Header = filterHeaders(req.Header)
if req.URL != nil {
req.URL.User = nil
}
}
// crossedOrigin reports whether any hop so far, including the pending one, has
// left origin.
//
// Every comparison asks the relation in one direction: from the origin the
// credential was issued for, towards the hop being judged. The relation is
// asymmetric — an http origin on port 80 may upgrade to https on 443 of the
// same host, never the reverse — so asking it the other way round reads an
// upgrade already taken as a downgrade and withholds the credential from a hop
// it was entitled to reach, which is a clone that stops working.
func crossedOrigin(origin *url.URL, req *http.Request, via []*http.Request) bool {
if req.URL == nil || !credentialsMayFollow(origin, req.URL) {
return true
}
for _, prev := range via[1:] {
if prev.URL == nil || !credentialsMayFollow(origin, prev.URL) {
return true
}
}
return false
}
// checkRedirect implements Git's http.followRedirects policies. The default
// policy is "initial", where only the GET /info/refs discovery request may
// follow redirects.
//
// It decides only whether a hop may proceed; credentials on a permitted hop are
// stripCredentials' business. net/http's Client applies its own rule first, but
// that rule forwards credentials to subdomains, ignores port and scheme, and
// recognises only a fixed set of header names.
func checkRedirect(req *http.Request, via []*http.Request, policy RedirectPolicy) error {
if len(via) != 0 {
prev := via[len(via)-1]
if prev.URL != nil && prev.URL.Scheme == "https" && req.URL.Scheme == "http" {
return fmt.Errorf("http transport: redirect downgrades scheme to %s", redactedURL(req.URL))
}
}
switch policy {
case FollowRedirects:
case NoFollowRedirects:
return fmt.Errorf("http transport: redirects disabled to %s", redactedURL(req.URL))
case FollowInitialRedirects:
if !isInitialRequest(req) {
return fmt.Errorf("http transport: redirect on non-initial request to %s", redactedURL(req.URL))
}
default:
return fmt.Errorf("http transport: invalid redirect policy %q", policy)
}
if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
// The scheme is the one part of a Location this prints without going
// through redactedURL, so it is bounded here instead.
return fmt.Errorf("http transport: redirect to unsupported scheme %q", bounded(req.URL.Scheme))
}
if len(via) >= 10 {
return fmt.Errorf("http transport: too many redirects")
}
return nil
}
package http
import (
"context"
"errors"
"fmt"
"net/http"
"net/url"
transport "github.com/go-git/go-git/v6/plumbing/transport"
)
// errRetryRedirected is returned when the re-authentication retry is answered
// with a redirect. The retry does not follow one: it exists to spend a
// credential at exactly one known origin, and a hop would both carry it further
// and buy a second redirect budget, since checkRedirect's cap is per Client.Do.
//
// This is a tightening rather than parity: git's retries do follow redirects,
// because http_request_recoverable() re-issues through the same
// http_get_options, whose initial_request marker — the one that lets the
// discovery GET follow a redirect at all — is never cleared.
var errRetryRedirected = errors.New("http transport: re-authentication retry was redirected")
// noRedirectClient returns a shallow copy of c whose CheckRedirect refuses
// every hop.
func noRedirectClient(c *http.Client) *http.Client {
cp := *c
// A cookie is a credential, and this request is by construction one the
// caller's credentials may not travel on. net/http adds jar cookies in
// Client.send, after CheckRedirect has run and so beyond anything
// stripCredentials can reach, which leaves dropping the jar here as the only
// way to keep them off it. Only this request: a cookie on a hop the client
// followed is net/http's to decide, and Options.Client says so.
cp.Jar = nil
cp.CheckRedirect = func(*http.Request, []*http.Request) error { return errRetryRedirected }
return &cp
}
// originCredential pairs a credential with the origin it was acquired for; the
// origin is what the session's credential gate is re-anchored on.
//
// When reauthenticate returns one, the credential is everything that origin is
// authenticated with — the repository URL's userinfo and the caller's source
// both, where each may travel there — not one of the two.
type originCredential struct {
origin *url.URL
credential *Credential
}
// acquire asks the caller for a credential belonging to target's origin. This
// is the only place Options.Credentials is called, from either path. It returns
// (nil, nil) when no hook is configured or the hook declines.
//
// repository is a separate parameter and never derived from target, so a call
// site cannot pass one and have the other default to it: a credential source
// comparing them would then find every origin to be the caller's own.
//
// The caller must have validated target through applyRedirect first: a target
// that cannot become a base URL must not be able to attract a credential.
func (t *Transport) acquire(ctx context.Context, target, repository *url.URL, redirected bool) (*originCredential, error) {
if t.opts.Credentials == nil {
return nil, nil
}
origin := originOf(target)
cred, err := t.opts.Credentials(ctx, &CredentialRequest{
TargetOrigin: origin,
TargetPath: target.EscapedPath(),
RepositoryURL: withoutUserinfo(repository),
Redirected: redirected,
})
if err != nil {
return nil, err
}
if cred == nil || cred.Authorizer == nil {
return nil, nil
}
// A second origin, not the one the hook was handed: CredentialRequest
// promises its URLs are copies made for that one call, so a hook that writes
// to req.TargetOrigin.Host must not reach what the transport keeps.
return &originCredential{origin: originOf(target), credential: cred}, nil
}
// reauthenticate implements the discovery-request half of canonical git's
// HTTP_REAUTH loop: when a redirect carried the discovery request to an origin
// the caller's credential may not be sent to, and that origin challenges, build
// a credential for the new origin and try again. It returns the response and
// error the caller should proceed with, and when it does not act it returns
// resp and err untouched.
//
// One attempt, where git's loop makes up to two, each preceded by a fresh
// credential_fill(): the caller's source has already answered for this origin,
// and asking it again unchanged only repeats a question a credential store
// would have answered differently. The attempt cannot be redirected either,
// where git's can — see errRetryRedirected. Both tighten the loop rather than
// follow it.
//
// The credential is built from both of the transport's sources under the same
// relations the session settles on, so a chain that returns to the origin the
// caller named is retried with the same credential whichever way it was
// supplied. Returning the pair rather than the caller's half alone is what
// makes a non-nil return mean "a credential was spent at this origin".
//
// The returned *http.Response is always the one the caller must close; on the
// paths that close the original first, closing again is a no-op.
func (t *Transport) reauthenticate(
ctx context.Context,
client *http.Client,
baseURL *url.URL,
d discovery,
resp *http.Response,
err error,
) (*originCredential, *http.Response, error) {
// A nil resp is the ordinary path: doRequest returns one for any client.Do
// failure. The other three are net/http's to populate, and a response whose
// target cannot be read is one whose origin cannot be checked.
if resp == nil || resp.Request == nil || resp.Request.URL == nil || resp.Body == nil {
return nil, resp, err
}
// 401 only. A 403 is what a WAF or CDN answers with and says nothing about
// authentication.
if !errors.Is(err, transport.ErrAuthenticationRequired) {
return nil, resp, err
}
// The whole-chain record, not the two endpoints: stripping is sticky, so a
// chain that left the repository's origin and came back arrived here with
// nothing, where comparing baseURL against the final URL would read it as
// still holding its credential. A permitted upgrade sets no record, because
// the credential followed it. A missing record declines, the safe direction.
if !redirectRecordFrom(resp.Request).crossed() {
return nil, resp, err
}
// Validate before consulting the caller: nothing has yet checked the
// /info/refs tail or the scheme rule. See acquire.
newBase, rerr := applyRedirect(resp, baseURL)
if rerr != nil {
return nil, resp, err
}
// checkError has already taken what it needs of the body.
_ = resp.Body.Close()
// The repository URL's userinfo, where the chain ended somewhere it may
// travel to: that is the origin the caller named, where the first request
// already spent it and the detour never saw it. Withholding it while the
// caller's source is re-offered would make one credential behave two ways
// depending only on how it was supplied.
var fromURL Authorizer
if credentialsMayFollow(baseURL, newBase) {
fromURL = basicAuth(baseURL.User)
}
reacq, aerr := t.acquire(ctx, newBase, baseURL, true)
if aerr != nil {
return nil, resp, aerr
}
var fromHook Authorizer
if reacq != nil {
fromHook = reacq.credential.Authorizer
}
// Hop 0's order — userinfo first, the caller's source after it — so the retry
// carries what the first request carried. combine yields nil when neither
// source answers for this origin, and there is nothing to spend.
retryAuth := combine(fromURL, fromHook)
if retryAuth == nil {
return nil, resp, err
}
spent := &originCredential{
origin: originOf(newBase),
credential: &Credential{Authorizer: retryAuth},
}
// Re-issue at the validated target, through the same constructor as the
// original request, so nothing the server chose — userinfo, query, fragment
// — can travel on a request this credential authenticates.
retryReq, nerr := d.request(ctx, newBase)
if nerr != nil {
return nil, resp, err
}
if authErr := retryAuth(retryReq); authErr != nil {
return nil, resp, authErr
}
// One attempt, redirects refused: see errRetryRedirected.
retryResp, retryErr := doRequest(noRedirectClient(client), retryReq)
if retryResp == nil {
// spent, not nil: a credential was offered at this origin, so what
// failed is authentication there, not a credential withheld on the
// way. The original status stays the error the caller sees and
// retryErr becomes its cause — the redirect refusal, or whatever else
// client.Do returned, already redacted by doRequest.
if stopped(retryErr) {
// A clone the caller stopped is not a clone that needs credentials, so
// the 401 renders in the message but leaves the error chain: %s, not %w.
// Otherwise a caller that classifies authentication before cancellation
// prompts for a password on a clone the user aborted.
return spent, resp, fmt.Errorf("%s: %w", err, retryErr)
}
return spent, resp, fmt.Errorf("%w: %w", err, retryErr)
}
return spent, retryResp, retryErr
}
// stopped reports whether err is the caller withdrawing: a cancelled context, a
// context deadline, or an http.Client timeout, which net/http reports as the
// context deadline (transport.go's timeoutError).
func stopped(err error) bool {
return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded)
}
package transport
import (
"bufio"
"fmt"
"io"
"net/url"
"path/filepath"
"strings"
"github.com/go-git/go-billy/v6"
"github.com/go-git/go-billy/v6/osfs"
"github.com/go-git/go-git/v6/plumbing/cache"
"github.com/go-git/go-git/v6/storage"
"github.com/go-git/go-git/v6/storage/filesystem"
)
// Loader loads a storage.Storer from a URL.
type Loader interface {
Load(u *url.URL) (storage.Storer, error)
}
// DefaultLoader is a filesystem loader that resolves paths against the
// root filesystem.
var DefaultLoader Loader = NewFilesystemLoader(osfs.New(""), false)
// FilesystemLoader loads repositories from a billy.Filesystem.
type FilesystemLoader struct {
base billy.Filesystem
strict bool
}
// NewFilesystemLoader creates a Loader that resolves URL paths against the
// given base filesystem.
func NewFilesystemLoader(base billy.Filesystem, strict bool) *FilesystemLoader {
return &FilesystemLoader{base: base, strict: strict}
}
// Load resolves the URL path to a repository on the filesystem.
func (l *FilesystemLoader) Load(u *url.URL) (storage.Storer, error) {
return l.load(u.Path, false)
}
func (l *FilesystemLoader) load(path string, tried bool) (storage.Storer, error) {
fs, err := l.base.Chroot(path)
if err != nil {
return nil, err
}
// Check for .git first (directory or gitfile) to match git's behavior
// Git prefers .git/ over a bare repository in the same directory
if !tried && !l.strict {
if fi, err := fs.Lstat(".git"); err == nil {
tried = true
if fi.IsDir() {
// .git is a directory, use it
path = filepath.Join(path, ".git")
} else {
// .git is a file (gitfile), read the gitdir path
gitdir, err := readGitfile(fs)
if err != nil {
return nil, err
}
// gitdir can be absolute or relative
if filepath.IsAbs(gitdir) {
path = gitdir
} else {
path = filepath.Join(path, gitdir)
}
}
return l.load(path, tried)
}
}
// Check for config file to detect bare repository
fi, err := fs.Lstat("config")
if err != nil || fi.IsDir() {
if !l.strict && !tried {
// No .git and no config, try appending .git
tried = true
path += ".git"
return l.load(path, tried)
}
return nil, ErrRepositoryNotFound
}
return filesystem.NewStorageWithOptions(fs, cache.NewObjectLRUDefault(), filesystem.Options{}), nil
}
// readGitfile reads a .git file and extracts the gitdir path.
// The .git file should contain a single line: "gitdir: <path>"
func readGitfile(fs billy.Filesystem) (string, error) {
f, err := fs.Open(".git")
if err != nil {
return "", err
}
defer func() { _ = f.Close() }()
reader := bufio.NewReader(f)
line, err := reader.ReadString('\n')
if err != nil && err != io.EOF {
return "", err
}
const prefix = "gitdir: "
if !strings.HasPrefix(line, prefix) {
return "", fmt.Errorf(".git file has no %s prefix", prefix)
}
gitdir := strings.TrimSpace(line[len(prefix):])
return gitdir, nil
}
// MapLoader is a Loader that uses a lookup map keyed by URL path.
type MapLoader map[string]storage.Storer
// Load returns a storer for the given URL path.
func (l MapLoader) Load(u *url.URL) (storage.Storer, error) {
s, ok := l[u.Path]
if !ok {
return nil, ErrRepositoryNotFound
}
return s, nil
}
package transport
import (
"context"
"errors"
"fmt"
"io"
"slices"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/format/config"
"github.com/go-git/go-git/v6/plumbing/format/pktline"
"github.com/go-git/go-git/v6/plumbing/protocol/capability"
"github.com/go-git/go-git/v6/plumbing/protocol/packp"
"github.com/go-git/go-git/v6/storage"
"github.com/go-git/go-git/v6/utils/ioutil"
xstorage "github.com/go-git/go-git/v6/x/storage"
)
const (
initialFlush = 16
pipeSafeFlush = 32
largeFlush = 16384
maxInVein = 256
)
func nextFlush(statelessRPC bool, count int) int {
if statelessRPC {
if count < largeFlush {
return count << 1
}
return count * 11 / 10
}
if count < pipeSafeFlush {
return count << 1
}
return count + pipeSafeFlush
}
func applyServerACKs(
statelessRPC bool,
acks []packp.ACK,
common map[plumbing.Hash]struct{},
statelessCommon *[]plumbing.Hash,
gotContinue *bool,
gotReady *bool,
inVein *int,
) {
for _, ack := range acks {
if !*gotContinue && ack.Status > 0 {
*gotContinue = true
}
switch ack.Status {
case packp.ACKContinue:
*inVein = 0
case packp.ACKReady:
*gotReady = true
*inVein = 0
case packp.ACKCommon:
_, alreadyCommon := common[ack.Hash]
common[ack.Hash] = struct{}{}
if statelessRPC && !alreadyCommon {
*statelessCommon = append(*statelessCommon, ack.Hash)
*inVein = 0
}
}
}
}
// NegotiatePack performs the pack negotiation phase of the fetch operation.
func NegotiatePack(
ctx context.Context,
st storage.Storer,
caps capability.List,
statelessRPC bool,
reader io.Reader,
writer io.WriteCloser,
req *FetchRequest,
) (shallowInfo *packp.ShallowUpdate, err error) {
reader = ioutil.NewContextReader(ctx, reader)
writer = ioutil.NewContextWriteCloser(ctx, writer)
upreq := &packp.UploadRequest{}
multiAck := caps.Supports(capability.MultiACK)
multiAckDetailed := caps.Supports(capability.MultiACKDetailed)
if multiAckDetailed {
upreq.Capabilities.Set(capability.MultiACKDetailed)
} else if multiAck {
upreq.Capabilities.Set(capability.MultiACK)
}
if req.Progress != nil {
if caps.Supports(capability.Sideband64k) {
upreq.Capabilities.Set(capability.Sideband64k)
} else if caps.Supports(capability.Sideband) {
upreq.Capabilities.Set(capability.Sideband)
}
} else if caps.Supports(capability.NoProgress) {
upreq.Capabilities.Set(capability.NoProgress)
}
if caps.Supports(capability.ObjectFormat) {
var clientFormat, serverFormat config.ObjectFormat
if capValues := caps.Get(capability.ObjectFormat); len(capValues) > 0 {
of := config.ObjectFormat(capValues[0])
switch of {
case config.SHA1, config.SHA256:
serverFormat = of
}
}
cfg, err := st.Config()
if err == nil {
clientFormat = cfg.Extensions.ObjectFormat
}
if clientFormat == config.UnsetObjectFormat && serverFormat == config.SHA256 {
ref, err := st.Reference(plumbing.HEAD)
if err == nil && ref.Target().String() == "refs/heads/.invalid" {
if setter, ok := st.(xstorage.ObjectFormatSetter); ok {
err := setter.SetObjectFormat(serverFormat)
if err != nil {
return nil, fmt.Errorf("unable to set object format: %w", err)
}
clientFormat = serverFormat
}
}
}
if clientFormat == config.UnsetObjectFormat {
clientFormat = config.SHA1
}
if serverFormat != clientFormat {
return nil, fmt.Errorf("mismatched algorithms: client %s; server %s", clientFormat, serverFormat)
}
upreq.Capabilities.Set(capability.ObjectFormat, clientFormat.String())
}
if caps.Supports(capability.OFSDelta) {
upreq.Capabilities.Set(capability.OFSDelta)
}
if caps.Supports(capability.Agent) {
upreq.Capabilities.Set(capability.Agent, capability.DefaultAgent())
}
if req.IncludeTags && caps.Supports(capability.IncludeTag) {
upreq.Capabilities.Set(capability.IncludeTag)
}
if req.Filter != "" {
if caps.Supports(capability.Filter) {
upreq.Filter = req.Filter
upreq.Capabilities.Set(capability.Filter)
} else {
return nil, ErrFilterNotSupported
}
}
upreq.Wants = req.Wants
if req.Depth > 0 {
if !caps.Supports(capability.Shallow) {
return nil, ErrShallowNotSupported
}
upreq.Depth = packp.DepthRequest{Deepen: req.Depth}
upreq.Shallows, err = st.Shallow()
if err != nil {
return nil, err
}
}
if isSubset(req.Wants, req.Haves) && len(upreq.Shallows) == 0 {
if err := pktline.WriteFlush(writer); err != nil {
return nil, err
}
if err := writer.Close(); err != nil && !errors.Is(err, io.EOF) {
return nil, fmt.Errorf("closing writer: %w", err)
}
return nil, ErrNoChange
}
common := map[plumbing.Hash]struct{}{}
var statelessCommon []plumbing.Hash
var inVein int
var done bool
var gotContinue bool
var gotReady bool
var sendDoneAfterReady bool
firstRound := true
flushAt := initialFlush
for !done {
var uphav packp.UploadHaves
if statelessRPC && !sendDoneAfterReady {
uphav.Haves = append(uphav.Haves, statelessCommon...)
}
batchSize := 0
if !sendDoneAfterReady {
batchSize = flushAt
if gotContinue {
remaining := maxInVein - inVein
if remaining <= 0 {
batchSize = 0
} else if batchSize > remaining {
batchSize = remaining
}
}
}
for i := 0; i < batchSize && len(req.Haves) > 0; i++ {
uphav.Haves = append(uphav.Haves, req.Haves[len(req.Haves)-1])
req.Haves = req.Haves[:len(req.Haves)-1]
inVein++
}
done = sendDoneAfterReady || len(req.Haves) == 0 || (gotContinue && inVein >= maxInVein)
uphav.Done = done
if isSubset(req.Wants, uphav.Haves) && len(upreq.Shallows) == 0 {
if err := pktline.WriteFlush(writer); err != nil {
return nil, err
}
if err := writer.Close(); err != nil && !errors.Is(err, io.EOF) {
return nil, fmt.Errorf("closing writer: %w", err)
}
return nil, ErrNoChange
}
if firstRound || statelessRPC {
if err := upreq.Encode(writer); err != nil {
return nil, fmt.Errorf("sending upload-request: %w", err)
}
}
readc := make(chan error)
if !statelessRPC {
go func() { readc <- readShallows(statelessRPC, reader, req, &shallowInfo, firstRound) }()
}
if err := uphav.Encode(writer); err != nil {
return nil, fmt.Errorf("sending upload-haves: %w", err)
}
if statelessRPC {
if err := writer.Close(); err != nil {
return nil, fmt.Errorf("closing writer: %w", err)
}
if err := readShallows(statelessRPC, reader, req, &shallowInfo, firstRound); err != nil {
return nil, err
}
} else {
if err := <-readc; err != nil {
return nil, err
}
}
go func() {
defer close(readc)
if done || len(uphav.Haves) > 0 {
var srvrs packp.ServerResponse
if err := srvrs.Decode(reader); err != nil {
readc <- fmt.Errorf("decoding server-response: %w", err)
return
}
applyServerACKs(statelessRPC, srvrs.ACKs, common, &statelessCommon, &gotContinue, &gotReady, &inVein)
}
readc <- nil
}()
if err := <-readc; err != nil {
return nil, err
}
if sendDoneAfterReady {
break
}
if gotReady {
sendDoneAfterReady = true
}
firstRound = false
flushAt = nextFlush(statelessRPC, flushAt)
}
if !statelessRPC {
if err := writer.Close(); err != nil && !errors.Is(err, io.EOF) {
return nil, fmt.Errorf("closing writer: %w", err)
}
}
return shallowInfo, nil
}
func isSubset(needle, haystack []plumbing.Hash) bool {
for _, h := range needle {
if !slices.Contains(haystack, h) {
return false
}
}
return true
}
func readShallows(
statelessRPC bool,
r io.Reader,
req *FetchRequest,
shallowInfo **packp.ShallowUpdate,
firstRound bool,
) error {
if (firstRound || statelessRPC) && req.Depth > 0 {
var shupd packp.ShallowUpdate
if err := shupd.Decode(r); err != nil {
return fmt.Errorf("decoding shallow-update: %w", err)
}
if *shallowInfo == nil {
*shallowInfo = &shupd
}
}
return nil
}
// ReconcileObjectFormatV2 aligns the storer's object format with the Protocol
// v2 server's advertised object-format before any packfile is requested. On a
// fresh clone the storer's format is unset (HEAD still points at the
// refs/heads/.invalid placeholder) and the server's sha256 is adopted;
// otherwise a mismatch is a hard error, since indexing a sha256 pack as sha1
// (or vice versa) corrupts the store and only surfaces later as a checksum
// failure. It mirrors NegotiatePack's v0/v1 object-format handling and git's
// fetch-pack.c, including the case where the server omits object-format (it
// only speaks sha1) but the client repository uses another algorithm.
func ReconcileObjectFormatV2(st storage.Storer, caps capability.List) error {
var clientFormat config.ObjectFormat
if cfg, err := st.Config(); err == nil && cfg != nil {
clientFormat = cfg.Extensions.ObjectFormat
}
advertised := caps.Get(capability.ObjectFormat)
if len(advertised) == 0 {
// The server advertised no object-format, so it only speaks sha1.
// Upstream errors when the client repo uses a different algorithm
// rather than letting it fail later on a checksum mismatch.
if clientFormat != config.UnsetObjectFormat && clientFormat != config.SHA1 {
return fmt.Errorf("the server does not support algorithm '%s'", clientFormat)
}
return nil
}
var serverFormat config.ObjectFormat
switch v := config.ObjectFormat(advertised[0]); v {
case config.SHA1, config.SHA256:
serverFormat = v
case config.UnsetObjectFormat:
// An empty value carries no algorithm; treat it exactly as an absent
// object-format (the server only speaks sha1). Apply the same guard as
// the len(advertised)==0 branch so a client repo on a different
// algorithm is rejected rather than slipping past to fail later on a
// checksum mismatch.
if clientFormat != config.UnsetObjectFormat && clientFormat != config.SHA1 {
return fmt.Errorf("the server does not support algorithm '%s'", clientFormat)
}
return nil
default:
// An algorithm go-git does not speak. Fail fast rather than proceed
// with the wrong hash format, matching NegotiatePack's v0/v1 handling.
return fmt.Errorf("server advertised unsupported object-format %q", v)
}
// Adopt the server format on a fresh clone: unset client + sha256 server,
// with HEAD still at the clone placeholder.
if clientFormat == config.UnsetObjectFormat && serverFormat == config.SHA256 {
if ref, err := st.Reference(plumbing.HEAD); err == nil && ref.Target().String() == "refs/heads/.invalid" {
if setter, ok := st.(xstorage.ObjectFormatSetter); ok {
if err := setter.SetObjectFormat(serverFormat); err != nil {
return fmt.Errorf("unable to set object format: %w", err)
}
clientFormat = serverFormat
}
}
}
if clientFormat == config.UnsetObjectFormat {
clientFormat = config.SHA1
}
if serverFormat != clientFormat {
return fmt.Errorf("mismatched algorithms: client %s; server %s", clientFormat, serverFormat)
}
return nil
}
package transport
import (
"context"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/protocol/capability"
"github.com/go-git/go-git/v6/plumbing/protocol/packp"
"github.com/go-git/go-git/v6/storage"
)
// Commander is an optional capability that Protocol v2-capable sessions
// implement. It provides access to arbitrary v2 commands beyond the
// built-in Fetch and Push operations.
//
// Sessions that negotiate Protocol v2 (version 2) implement this interface.
// The Command method executes a named v2 command: req carries the
// command-specific arguments and is encoded into the request, while resp
// decodes the response. For example, GetRemoteRefs runs
// Command(ctx, "ls-refs", lsRefsArgs, lsRefsOutput). The session builds the v2
// request envelope (command name, the capabilities collected during the
// handshake, delim-pkt, the arguments, and flush-pkt) and, for HTTP, handles
// the response-end packet.
type Commander interface {
Command(ctx context.Context, cmd string, req packp.CommandArgs, resp packp.Decoder) error
}
// Transport is implemented by transports that speak the Git pack
// protocol. Each transport implements this directly — stream transports
// use the NewStreamSession helper, HTTP handles smart/dumb internally.
type Transport interface {
Handshake(ctx context.Context, req *Request) (Session, error)
}
// Session is returned by Transport.Handshake.
type Session interface {
Capabilities() *capability.List
GetRemoteRefs(ctx context.Context, opts *GetRemoteRefsOptions) (*RemoteRefs, error)
Fetch(ctx context.Context, st storage.Storer, req *FetchRequest) error
Push(ctx context.Context, st storage.Storer, req *PushRequest) error
Close() error
}
// GetRemoteRefsOptions configures Session.GetRemoteRefs. A nil pointer
// requests all references with default behavior, matching git's
// transport_get_remote_refs(transport, NULL).
type GetRemoteRefsOptions struct {
// RefPrefixes limits the returned references to those matching the
// given prefixes. For Protocol v2 these map directly to ls-refs
// ref-prefix arguments. For v0/v1 the server always advertises every
// reference, so prefixes are ignored.
RefPrefixes []string
}
// RemoteRefs holds the result of Session.GetRemoteRefs. It is a struct so
// that new output fields can be added without changing the interface.
type RemoteRefs struct {
// References are the advertised references, with HEAD resolved to a
// symbolic reference when the server reports a symref target.
References []*plumbing.Reference
// Unborn is the symref target of HEAD when HEAD points at an unborn
// branch. It is empty when HEAD is not unborn or the server does not
// report it (v0/v1).
Unborn plumbing.ReferenceName
}
// NewRemoteRefs builds a RemoteRefs from a resolved reference list,
// detecting an unborn HEAD: a symbolic HEAD whose target has no
// corresponding hash reference in the advertisement.
func NewRemoteRefs(refs []*plumbing.Reference) *RemoteRefs {
// A detached remote HEAD is advertised as a bare hash. The v0/v1
// advertisement resolves it to a symbolic HEAD during decode; the v2
// ls-refs path does not, so apply the same hash→branch heuristic here so a
// clone records a symbolic HEAD rather than a detached one (matching git).
for i, ref := range refs {
if ref.Name() == plumbing.HEAD && ref.Type() == plumbing.HashReference {
refs[i] = packp.ResolveHeadFromHashHeuristic(ref, refs)
break
}
}
rr := &RemoteRefs{References: refs}
var headTarget plumbing.ReferenceName
hashRefs := make(map[plumbing.ReferenceName]struct{}, len(refs))
for _, ref := range refs {
switch {
case ref.Type() == plumbing.HashReference:
hashRefs[ref.Name()] = struct{}{}
case ref.Name() == plumbing.HEAD && ref.Type() == plumbing.SymbolicReference:
headTarget = ref.Target()
}
}
if headTarget != "" {
if _, ok := hashRefs[headTarget]; !ok {
rr.Unborn = headTarget
}
}
return rr
}
package transport
import (
"bufio"
"context"
"errors"
"io"
"strings"
internal "github.com/go-git/go-git/v6/internal/transport"
"github.com/go-git/go-git/v6/plumbing/protocol"
"github.com/go-git/go-git/v6/plumbing/protocol/capability"
"github.com/go-git/go-git/v6/plumbing/protocol/packp"
"github.com/go-git/go-git/v6/storage"
"github.com/go-git/go-git/v6/utils/ioutil"
)
// StreamSession implements Session over a full-duplex stream.
// Stream transports (SSH, Git TCP, file) call NewStreamSession from
// their Handshake implementation.
type StreamSession struct {
conn Conn
r *bufio.Reader
w io.WriteCloser
svc string
version protocol.Version
caps capability.List
refs *packp.AdvRefs
}
// NewStreamSession creates a session from an open Conn.
// For pack services (upload-pack, receive-pack), it reads the version
// and advertised refs from the stream. For upload-archive, it skips
// that — the archive protocol has no ref advertisement.
func NewStreamSession(conn Conn, service string) (*StreamSession, error) {
r := bufio.NewReader(conn.Reader())
w := conn.Writer()
s := &StreamSession{
conn: conn,
r: r,
w: w,
svc: service,
}
if service == UploadArchiveService {
return s, nil
}
ver, err := DiscoverVersion(r)
if err != nil {
_ = conn.Close()
return nil, err
}
s.version = ver
if ver == protocol.V2 {
// Protocol v2: the server sends a capability advertisement
// (version line + capability lines) instead of the v0/v1 ref
// advertisement. References are retrieved lazily via the ls-refs
// command, so nothing is read here beyond the advertisement.
adv := &packp.CapabilityAdv{}
if err := adv.Decode(r); err != nil {
_ = conn.Close()
return nil, err
}
s.caps = adv.Capabilities
// Protocol v2 fetch accepts "want <oid>" without the server
// advertising allow-*-sha1-in-want, so surface the gate as
// satisfied for exact-SHA1 refspecs (isSupportedRefSpec). The
// v2 client only sends agent/object-format on the wire, so these
// never leak into the request (internal.ClientCapabilities).
s.caps.Set(capability.AllowReachableSHA1InWant)
s.caps.Set(capability.AllowTipSHA1InWant)
return s, nil
}
ar := &packp.AdvRefs{}
if err := ar.Decode(r); err != nil && !errors.Is(err, packp.ErrEmptyAdvRefs) {
_ = conn.Close()
return nil, err
}
// Validate capabilities before returning the session.
if err := capability.Validate(&ar.Capabilities); err != nil {
_ = conn.Close()
return nil, err
}
// Source the advertisement's version from the version DiscoverVersion
// already established, so s.version is the single source of truth rather
// than relying on AdvRefs.Decode's independent parse of the same line.
ar.Version = ver
s.caps = ar.Capabilities
s.refs = ar
return s, nil
}
// Capabilities implements PackSession.
func (s *StreamSession) Capabilities() *capability.List { return &s.caps }
// GetRemoteRefs implements Session. For v0/v1 the server advertises every
// reference during the handshake, so opts is ignored. For v2 the references
// are retrieved on demand via the ls-refs command, honoring the ref-prefix
// filters in opts.
func (s *StreamSession) GetRemoteRefs(ctx context.Context, opts *GetRemoteRefsOptions) (*RemoteRefs, error) {
forPush := s.svc == ReceivePackService
if s.version == protocol.V2 {
var prefixes []string
if opts != nil {
prefixes = opts.RefPrefixes
}
refs, err := internal.LsRefs(ctx, s.Command, s.caps, prefixes)
if err != nil {
return nil, err
}
if !forPush && !internal.HasHashRef(refs) {
return nil, ErrEmptyRemoteRepository
}
return NewRemoteRefs(refs), nil
}
if s.refs == nil {
return nil, ErrEmptyRemoteRepository
}
if !forPush && s.refs.IsEmpty() {
return nil, ErrEmptyRemoteRepository
}
refs, err := s.refs.ResolvedReferences()
if err != nil {
return nil, err
}
return NewRemoteRefs(refs), nil
}
// Fetch implements PackSession.
func (s *StreamSession) Fetch(ctx context.Context, st storage.Storer, req *FetchRequest) error {
if s.version == protocol.V2 {
if req.Filter != "" && !internal.FetchSupports(s.caps, "filter") {
return ErrFilterNotSupported
}
if req.Depth > 0 && !internal.FetchSupports(s.caps, "shallow") {
return ErrShallowNotSupported
}
if err := ReconcileObjectFormatV2(st, s.caps); err != nil {
return err
}
// Each negotiation round reuses the persistent stream: Command writes
// the request and decodes the metadata, leaving s.r at the packfile.
round := func(args *packp.FetchArgs) (*packp.FetchOutput, io.Reader, error) {
out := &packp.FetchOutput{}
if err := s.Command(ctx, "fetch", args, out); err != nil {
return nil, nil, err
}
return out, s.r, nil
}
if err := internal.FetchV2(ctx, st, req, round); err != nil {
return s.wrapStderr(err)
}
return nil
}
shallows, err := NegotiatePack(ctx, st, s.caps, false, s.r, s.w, req)
if err != nil {
return s.wrapStderr(err)
}
if err := FetchPack(ctx, st, s.caps, io.NopCloser(s.r), shallows, req); err != nil {
return s.wrapStderr(err)
}
return nil
}
// Push implements PackSession.
func (s *StreamSession) Push(ctx context.Context, st storage.Storer, req *PushRequest) error {
if err := SendPack(ctx, st, s.caps, s.w, io.NopCloser(s.r), req); err != nil {
return s.wrapStderr(err)
}
return nil
}
// Command implements Commander. It builds a Protocol v2 request envelope for
// the named command, encodes it, and decodes the response. The request
// carries the capabilities collected during the handshake (the agent and the
// server's object-format), so callers only provide the command arguments.
//
// Command is only valid on a session that negotiated Protocol v2.
func (s *StreamSession) Command(ctx context.Context, cmd string, req packp.CommandArgs, resp packp.Decoder) error {
if s.version != protocol.V2 {
return ErrUnsupportedVersion
}
cr := &packp.CommandRequest{
Command: cmd,
Capabilities: s.commandCapabilities(),
Args: req,
}
if err := cr.Encode(ioutil.NewContextWriter(ctx, s.w)); err != nil {
return s.wrapStderr(err)
}
if resp != nil {
if err := resp.Decode(ioutil.NewContextReader(ctx, s.r)); err != nil {
return s.wrapStderr(err)
}
}
return nil
}
// commandCapabilities returns the capabilities the client sends with each v2
// command. Both the agent and the object-format are gated on the server having
// advertised them (see internal.ClientCapabilities), so the client never sends
// a capability the server did not offer; the object-format is echoed back so
// both sides agree on the hash algorithm.
func (s *StreamSession) commandCapabilities() capability.List {
return internal.ClientCapabilities(s.caps)
}
// wrapStderr checks if the underlying connection has stderr output and
// returns it as a RemoteError so that remote error messages surface at
// the operation site rather than at Close time.
func (s *StreamSession) wrapStderr(err error) error {
type stderrer interface {
Stderr() io.Reader
}
if se, ok := s.conn.(stderrer); ok {
if r := se.Stderr(); r != nil {
b, readErr := io.ReadAll(r)
if readErr == nil && len(b) > 0 {
return NewRemoteError(strings.TrimSpace(string(b)))
}
}
}
return err
}
// Close implements Session.
func (s *StreamSession) Close() error { return s.conn.Close() }
// Archive implements Archiver. It speaks the git-upload-archive wire
// protocol over the session's existing connection.
func (s *StreamSession) Archive(ctx context.Context, req *ArchiveRequest) (io.ReadCloser, error) {
if s.svc != UploadArchiveService {
return nil, ErrArchiveUnsupported
}
rc := ioutil.NewReadCloser(s.conn.Reader(), s.conn)
archive, err := Archive(ctx, s.conn.Writer(), rc, req)
if err != nil {
_ = rc.Close()
return nil, err
}
return archive, nil
}
var (
_ Session = (*StreamSession)(nil)
_ Archiver = (*StreamSession)(nil)
_ Commander = (*StreamSession)(nil)
)
package transport
import (
"fmt"
"github.com/go-git/go-git/v6/plumbing/protocol"
)
// GitProtocolEnv returns the value for the GIT_PROTOCOL environment variable
// corresponding to the given protocol version. Returns an empty string for
// protocol V0, which does not use GIT_PROTOCOL.
func GitProtocolEnv(v protocol.Version) string {
switch v {
case protocol.V0, protocol.Undefined:
return ""
default:
return fmt.Sprintf("version=%s", v)
}
}
package transport
import (
"context"
"errors"
"fmt"
"io"
"github.com/go-git/go-git/v6/plumbing/protocol/capability"
"github.com/go-git/go-git/v6/plumbing/protocol/packp"
"github.com/go-git/go-git/v6/plumbing/protocol/packp/sideband"
"github.com/go-git/go-git/v6/storage"
"github.com/go-git/go-git/v6/utils/ioutil"
)
// SendPack sends a packfile to a remote server.
func SendPack(
ctx context.Context,
_ storage.Storer,
caps capability.List,
writer io.WriteCloser,
reader io.ReadCloser,
req *PushRequest,
) error {
writer = ioutil.NewContextWriteCloser(ctx, writer)
reader = ioutil.NewContextReadCloser(ctx, reader)
var needPackfile bool
for _, cmd := range req.Commands {
if cmd.Action() != packp.Delete {
needPackfile = true
break
}
}
if !needPackfile && req.Packfile != nil {
return fmt.Errorf("packfile is not accepted for push request without new objects")
}
if needPackfile && req.Packfile == nil {
return fmt.Errorf("packfile is required for push request with new objects")
}
upreq := buildUpdateRequests(caps, req)
if err := upreq.Encode(writer); err != nil {
return err
}
if upreq.Capabilities.Supports(capability.PushOptions) {
var opts packp.PushOptions
opts.Options = req.Options
if err := opts.Encode(writer); err != nil {
return fmt.Errorf("encoding push-options: %w", err)
}
}
if req.Packfile != nil {
if _, err := ioutil.CopyBufferPool(writer, req.Packfile); err != nil {
return err
}
if err := req.Packfile.Close(); err != nil {
return fmt.Errorf("closing packfile: %w", err)
}
}
if err := writer.Close(); err != nil {
return err
}
var reportStatus int
if upreq.Capabilities.Supports(capability.ReportStatusV2) {
reportStatus = 2
} else if upreq.Capabilities.Supports(capability.ReportStatus) {
reportStatus = 1
}
if reportStatus == 0 {
return nil
}
var r io.Reader = reader
if req.Progress != nil {
var d *sideband.Demuxer
if upreq.Capabilities.Supports(capability.Sideband64k) {
d = sideband.NewDemuxer(sideband.Sideband64k, reader)
} else if upreq.Capabilities.Supports(capability.Sideband) {
d = sideband.NewDemuxer(sideband.Sideband, reader)
}
if d != nil {
if !upreq.Capabilities.Supports(capability.Quiet) {
d.Progress = req.Progress
}
r = d
}
}
report := &packp.ReportStatus{}
if err := report.Decode(r); err != nil {
return fmt.Errorf("decode report-status: %w", err)
}
reportError := report.Error()
if reportStatus > 0 && len(upreq.Commands) > 0 {
_, err := io.ReadAll(r)
if err != nil && !errors.Is(err, io.EOF) {
_ = reader.Close()
if reportError != nil {
return reportError
}
return fmt.Errorf("reading progress messages: %w", err)
}
}
if err := reader.Close(); err != nil {
if reportError != nil {
return reportError
}
return fmt.Errorf("closing reader: %w", err)
}
return reportError
}
func buildUpdateRequests(caps capability.List, req *PushRequest) *packp.UpdateRequests {
upreq := &packp.UpdateRequests{}
if caps.Supports(capability.ReportStatus) {
upreq.Capabilities.Set(capability.ReportStatus)
}
if req.Progress != nil {
if caps.Supports(capability.Sideband64k) {
upreq.Capabilities.Set(capability.Sideband64k)
} else if caps.Supports(capability.Sideband) {
upreq.Capabilities.Set(capability.Sideband)
}
if req.Quiet && caps.Supports(capability.Quiet) {
upreq.Capabilities.Set(capability.Quiet)
}
}
if req.Atomic && caps.Supports(capability.Atomic) {
upreq.Capabilities.Set(capability.Atomic)
}
if len(req.Options) > 0 && caps.Supports(capability.PushOptions) {
upreq.Capabilities.Set(capability.PushOptions)
}
if caps.Supports(capability.Agent) {
upreq.Capabilities.Set(capability.Agent, capability.DefaultAgent())
}
upreq.Commands = req.Commands
return upreq
}
package transport
import (
"bufio"
"context"
"errors"
"fmt"
"io"
"github.com/go-git/go-git/v6/internal/pathutil"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/format/packfile"
"github.com/go-git/go-git/v6/plumbing/format/pktline"
"github.com/go-git/go-git/v6/plumbing/protocol"
"github.com/go-git/go-git/v6/plumbing/protocol/capability"
"github.com/go-git/go-git/v6/plumbing/protocol/packp"
"github.com/go-git/go-git/v6/plumbing/protocol/packp/sideband"
"github.com/go-git/go-git/v6/plumbing/storer"
"github.com/go-git/go-git/v6/storage"
"github.com/go-git/go-git/v6/utils/ioutil"
)
// ReceivePackRequest is a set of options for the ReceivePack service.
type ReceivePackRequest struct {
GitProtocol string
AdvertiseRefs bool
StatelessRPC bool
// Hooks are optional server-side callbacks. The zero value installs none.
Hooks ReceivePackHooks
}
// ReceivePackHooks holds server-side callbacks for ReceivePack.
//
// These are the in-process equivalent of git's pre-receive and post-receive
// hooks. They run after the packfile has been unpacked into the storer but
// before (PreReceive) and after (PostReceive) ref updates, so a server can
// enforce branch protection, signed-commit checks, or other policy without
// reimplementing receive-pack.
type ReceivePackHooks struct {
// PreReceive runs after the packfile is unpacked but before any ref is
// updated. Returning a non-nil error refuses every ref with err.Error()
// as the report-status reason; refs are not updated and PostReceive is
// not run.
PreReceive func(context.Context, *PreReceiveInfo) error
// PostReceive runs after refs are updated. Any returned error is ignored
// for transport purposes: the refs have already moved and the
// report-status sent to the client reflects the ref-update outcome, not
// this error. The hook itself must handle or log failures it cares about.
PostReceive func(context.Context, *PostReceiveInfo) error
}
// PreReceiveInfo carries the inputs to a PreReceive hook.
type PreReceiveInfo struct {
// Storer reads the proposed new state: the objects from this push are
// already present alongside the existing repository.
Storer storage.Storer
// Commands are the proposed ref updates. Treat as read-only.
Commands []*packp.Command
// PushOptions are the client's push options (empty if none).
PushOptions []string
// Progress writes to the client's sideband progress channel (band 2) when
// negotiated, or is io.Discard otherwise. Valid only during the call.
Progress io.Writer
}
// PostReceiveInfo carries the inputs to a PostReceive hook.
type PostReceiveInfo struct {
// Storer reads the committed repository state.
Storer storage.Storer
// Commands are the ref updates that were applied successfully. Refs whose
// update failed are omitted. Treat as read-only.
Commands []*packp.Command
// PushOptions are the client's push options (empty if none).
PushOptions []string
// Progress writes to the client's sideband progress channel (band 2) when
// negotiated, or is io.Discard otherwise. Valid only during the call.
Progress io.Writer
}
// ReceivePack is a server command that serves the receive-pack service.
// It closes w on every return, including malformed requests and advertisement-only
// exchanges. A close error is returned only if no earlier error occurred.
// Callers that retain ownership of their streams should wrap r with [io.NopCloser]
// and w with [ioutil.WriteNopCloser].
//
// Commands may name only references under refs/ with valid syntax and safe path
// components. Refusals are reported per command when report-status is negotiated.
// A request naming the same reference twice is rejected before hooks or reference
// updates run. See ErrFunnyRefname and ErrDuplicateRefname.
// Commands execute even when report-status is not requested. Updates use the
// storer's compare-and-set operation on the resolved target. Symbolic resolution
// is separate from the update, and deletes check the old value before a separate
// removal. ReferenceStorer has no transaction covering these steps: callers must
// serialize concurrent writers if they require the entire operation to be atomic.
func ReceivePack(
ctx context.Context,
st storage.Storer,
r io.ReadCloser,
w io.WriteCloser,
opts *ReceivePackRequest,
) (err error) {
if w == nil {
return fmt.Errorf("nil writer")
}
w = ioutil.NewContextWriteCloser(ctx, w)
// Every exit from here on closes the writer, because the close is what ends
// the response for the caller's transport: a return that skips it leaves a
// client waiting on a stream that will never end. That holds for the early
// returns too, where nothing has been written yet, and for a refused ref,
// where the "ng <ref> <reason>" line is the response.
//
// The close error only surfaces when nothing else went wrong: a rejected
// command or a malformed request describes the exchange better than a
// failure to hang up does.
defer func() {
if closeErr := closeWriter(w); closeErr != nil && err == nil {
err = closeErr
}
}()
if opts == nil {
opts = &ReceivePackRequest{}
}
if opts.AdvertiseRefs || !opts.StatelessRPC {
v := ProtocolVersion(opts.GitProtocol)
switch v {
case protocol.V0, protocol.V1, protocol.V2:
// version emission (if any) is handled inside AdvertiseRefs for correct
// ordering with the HTTP smart-reply prefix when applicable.
default:
return fmt.Errorf("%w: %q", ErrUnsupportedVersion, v)
}
if err := AdvertiseRefs(ctx, st, w, ReceivePackService, opts.StatelessRPC, v); err != nil {
return err
}
}
if opts.AdvertiseRefs {
// Done, there's nothing else to do
return nil
}
if r == nil {
return fmt.Errorf("nil reader")
}
r = ioutil.NewContextReadCloser(ctx, r)
rd := bufio.NewReader(r)
l, _, err := pktline.PeekLine(rd)
if err != nil {
return err
}
// At this point, if we get a flush packet, it means the client
// has nothing to send, so we can return early.
if l == pktline.Flush {
return nil
}
updreq := &packp.UpdateRequests{}
if err := updreq.Decode(rd); err != nil {
return err
}
var (
caps = updreq.Capabilities
needPackfile bool
pushOpts packp.PushOptions
)
if updreq.Capabilities.Supports(capability.PushOptions) {
if err := pushOpts.Decode(rd); err != nil {
return fmt.Errorf("decoding push-options: %w", err)
}
}
// Should we expect a packfile?
for _, cmd := range updreq.Commands {
if cmd.Action() != packp.Delete {
needPackfile = true
break
}
}
// Receive the packfile
var unpackErr error
if needPackfile {
unpackErr = packfile.UpdateObjectStorage(st, rd)
}
// Done with the request, now close the reader
// to indicate that we are done reading from it.
if err := r.Close(); err != nil {
return fmt.Errorf("closing reader: %w", err)
}
reportStatus := caps.Supports(capability.ReportStatus) || caps.Supports(capability.ReportStatusV2)
var (
useSideband bool
writer io.Writer = w
progress = io.Writer(io.Discard)
)
if !caps.Supports(capability.NoProgress) {
var mux *sideband.Muxer
if caps.Supports(capability.Sideband64k) {
mux = sideband.NewMuxer(sideband.Sideband64k, w)
} else if caps.Supports(capability.Sideband) {
mux = sideband.NewMuxer(sideband.Sideband, w)
}
if mux != nil {
writer = mux
progress = sidebandProgress{mux}
useSideband = true
}
}
writeCloser := ioutil.NewWriteCloser(writer, w)
// report is how every remaining exit answers the client: the report-status,
// then the flush that ends the sideband stream when one is in use.
// ReportStatus.Encode writes a flush of its own, but on a sideband exchange
// that one is muxed into band 1 along with the rest of the report, so it
// does not terminate the stream the client is demuxing. Routing all three
// exits through one function is what stops one of them from answering
// without that second flush.
report := func(unpackErr error, cmdStatus map[plumbing.ReferenceName]error) error {
if reportStatus {
if err := sendReportStatus(writeCloser, updreq.Commands, unpackErr, cmdStatus); err != nil {
return err
}
}
if !useSideband {
return nil
}
if err := pktline.WriteFlush(w); err != nil {
return fmt.Errorf("flushing sideband: %w", err)
}
return nil
}
if unpackErr != nil {
// No command was attempted, so there is no per-ref outcome to give and
// the unpack line carries the whole reason. The error still goes back
// to the caller: writing the report successfully does not turn a failed
// push into a successful exchange, and the two failure exits below
// answer the same way.
if err := report(unpackErr, nil); err != nil {
return err
}
return unpackErr
}
// A name carried by more than one command makes the push unapplyable: the
// commands contradict each other, and cmdStatus holds a single outcome per
// name, so running both would report one of them and hide the other.
//
// git refuses such a push outright. It batches the ref updates into a
// transaction, and a repeated name aborts the transaction with "multiple
// updates for ref <name> not allowed", so no ref in the batch moves and
// every command is answered "ng" — including the commands that named a
// reference only once. Refuse the whole request the same way, rather than
// only the duplicated name, so a push either applies as sent or not at all.
//
// Unlike git this runs before PreReceive. A hook is a policy gate that may
// have side effects of its own, so it is not asked to authorise a request
// that cannot be applied whatever it answers.
if dup, ok := duplicateRefname(updreq.Commands); ok {
rejected := make(map[plumbing.ReferenceName]error, len(updreq.Commands))
for _, cmd := range updreq.Commands {
// sendReportStatus writes Error() into the "ng <ref> <reason>"
// line, so the reason stays the bare sentinel: the line already
// names the ref it speaks for. Git sends "failed to update refs"
// here; this says more, and both are opaque to a client. Which
// name was duplicated goes to the caller instead.
rejected[cmd.Name] = ErrDuplicateRefname
}
if err := report(nil, rejected); err != nil {
return err
}
return fmt.Errorf("%w: %q", ErrDuplicateRefname, dup)
}
if opts.Hooks.PreReceive != nil {
info := &PreReceiveInfo{
Storer: st,
Commands: updreq.Commands,
PushOptions: pushOpts.Options,
Progress: progress,
}
if hookErr := opts.Hooks.PreReceive(ctx, info); hookErr != nil {
rejected := make(map[plumbing.ReferenceName]error, len(updreq.Commands))
for _, cmd := range updreq.Commands {
rejected[cmd.Name] = hookErr
}
if err := report(nil, rejected); err != nil {
return err
}
return hookErr
}
}
var firstErr error
cmdStatus := make(map[plumbing.ReferenceName]error)
updateReferences(st, updreq, cmdStatus, &firstErr)
if opts.Hooks.PostReceive != nil {
applied := make([]*packp.Command, 0, len(updreq.Commands))
for _, cmd := range updreq.Commands {
if cmdStatus[cmd.Name] == nil {
applied = append(applied, cmd)
}
}
info := &PostReceiveInfo{
Storer: st,
Commands: applied,
PushOptions: pushOpts.Options,
Progress: progress,
}
_ = opts.Hooks.PostReceive(ctx, info)
}
// The unpack status reports on the packfile, not on the ref updates: it is
// "ok" here because unpackErr was handled above. Per-command failures are
// carried by cmdStatus as "ng <ref> <reason>" lines, exactly as the
// PreReceive rejection path does; folding firstErr into the unpack status
// would make a client treat a single refused ref as a corrupt push.
if err := report(nil, cmdStatus); err != nil {
return err
}
return firstErr
}
type sidebandProgress struct{ mux *sideband.Muxer }
func (p sidebandProgress) Write(b []byte) (int, error) {
return p.mux.WriteChannel(sideband.ProgressMessage, b)
}
func closeWriter(w io.WriteCloser) error {
if err := w.Close(); err != nil {
return fmt.Errorf("closing writer: %w", err)
}
return nil
}
// sendReportStatus writes the report-status for the exchange: the unpack line,
// then one status line per command.
//
// The lines follow cmds, not the cmdStatus map. Git reports in the order the
// commands arrived and a client is entitled to pair the two up positionally,
// whereas ranging over the map orders them differently on every push. A command
// with no entry in cmdStatus was never attempted and is not reported.
//
// One line is written per command, not per distinct name, which is what keeps
// that pairing positional: git answers a name carried by two commands with two
// ng lines. Both lines say the same thing here, because a duplicated name is
// refused before any command runs and the map holds one outcome for it.
func sendReportStatus(w io.WriteCloser, cmds []*packp.Command, unpackErr error, cmdStatus map[plumbing.ReferenceName]error) error {
rs := &packp.ReportStatus{}
rs.UnpackStatus = "ok"
if unpackErr != nil {
rs.UnpackStatus = unpackErr.Error()
}
for _, cmd := range cmds {
err, ok := cmdStatus[cmd.Name]
if !ok {
continue
}
msg := "ok"
if err != nil {
msg = err.Error()
}
status := &packp.CommandStatus{
ReferenceName: cmd.Name,
Status: msg,
}
rs.CommandStatuses = append(rs.CommandStatuses, status)
}
if err := rs.Encode(w); err != nil {
return err
}
return nil
}
// duplicateRefname returns the first reference name that more than one command
// in cmds updates, and whether there was one.
func duplicateRefname(cmds []*packp.Command) (plumbing.ReferenceName, bool) {
seen := make(map[plumbing.ReferenceName]struct{}, len(cmds))
for _, cmd := range cmds {
if _, ok := seen[cmd.Name]; ok {
return cmd.Name, true
}
seen[cmd.Name] = struct{}{}
}
return "", false
}
func setStatus(cmdStatus map[plumbing.ReferenceName]error, firstErr *error, ref plumbing.ReferenceName, err error) {
cmdStatus[ref] = err
if *firstErr == nil && err != nil {
*firstErr = err
}
}
// checkRefname returns [ErrFunnyRefname] if receive-pack must refuse a command
// naming ref, and nil if the name may reach the storer.
//
// It mirrors the gate in Git's builtin/receive-pack.c (execute_commands_non_atomic
// -> update, "only refs/... are allowed"), which refuses a command whose name
// is not under refs/ or fails check_refname_format, reporting "funny refname".
// Without it, a push can name HEAD, CONFIG, INDEX or SHALLOW and reach the
// storer: writing HEAD repoints the repository's default branch for every
// later clone on any filesystem, and on a case-insensitive one the shouting
// names land on .git/config, .git/index and .git/shallow. A Delete command
// needs no packfile at all, so the same names also give an unauthenticated
// "remove .git/config" primitive.
//
// The name is checked in four steps:
//
// - the refs/ prefix, which is what stops HEAD and every other root ref.
// Git relaxes its format check for deletes (REFNAME_ALLOW_ONELEVEL) but
// never relaxes this prefix, and neither do we: deleting a ref is the
// cheapest form of this attack, not the most benign.
// - ReferenceName.IsSafe, Git's refname_is_safe, for names that escape the
// refs/ sub-tree or alias another path once joined.
// - pathutil.HasUnsafeComponent, for the escapes IsSafe's literal ".."
// comparison misses: control characters, and the components an HFS+ or
// NTFS filesystem folds back to "." or "..". The dotgit storage layer
// applies the same helper, but this gate cannot lean on it: ReceivePack is
// exported and can be handed any storer, including one that never reaches
// a filesystem.
// - ReferenceName.Validate, go-git's check_refname_format, for the remaining
// character and component rules.
//
// Three of the four are decisive somewhere. The refs/ prefix is the only thing
// that refuses HEAD, which IsSafe accepts, HasUnsafeComponent passes, and
// Validate carves out by name. IsSafe is the exception: with the prefix already
// required, every name it rejects is one Validate also rejects, by rules 1, 3,
// 6 and 10. It stays because this gate should not depend on that overlap
// holding as either function changes.
//
// Validating the full name, rather than Git's suffix after refs/, has two
// compatibility differences:
//
// - Git runs check_refname_format on the part after "refs/" and passes
// REFNAME_ALLOW_ONELEVEL only for deletes, so it refuses to *create*
// refs/stash ("funny refname") while allowing it to be deleted. go-git
// validates the whole name, which accepts one level under refs/ for every
// action. refs/stash is a first-class ref here, and a single component
// under refs/ cannot escape the sub-tree, so the asymmetry would cost
// compatibility and buy no safety.
// - For refs/@, Git tests the suffix "@" and rejects it for every action.
// go-git accepts it because "@" is not the entire reference name. This
// preserves the existing support for names accepted by Validate.
//
// An additional filesystem-safety restriction applies to every action:
//
// - HasUnsafeComponent refuses a component whose first non-ignorable code
// point is a lone ".", which check_refname_format accepts:
// refs/heads/<U+200C>./x is "ok" to Git and "funny refname" here. On HFS+
// that component normalises away and the name lands on refs/heads/x, which
// is a filesystem hazard Git's format check does not model.
//
// Git relaxes only the component count for a delete and keeps every other
// format rule at its receive-pack gate. The wider
// relaxation it grants in ref_transaction_update — refname_is_safe on its own —
// is for a local caller rather than a remote one.
//
// The error is returned bare on purpose: sendReportStatus writes Error()
// verbatim into the "ng <ref> <reason>" line, so wrapping it with extra context
// would hand the client a status git never sends.
//
// https://github.com/git/git/blob/1630431f326e15fcde608827b5ff38422528eb59/builtin/receive-pack.c#L1491-L1499
func checkRefname(ref plumbing.ReferenceName) error {
if !ref.IsUnderRefs() {
return ErrFunnyRefname
}
if !ref.IsSafe() {
return ErrFunnyRefname
}
if pathutil.HasUnsafeComponent(ref.String()) {
return ErrFunnyRefname
}
if err := ref.Validate(); err != nil {
return ErrFunnyRefname
}
return nil
}
func updateReferences(st storage.Storer, req *packp.UpdateRequests, cmdStatus map[plumbing.ReferenceName]error, firstErr *error) {
for _, cmd := range req.Commands {
if err := checkRefname(cmd.Name); err != nil {
setStatus(cmdStatus, firstErr, cmd.Name, err)
continue
}
var current *plumbing.Reference
var err error
if cmd.Action() == packp.Create {
// An existing symbolic ref still occupies the name when its
// target is missing. A create must not overwrite that alias.
current, err = st.Reference(cmd.Name)
} else {
current, err = storer.ResolveReference(st, cmd.Name)
}
exists := err == nil
if err != nil && !errors.Is(err, plumbing.ErrReferenceNotFound) {
setStatus(cmdStatus, firstErr, cmd.Name, err)
continue
}
switch cmd.Action() {
case packp.Create:
if exists {
setStatus(cmdStatus, firstErr, cmd.Name, ErrUpdateReference)
continue
}
ref := plumbing.NewHashReference(cmd.Name, cmd.New)
err := st.SetReference(ref)
setStatus(cmdStatus, firstErr, cmd.Name, err)
case packp.Delete:
if !exists {
setStatus(cmdStatus, firstErr, cmd.Name, ErrUpdateReference)
continue
}
if current.Hash() != cmd.Old {
// Git permits removal of a corrupt ref when the supplied old
// object is missing. A present old object must match the ref.
// See https://github.com/git/git/blob/1630431f326e15fcde608827b5ff38422528eb59/builtin/receive-pack.c#L1604-L1619.
_, objectErr := st.EncodedObject(plumbing.AnyObject, cmd.Old)
if objectErr == nil {
setStatus(cmdStatus, firstErr, cmd.Name, storage.ErrReferenceHasChanged)
continue
}
if !errors.Is(objectErr, plumbing.ErrObjectNotFound) {
setStatus(cmdStatus, firstErr, cmd.Name, objectErr)
continue
}
}
err := st.RemoveReference(current.Name())
setStatus(cmdStatus, firstErr, cmd.Name, err)
case packp.Update:
if !exists {
setStatus(cmdStatus, firstErr, cmd.Name, ErrUpdateReference)
continue
}
ref := plumbing.NewHashReference(current.Name(), cmd.New)
old := plumbing.NewHashReference(current.Name(), cmd.Old)
err := st.CheckAndSetReference(ref, old)
setStatus(cmdStatus, firstErr, cmd.Name, err)
}
}
}
package transport
import (
"context"
"errors"
"fmt"
"io"
"github.com/go-git/go-git/v6/internal/reference"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/format/config"
"github.com/go-git/go-git/v6/plumbing/object"
"github.com/go-git/go-git/v6/plumbing/protocol"
"github.com/go-git/go-git/v6/plumbing/protocol/capability"
"github.com/go-git/go-git/v6/plumbing/protocol/packp"
"github.com/go-git/go-git/v6/plumbing/storer"
"github.com/go-git/go-git/v6/storage"
"github.com/go-git/go-git/v6/utils/trace"
)
// ErrUpdateReference is returned when a reference update fails.
var ErrUpdateReference = errors.New("failed to update ref")
// ErrFunnyRefname is returned when a push names a reference the server refuses
// to touch: one that is not under refs/, or one whose name is malformed.
var ErrFunnyRefname = errors.New("funny refname")
// ErrDuplicateRefname is reported for every command of a push whose command
// list updates one reference more than once.
var ErrDuplicateRefname = errors.New("multiple updates for ref not allowed")
// AdvertiseRefs is a server command that implements the reference
// discovery phase of the v0/v1 Git transfer protocol. Protocol v2 advertises
// capabilities only, via [AdvertiseCapabilities]; the sole reason this function
// accepts protocol.V2 is the receive-pack fallback: v2 has no push, so when a
// client requests v2 for receive-pack git ignores it and serves a classic
// advertisement (builtin/receive-pack.c), while http-backend still suppresses
// the "# service=..." smart-reply line for the v2 request (http-backend.c
// get_info_refs). Both behaviours are reproduced below.
func AdvertiseRefs(
_ context.Context,
st storage.Storer,
w io.Writer,
service string,
smart bool,
version protocol.Version,
) error {
switch service {
case UploadPackService, ReceivePackService:
default:
return fmt.Errorf("%w: %s", ErrUnsupportedService, service)
}
forPush := service == ReceivePackService
ar := &packp.AdvRefs{}
// Set server default capabilities
ar.Capabilities.Set(capability.Agent, capability.DefaultAgent())
ar.Capabilities.Set(capability.OFSDelta)
ar.Capabilities.Set(capability.Sideband64k)
if forPush {
// TODO: support thin-pack
ar.Capabilities.Set(capability.NoThin)
// TODO: support atomic
ar.Capabilities.Set(capability.DeleteRefs)
ar.Capabilities.Set(capability.ReportStatus)
ar.Capabilities.Set(capability.PushOptions)
ar.Capabilities.Set(capability.Quiet)
} else {
// TODO: support include-tag
// TODO: support deepen
// TODO: support deepen-since
ar.Capabilities.Set(capability.MultiACK)
ar.Capabilities.Set(capability.MultiACKDetailed)
ar.Capabilities.Set(capability.Sideband)
ar.Capabilities.Set(capability.NoProgress)
ar.Capabilities.Set(capability.Shallow)
ar.Capabilities.Set(capability.ObjectFormat, objectFormat(st).String())
}
// Set references
if err := addReferences(st, ar, !forPush); err != nil {
return err
}
// Validate capabilities before sending the response.
if err := capability.Validate(&ar.Capabilities); err != nil {
return fmt.Errorf("invalid capabilities: %w", err)
}
// git's http-backend omits the "# service=..." smart reply whenever the
// requested protocol is v2, even for receive-pack which then falls back to
// a v0 advertisement (http-backend.c get_info_refs).
if smart && version != protocol.V2 {
smartReply := packp.SmartReply{
Service: service,
}
if err := smartReply.Encode(w); err != nil {
return fmt.Errorf("failed to encode smart reply: %w", err)
}
}
// V1 prefixes the advertisement with an explicit version packet (V0 emits
// none). AdvRefs.Encode writes it from ar.Version, so set the field rather
// than writing the line by hand — a single source for the encoded version.
// A v2 request with no v2 service (e.g. receive-pack) falls back to a v0
// advertisement, so only V1 sets the field here; V2 stays at the V0 default.
if version == protocol.V1 {
ar.Version = protocol.V1
}
return ar.Encode(w)
}
// AdvertiseCapabilities implements the Protocol v2 capability advertisement for
// the upload-pack service. Unlike the v0/v1 [AdvertiseRefs], it does not list
// references (clients retrieve them with the ls-refs command) and it does not
// emit the smart-HTTP "# service=..." prefix: git omits that line for v2
// (http-backend.c get_info_refs), the response starts directly with the version
// packet.
func AdvertiseCapabilities(_ context.Context, st storage.Storer, w io.Writer, service string) error {
if service != UploadPackService {
return fmt.Errorf("%w: %s", ErrUnsupportedService, service)
}
adv := &packp.CapabilityAdv{
Version: protocol.V2,
Capabilities: serverV2Capabilities(st),
}
return adv.Encode(w)
}
// serverV2Capabilities builds the v2 capabilities this server implements. Only
// commands and features that are actually handled are advertised: advertising a
// feature that isn't handled makes clients request it and then mis-handle the
// reply.
//
// The fetch "shallow" feature covers the whole deepen family (deepen <n>,
// deepen-since, deepen-not and deepen-relative), all of which are handled, so
// it is advertised as the single token upstream uses.
//
// TODO: advertise these once implemented:
// - ls-refs=unborn report an unborn HEAD on an empty repository
// - fetch=filter partial-clone object filters
// - fetch=ref-in-want want-ref negotiation
// - fetch=sideband-all sideband for the entire response, not just the packfile
// - fetch=packfile-uris offload pack data to out-of-band URIs
// - fetch=wait-for-done negotiate-only fetch (git fetch --negotiate-only)
// - server-option process client "server-option" lines
// - object-info object size/type queries without a fetch
func serverV2Capabilities(st storage.Storer) capability.List {
var caps capability.List
caps.Set(capability.Agent, capability.DefaultAgent())
caps.Set(capability.LsRefs)
caps.Set(capability.FetchCmd, "shallow")
caps.Set(capability.ObjectFormat, objectFormat(st).String())
return caps
}
// objectFormat returns the repository's configured object format, defaulting to
// the package default when the config is missing or unset.
func objectFormat(st storage.Storer) config.ObjectFormat {
cfg, err := st.Config()
if err != nil || cfg == nil {
return config.DefaultObjectFormat
}
if cfg.Extensions.ObjectFormat == config.UnsetObjectFormat {
return config.DefaultObjectFormat
}
return cfg.Extensions.ObjectFormat
}
// advertisable reports whether a reference name may be put on the wire.
//
// The reference store reports what is on disk, malformed names included,
// because a caller that cannot see a name cannot repair it and because
// anything that prunes or repacks from that listing has to see every name that
// is really there. The advertisement excludes malformed wire names. A valid
// Git name is retained even if the filesystem storer applies stricter path
// safety rules, such as rejecting an HFS+ component that folds to a dot.
//
// Git's upload-pack.c send_ref does not consult REF_BAD_NAME: malformed names
// that reach it can be advertised with a zero object id, leaving the dropping
// to fetch-pack.c's filter_refs. The files backend excludes some entries,
// including loose .lock files, before send_ref. The ref-filter.c pass that
// warns and skips is the porcelain layer behind for-each-ref and git branch,
// not upload-pack's path.
//
// Dropping rather than zeroing is the choice here, because a zero id is a
// thing every client has to be taught to read, while a name a peer cannot
// store is one it has no use for. The cost is that git ls-remote against a
// go-git server does not show such a name at all, where Git may show it with
// a zero object id.
//
// HEAD is the one name outside refs/ that belongs on the wire, and Validate
// carves it out already.
//
// https://github.com/git/git/blob/1630431f326e15fcde608827b5ff38422528eb59/upload-pack.c#L1196-L1242
// https://github.com/git/git/blob/1630431f326e15fcde608827b5ff38422528eb59/refs/files-backend.c#L345-L350
func advertisable(name plumbing.ReferenceName) bool {
if name == plumbing.HEAD {
return true
}
return name.IsUnderRefs() && name.Validate() == nil
}
func addReferences(st storage.Storer, ar *packp.AdvRefs, addHead bool) error {
iter, err := st.IterReferences()
if err != nil {
return err
}
// Add references and their peeled values
return iter.ForEach(func(r *plumbing.Reference) error {
hash, name := r.Hash(), r.Name()
var target plumbing.ReferenceName
if !advertisable(name) {
// One malformed name must not prevent advertising the usable refs.
trace.General.Printf("ignoring ref with broken name %q", string(name))
return nil
}
if r.Type() == plumbing.SymbolicReference {
ref, err := storer.ResolveReference(st, r.Target())
// Missing, rejected and cyclic referents cost this one entry.
// Other errors still propagate: an unavailable store must not
// turn into a successful but incomplete advertisement.
if reference.IsUnresolvableForAdvertisement(err) {
trace.General.Printf("ignoring ref %q with unresolvable target %q",
string(name), r.Target().String())
return nil
}
if err != nil {
return err
}
hash = ref.Hash()
// Git advertises the terminal name, including for symbolic chains.
// See https://github.com/git/git/blob/1630431f326e15fcde608827b5ff38422528eb59/upload-pack.c#L1245-L1259.
target = ref.Name()
}
if name == plumbing.HEAD {
if !addHead {
return nil
}
// Only advertise a symref when HEAD is symbolic. A detached HEAD
// (HashReference) has no branch target to advertise; emitting
// "HEAD:" with an empty target corrupts the capability list and
// causes the client to store an unresolvable HEAD symref.
//
// The target is checked too. It is a name leaving the process
// like any other, and naming a ref that was just withheld would
// produce an advertisement contradicting itself: the client is
// told HEAD points somewhere it will never be told about, and
// go-git's own client fails such a clone outright. HEAD keeps its
// object id, so the peer still has a starting point.
if r.Type() == plumbing.SymbolicReference && advertisable(target) {
ar.Capabilities.Add(capability.SymRef, fmt.Sprintf("%s:%s", name, target))
}
ar.References = append([]*plumbing.Reference{plumbing.NewHashReference(name, hash)}, ar.References...)
return nil
}
ar.References = append(ar.References, plumbing.NewHashReference(name, hash))
if r.Name().IsTag() {
if tag, err := object.GetTag(st, hash); err == nil {
ar.References = append(ar.References, plumbing.NewHashReference(
plumbing.ReferenceName(name.String()+"^{}"), tag.Target,
))
}
}
return nil
})
}
package transport
import (
"fmt"
"github.com/go-git/go-billy/v6"
"github.com/go-git/go-git/v6/internal/repository"
"github.com/go-git/go-git/v6/plumbing/storer"
"github.com/go-git/go-git/v6/storage"
)
// UpdateServerInfo updates the server info files in the repository.
//
// It generates a list of available refs for the repository.
// Used by git http transport (dumb), for more information refer to:
// https://git-scm.com/book/id/v2/Git-Internals-Transfer-Protocols#_the_dumb_protocol
func UpdateServerInfo(s storage.Storer, fs billy.Filesystem) error {
pos, ok := s.(storer.PackedObjectStorer)
if !ok {
return ErrPackedObjectsNotSupported
}
infoRefs, err := fs.Create("info/refs")
if err != nil {
return err
}
defer func() { _ = infoRefs.Close() }()
refsIter, err := s.IterReferences()
if err != nil {
return err
}
defer refsIter.Close()
if err := repository.WriteInfoRefs(infoRefs, s); err != nil {
return fmt.Errorf("failed to write info/refs: %w", err)
}
infoPacks, err := fs.Create("objects/info/packs")
if err != nil {
return err
}
defer func() { _ = infoPacks.Close() }()
if err := repository.WriteObjectsInfoPacks(infoPacks, pos); err != nil {
return fmt.Errorf("failed to write objects/info/packs: %w", err)
}
return nil
}
package transport
import "strings"
// Git service command names.
const (
UploadPackService = "git-upload-pack"
UploadArchiveService = "git-upload-archive"
ReceivePackService = "git-receive-pack"
)
// ServiceName returns the service name without the "git-" prefix.
func ServiceName(service string) string {
return strings.TrimPrefix(service, "git-")
}
package transport
import (
"context"
"fmt"
"io"
"strings"
"github.com/go-git/go-git/v6/internal/archive"
"github.com/go-git/go-git/v6/plumbing/format/pktline"
"github.com/go-git/go-git/v6/plumbing/protocol/packp/sideband"
"github.com/go-git/go-git/v6/storage"
"github.com/go-git/go-git/v6/utils/ioutil"
)
// UploadArchiveRequest configures the server-side upload-archive service.
type UploadArchiveRequest struct{}
// UploadArchive is a server command that serves the git-upload-archive service.
//
// It reads argument pkt-lines from r, sends ACK + flush, then generates the
// archive and streams it to w using sideband multiplexing.
//
// Wire protocol:
//
// Client → Server: "argument <arg>\n" pkt-lines + flush
// Server → Client: "ACK\n" pkt-line + flush
// Server → Client: sideband packets (band 1 = archive data, band 2 = progress)
func UploadArchive(
ctx context.Context,
st storage.Storer,
r io.ReadCloser,
w io.WriteCloser,
_ *UploadArchiveRequest,
) error {
w = ioutil.NewContextWriteCloser(ctx, w)
// Unreachable Git objects (i.e. objects not referenced by any refs or unreachable
// from the commit graph) are intentionally not supported due to security concerns.
//
// Support for handling such objects may be reconsidered in the future if a safe
// and performant approach is established.
allowUnreachable := false
args, err := readArchiveArgs(r)
if err != nil {
writeNACK(w, err.Error())
return err
}
if _, err := pktline.WriteString(w, "ACK\n"); err != nil {
return fmt.Errorf("upload-archive: writing ACK: %w", err)
}
if err := pktline.WriteFlush(w); err != nil {
return fmt.Errorf("upload-archive: writing flush: %w", err)
}
mux := sideband.NewMuxer(sideband.Sideband64k, w)
format := "tar"
prefix := ""
var treeish string
var paths []string
list := false
// Normalize arguments: convert "--format zip" to "--format=zip" etc.
normalized := make([]string, 0, len(args))
for i := 0; i < len(args); i++ {
arg := args[i]
switch arg {
case "--format", "--prefix":
i++
if i >= len(args) {
return muxError(mux, w, fmt.Errorf("%s requires an argument", arg))
}
normalized = append(normalized, arg+"="+args[i])
default:
normalized = append(normalized, arg)
}
}
for _, arg := range normalized {
switch {
case arg == "--list" || arg == "-l":
list = true
case strings.HasPrefix(arg, "--format="):
format = arg[len("--format="):]
case strings.HasPrefix(arg, "--prefix="):
prefix = arg[len("--prefix="):]
case arg == "--":
// paths are handled below
default:
if !strings.HasPrefix(arg, "-") {
if treeish == "" {
treeish = arg
}
} else {
if feature := unsupportedArchiveFeature(arg); feature != "" {
return muxError(mux, w, fmt.Errorf("unsupported feature: %s", feature))
}
return muxError(mux, w, fmt.Errorf("unknown option: %s", arg))
}
}
}
// Extract paths after treeish
for i, arg := range normalized {
if arg == treeish {
if i+1 < len(normalized) {
if normalized[i+1] == "--" {
paths = normalized[i+2:]
} else {
paths = normalized[i+1:]
}
}
break
}
}
if list {
// List-only mode: just write the list of supported formats.
for _, f := range archive.SupportedFormats() {
if _, err := fmt.Fprintf(mux, "%s\n", f); err != nil {
return err
}
}
if err := pktline.WriteFlush(w); err != nil {
return fmt.Errorf("upload-archive: writing flush: %w", err)
}
return nil
}
if treeish == "" {
return muxError(mux, w, fmt.Errorf("no tree-ish specified"))
}
tree, commitHash, commitTime, err := archive.ResolveTreeish(st, treeish, allowUnreachable)
if err != nil {
return muxError(mux, w, err)
}
if err = archive.WriteArchive(st, mux, tree, commitHash, commitTime, format, prefix, paths); err != nil {
return muxError(mux, w, err)
}
return pktline.WriteFlush(w)
}
const maxArchiveArgs = 64
// readArchiveArgs reads "argument <arg>\n" pkt-lines until flush.
func readArchiveArgs(r io.Reader) ([]string, error) {
var args []string
sc := pktline.NewScanner(r)
flushed := false
for sc.Scan() {
if sc.Len() == pktline.Flush {
flushed = true
break
}
if len(args) >= maxArchiveArgs {
return nil, fmt.Errorf("upload-archive: too many arguments (>%d)", maxArchiveArgs)
}
line := strings.TrimSuffix(sc.Text(), "\n")
if !strings.HasPrefix(line, "argument ") {
return nil, fmt.Errorf("upload-archive: expected 'argument' token, got: %s", line)
}
args = append(args, line[len("argument "):])
}
if err := sc.Err(); err != nil {
return nil, fmt.Errorf("upload-archive: reading argument: %w", err)
}
if !flushed {
return nil, fmt.Errorf("upload-archive: reading argument: %w", io.ErrUnexpectedEOF)
}
return args, nil
}
func writeNACK(w io.Writer, reason string) {
_, _ = pktline.WriteString(w, fmt.Sprintf("NACK %s\n", reason))
_ = pktline.WriteFlush(w)
}
// muxError writes an error to the sideband error channel and flushes.
// Returns the original error for convenience.
func muxError(mux *sideband.Muxer, w io.Writer, err error) error {
errMsg := fmt.Sprintf("upload-archive: %s", err.Error())
_, _ = mux.WriteChannel(sideband.ErrorMessage, []byte(errMsg))
_ = pktline.WriteFlush(w)
return err
}
func unsupportedArchiveFeature(arg string) string {
switch {
case arg == "--worktree-attributes":
return "export-ignore / export-subst"
case arg == "--add-file" || strings.HasPrefix(arg, "--add-file="):
return "--add-file"
case arg == "--add-virtual-file" || strings.HasPrefix(arg, "--add-virtual-file="):
return "--add-virtual-file"
case arg == "--mtime" || strings.HasPrefix(arg, "--mtime="):
return "--mtime"
case len(arg) == 2 && arg[0] == '-' && arg[1] >= '0' && arg[1] <= '9':
return "archive backend compression options"
}
return ""
}
package transport
import (
"bufio"
"context"
"errors"
"fmt"
"io"
"math"
"strings"
"time"
"github.com/go-git/go-git/v6/config"
"github.com/go-git/go-git/v6/internal/reference"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/format/packfile"
"github.com/go-git/go-git/v6/plumbing/format/pktline"
"github.com/go-git/go-git/v6/plumbing/object"
"github.com/go-git/go-git/v6/plumbing/protocol"
"github.com/go-git/go-git/v6/plumbing/protocol/capability"
"github.com/go-git/go-git/v6/plumbing/protocol/packp"
"github.com/go-git/go-git/v6/plumbing/protocol/packp/sideband"
"github.com/go-git/go-git/v6/plumbing/revlist"
"github.com/go-git/go-git/v6/plumbing/storer"
"github.com/go-git/go-git/v6/storage"
"github.com/go-git/go-git/v6/utils/ioutil"
"github.com/go-git/go-git/v6/utils/trace"
)
// UploadPackRequest is a set of options for the UploadPack service.
type UploadPackRequest struct {
GitProtocol string
AdvertiseRefs bool
StatelessRPC bool
// SkipDeltaCompression disables delta compression when encoding the
// packfile. When false, the repository pack.window configuration is used.
//
// Disabling delta compression significantly improves performance for local
// transfers where recomputing deltas is unnecessary.
SkipDeltaCompression bool
}
// UploadPack is a server command that serves the upload-pack service.
func UploadPack(
ctx context.Context,
st storage.Storer,
r io.ReadCloser,
w io.WriteCloser,
opts *UploadPackRequest,
) error {
if w == nil {
return fmt.Errorf("nil writer")
}
w = ioutil.NewContextWriteCloser(ctx, w)
if opts == nil {
opts = &UploadPackRequest{}
}
if opts.AdvertiseRefs || !opts.StatelessRPC {
v := ProtocolVersion(opts.GitProtocol)
switch v {
case protocol.V0, protocol.V1, protocol.V2:
// V0/V1 share the classic advertisement; V2 advertises
// capabilities only (refs come via ls-refs).
default:
return fmt.Errorf("%w: %q", ErrUnsupportedVersion, v)
}
if v == protocol.V2 {
if err := AdvertiseCapabilities(ctx, st, w, UploadPackService); err != nil {
return fmt.Errorf("advertising v2 capabilities: %w", err)
}
} else if err := AdvertiseRefs(ctx, st, w, UploadPackService, opts.StatelessRPC, v); err != nil {
return fmt.Errorf("advertising references: %w", err)
}
}
if opts.AdvertiseRefs {
// Done, there's nothing else to do
return nil
}
if r == nil {
return fmt.Errorf("nil reader")
}
r = ioutil.NewContextReadCloser(ctx, r)
rd := bufio.NewReader(r)
v := ProtocolVersion(opts.GitProtocol)
if v == protocol.V2 {
return serveUploadPackV2(ctx, st, rd, w, opts)
}
l, _, err := pktline.PeekLine(rd)
if err != nil {
return fmt.Errorf("peeking line: %w", err)
}
// In case the client has nothing to send, it sends a flush packet to
// indicate that it is done sending data. In that case, we're done
// here.
if l == pktline.Flush {
return nil
}
var done bool
var haves []plumbing.Hash
var upreq *packp.UploadRequest
var havesWithRef map[plumbing.Hash][]plumbing.Hash
var multiAck, multiAckDetailed bool
var caps capability.List
var wants []plumbing.Hash
var ack packp.ACK
firstRound := true
for !done {
writec := make(chan error)
if firstRound || opts.StatelessRPC {
upreq = &packp.UploadRequest{}
if err := upreq.Decode(rd); err != nil {
return fmt.Errorf("decoding upload-request: %w", err)
}
wants = upreq.Wants
caps = upreq.Capabilities
if err := r.Close(); err != nil {
return fmt.Errorf("closing reader: %w", err)
}
// Find common commits/objects
havesWithRef, err = revlist.ObjectsWithRef(st, wants, nil)
if err != nil {
return fmt.Errorf("getting objects with ref: %w", err)
}
// Encode objects to packfile and write to client
multiAck = caps.Supports(capability.MultiACK)
multiAckDetailed = caps.Supports(capability.MultiACKDetailed)
go func() {
// TODO: support deepen-since, and deepen-not
var shupd packp.ShallowUpdate
if !upreq.Depth.IsZero() {
if upreq.Depth.Deepen > 0 {
if err := getShallowCommits(st, wants, upreq.Depth.Deepen, &shupd); err != nil {
writec <- fmt.Errorf("getting shallow commits: %w", err)
return
}
} else {
writec <- fmt.Errorf("unsupported depth: %+v", upreq.Depth)
return
}
if err := shupd.Encode(w); err != nil {
writec <- fmt.Errorf("sending shallow-update: %w", err)
return
}
}
writec <- nil
}()
if err := <-writec; err != nil {
return err
}
}
var uphav packp.UploadHaves
if err := uphav.Decode(rd); err != nil {
return fmt.Errorf("decoding upload-haves: %w", err)
}
if err := r.Close(); err != nil {
return fmt.Errorf("closing reader: %w", err)
}
haves = append(haves, uphav.Haves...)
done = uphav.Done
var acks []packp.ACK
for _, hu := range uphav.Haves {
_, ok := havesWithRef[hu]
var status packp.ACKStatus
if multiAckDetailed {
status = packp.ACKCommon
if !ok {
status = packp.ACKReady
}
} else if multiAck {
status = packp.ACKContinue
}
if ok || multiAck || multiAckDetailed {
ack = packp.ACK{Hash: hu, Status: status}
acks = append(acks, ack)
if !multiAck && !multiAckDetailed {
break
}
}
}
go func() {
defer close(writec)
if len(haves) > 0 {
// Encode ACKs to client when we have haves
srvrsp := packp.ServerResponse{ACKs: acks}
if err := srvrsp.Encode(w); err != nil {
writec <- fmt.Errorf("sending acks server-response: %w", err)
return
}
}
switch {
case !done:
if multiAck || multiAckDetailed {
// Encode a NAK for multi-ack
srvrsp := packp.ServerResponse{}
if err := srvrsp.Encode(w); err != nil {
writec <- fmt.Errorf("sending nak server-response: %w", err)
return
}
}
case !ack.Hash.IsZero() && (multiAck || multiAckDetailed):
// We're done, send the final ACK
ack.Status = 0
srvrsp := packp.ServerResponse{ACKs: []packp.ACK{ack}}
if err := srvrsp.Encode(w); err != nil {
writec <- fmt.Errorf("sending final ack server-response: %w", err)
return
}
case ack.Hash.IsZero() && len(haves) == 0:
// No haves were sent. Emit the single terminal NAK.
//
// When haves *were* sent, the ServerResponse{ACKs: acks}
// write above already emitted a NAK (encodeServerResponse
// writes NAK when ACKs is empty). Emitting another one here
// would produce two consecutive "0008NAK\n" pktlines;
// ServerResponse.Decode consumes only the first, and the
// second would then be misread by the sideband demuxer as
// a frame with channel byte 'N' ("unknown channel NAK").
srvrsp := packp.ServerResponse{}
if err := srvrsp.Encode(w); err != nil {
writec <- fmt.Errorf("sending final nak server-response: %w", err)
return
}
}
writec <- nil
}()
if err := <-writec; err != nil {
return err
}
firstRound = false
}
// Done with the request, now close the reader
// to indicate that we are done reading from it.
if err := r.Close(); err != nil {
return fmt.Errorf("closing reader: %w", err)
}
objs, err := objectsToUpload(st, wants, haves)
if err != nil {
_ = w.Close()
return fmt.Errorf("getting objects to upload: %w", err)
}
var (
useSideband bool
writer io.Writer = w
)
if caps.Supports(capability.Sideband64k) {
writer = sideband.NewMuxer(sideband.Sideband64k, w)
useSideband = true
} else if caps.Supports(capability.Sideband) {
writer = sideband.NewMuxer(sideband.Sideband, w)
useSideband = true
}
// TODO: Support shallow-file
// TODO: Support thin-pack
var packWindow uint
if opts.SkipDeltaCompression {
packWindow = 0
} else if cfg, cerr := st.Config(); cerr == nil && cfg != nil {
packWindow = cfg.Pack.Window
} else {
packWindow = config.DefaultPackWindow
}
e := packfile.NewEncoder(writer, st, false)
_, err = e.Encode(objs, packWindow)
if err != nil {
return fmt.Errorf("encoding packfile: %w", err)
}
if useSideband {
if err := pktline.WriteFlush(w); err != nil {
return fmt.Errorf("flushing sideband: %w", err)
}
}
if err := w.Close(); err != nil {
return fmt.Errorf("closing writer: %w", err)
}
return nil
}
func objectsToUpload(st storage.Storer, wants, haves []plumbing.Hash) ([]plumbing.Hash, error) {
return revlist.Objects(st, wants, haves)
}
func getShallowCommits(st storage.Storer, heads []plumbing.Hash, depth int, upd *packp.ShallowUpdate) error {
var i, curDepth int
var commit *object.Commit
depths := map[*object.Commit]int{}
stack := []object.Object{}
for commit != nil || i < len(heads) || len(stack) > 0 {
if commit == nil {
if i < len(heads) {
obj, err := st.EncodedObject(plumbing.CommitObject, heads[i])
i++
if err != nil {
continue
}
commit, err = object.DecodeCommit(st, obj)
if err != nil {
commit = nil
continue
}
depths[commit] = 0
curDepth = 0
} else if len(stack) > 0 {
commit = stack[len(stack)-1].(*object.Commit)
stack = stack[:len(stack)-1]
curDepth = depths[commit]
}
}
curDepth++
if depth != math.MaxInt && curDepth >= depth {
upd.Shallows = append(upd.Shallows, commit.Hash)
commit = nil
continue
}
upd.Unshallows = append(upd.Unshallows, commit.Hash)
parents := commit.Parents()
commit = nil
for {
parent, err := parents.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
if depths[parent] != 0 && curDepth >= depths[parent] {
continue
}
depths[parent] = curDepth
if _, err := parents.Next(); err == nil {
stack = append(stack, parent)
} else {
commit = parent
curDepth = depths[commit]
}
}
}
return nil
}
// shallowFrontierDepth returns the depth, counted from the wants (a tip is at
// depth 1), of the closest commit in the client's shallow set, or 0 if none is
// reachable. It mirrors upstream get_shallows_depth (shallow.c): the value
// offsets a deepen-relative request so the new depth is measured from the
// client's existing shallow boundary rather than from the tips.
func shallowFrontierDepth(st storage.Storer, heads, shallows []plumbing.Hash) (int, error) {
shallowSet := make(map[plumbing.Hash]struct{}, len(shallows))
for _, h := range shallows {
shallowSet[h] = struct{}{}
}
best := 0
seen := map[plumbing.Hash]int{}
type frame struct {
hash plumbing.Hash
depth int // depth of this commit's predecessor; the commit sits at depth+1
}
var stack []frame
for _, h := range heads {
if c, ok := peelToCommit(st, h); ok {
stack = append(stack, frame{c.Hash, 0})
}
}
for len(stack) > 0 {
f := stack[len(stack)-1]
stack = stack[:len(stack)-1]
if d, ok := seen[f.hash]; ok && d <= f.depth {
continue
}
seen[f.hash] = f.depth
cur := f.depth + 1
if _, ok := shallowSet[f.hash]; ok {
if best == 0 || cur < best {
best = cur
}
// A client shallow commit is a normal commit on the server, so the
// walk continues past it, matching upstream get_shallows_or_depth.
}
c, err := object.GetCommit(st, f.hash)
if err != nil {
continue
}
for _, p := range c.ParentHashes {
stack = append(stack, frame{p, cur})
}
}
return best, nil
}
// serveUploadPackV2 handles the git protocol v2 for upload-pack (fetch/ls-refs).
// It is used when the client requests version=2 via GIT_PROTOCOL.
func serveUploadPackV2(ctx context.Context, st storage.Storer, rd *bufio.Reader, w io.WriteCloser, opts *UploadPackRequest) error {
for {
// Peek the command line to choose the argument decoder, then decode the
// whole request envelope through packp.CommandRequest (the same type the
// client encodes).
l, line, err := pktline.PeekLine(rd)
if err != nil {
if errors.Is(err, io.EOF) {
return nil
}
return err
}
if l == pktline.Flush {
// A lone flush-pkt ends the request.
_, _, _ = pktline.ReadLine(rd)
return nil
}
cmd := strings.TrimPrefix(strings.TrimSuffix(string(line), "\n"), "command=")
req := &packp.CommandRequest{}
switch cmd {
case "ls-refs":
req.Args = &packp.LsRefsArgs{}
case "fetch":
req.Args = &packp.FetchArgs{}
default:
_, _ = pktline.Writef(w, "error unknown-command %s\n", cmd)
_ = pktline.WriteFlush(w)
return fmt.Errorf("unsupported v2 command %q", cmd)
}
if err := req.Decode(rd); err != nil {
return fmt.Errorf("decoding %s request: %w", cmd, err)
}
switch cmd {
case "ls-refs":
if err := serveLsRefsV2(ctx, st, w, req.Args.(*packp.LsRefsArgs)); err != nil {
return err
}
// Stateless (HTTP) carries a single command per request; stateful
// transports may continue, but clients typically close after.
if opts.StatelessRPC {
return nil
}
case "fetch":
concluded, err := serveFetchV2(ctx, st, w, req.Args.(*packp.FetchArgs), opts)
if err != nil {
return err
}
if concluded {
return nil
}
// Stateful transport: the round was acknowledgments-only and the
// negotiation continues. Loop to read the client's next command.
}
}
}
// serveLsRefsV2 responds to a ls-refs command using the decoded arguments.
//
// The reference lines are encoded by writeV2Ref rather than packp.LsRefsOutput:
// a v2 HEAD line carries both a resolved object id and a symref-target
// attribute, which a single plumbing.Reference (hash XOR symbolic) cannot
// represent. writeV2Ref resolves the symref's hash from the storer, matching
// upstream git's send_ref.
func serveLsRefsV2(_ context.Context, st storage.Storer, w io.Writer, args *packp.LsRefsArgs) error {
iter, err := st.IterReferences()
if err != nil {
return err
}
defer iter.Close()
var refs []*plumbing.Reference
if err := iter.ForEach(func(r *plumbing.Reference) error {
// Use the same name gate as the v0/v1 advertisement. In the v2
// grammar a space in a name also introduces a ref-attribute.
if !advertisable(r.Name()) {
trace.General.Printf("ignoring ref with broken name %q", r.Name().String())
return nil
}
refs = append(refs, r)
return nil
}); err != nil {
return err
}
prefixes := args.RefPrefixes
// HEAD is emitted first, but only when it passes the ref-prefix filter,
// matching upstream's send_possibly_unborn_head -> send_ref (ls-refs.c),
// where HEAD is subject to ref_match like every other ref.
for _, r := range refs {
if r.Name() == plumbing.HEAD {
if len(prefixes) == 0 || refMatchesAnyPrefix(r.Name().String(), prefixes) {
if err := writeV2Ref(w, st, r, args.Symrefs, args.Peel); err != nil {
return err
}
}
break
}
}
for _, r := range refs {
if r.Name() == plumbing.HEAD {
continue
}
if len(prefixes) > 0 && !refMatchesAnyPrefix(r.Name().String(), prefixes) {
continue
}
if err := writeV2Ref(w, st, r, args.Symrefs, args.Peel); err != nil {
return err
}
}
return pktline.WriteFlush(w)
}
func refMatchesAnyPrefix(name string, prefixes []string) bool {
for _, p := range prefixes {
if strings.HasPrefix(name, p) {
return true
}
}
return false
}
// writeV2Ref writes an ls-refs response with the requested reference attributes.
// See https://github.com/git/git/blob/1630431f326e15fcde608827b5ff38422528eb59/ls-refs.c#L91-L117.
func writeV2Ref(w io.Writer, st storage.Storer, r *plumbing.Reference, symrefs, peel bool) error {
var hash plumbing.Hash
var target string
if r.Type() == plumbing.SymbolicReference {
ref, err := storer.ResolveReference(st, r.Target())
if reference.IsUnresolvableForAdvertisement(err) {
return nil
}
if err != nil {
return err
}
hash = ref.Hash()
target = ref.Name().String()
} else {
hash = r.Hash()
}
if hash.IsZero() {
return nil
}
// Protocol v2 ls-refs grammar:
// ref = obj-id SP refname *(SP ref-attribute) LF
// ref-attribute = (symref | peeled)
// Both symref-target and peeled are attributes on the ref's own line
// (symref-target first, matching upstream's send_ref ordering), not
// separate lines as in the v0/v1 advertisement format.
line := fmt.Sprintf("%s %s", hash, r.Name())
if symrefs && target != "" && advertisable(plumbing.ReferenceName(target)) {
line += " symref-target:" + target
}
if peel {
// Peel any ref whose object is (a chain of) annotated tags, not just
// refs/tags/*, and resolve all the way to the underlying non-tag object
// — matching upstream's reference_get_peeled_oid (ls-refs.c). Lightweight
// tags and branches don't point at tag objects, so they emit no attribute.
if peeled, ok := peelToNonTag(st, hash); ok {
line += " peeled:" + peeled.String()
}
}
if _, err := pktline.Writef(w, "%s\n", line); err != nil {
return err
}
return nil
}
// peelToNonTag follows annotated-tag objects from h down to the first non-tag
// object, mirroring upstream's reference_get_peeled_oid. It returns the peeled
// hash and true when h points at one or more tag objects; false when h is not a
// tag (a lightweight tag, branch, etc.) so no "peeled" attribute is emitted.
func peelToNonTag(st storage.Storer, h plumbing.Hash) (plumbing.Hash, bool) {
tag, err := object.GetTag(st, h)
if err != nil {
return plumbing.ZeroHash, false
}
for {
next := tag.Target
inner, err := object.GetTag(st, next)
if err != nil {
// next is a non-tag object (or missing); return it as the peeled
// value, as upstream's peel does.
return next, true
}
tag = inner
}
}
// serveFetchV2 handles command=fetch for v2 using the decoded arguments. The
// acknowledgments, shallow-info, and packfile-header sections are emitted
// through packp.FetchOutput; this function streams the packfile data after the
// header, matching the caller-owned streaming on the client side.
//
// It reports whether the fetch concluded. A packfile (or a terminal no-op)
// returns concluded=true and the connection is closed. An acknowledgments-only
// round on a stateful transport returns concluded=false with the connection
// left open, so the caller loops to read the client's next command=fetch round
// (the stateful negotiation continues until the server is ready). A stateless
// (HTTP) round always concludes, since the client re-POSTs each round.
func serveFetchV2(_ context.Context, st storage.Storer, w io.WriteCloser, args *packp.FetchArgs, opts *UploadPackRequest) (concluded bool, err error) {
wants := args.Wants
haves := args.Haves
clientShallows := args.Shallows
depth := args.Deepen
done := args.Done
// No 'want' lines: the client guessed it didn't want anything. Upstream
// emits no response at all here (upload-pack.c, UPLOAD_DONE), so write
// nothing and just close the stream, no stray flush packet.
if len(wants) == 0 {
return true, w.Close()
}
out := &packp.FetchOutput{}
// Negotiation (acknowledgments section), per gitprotocol-v2 "fetch":
//
// - done -> no acknowledgments section; packfile follows.
// - no haves -> clone-like; no acknowledgments section; packfile follows.
// - haves and !done -> emit an acknowledgments section. ACK every common
// object. "ready" is sent only once every want is
// reachable from the common haves (upstream's
// ok_to_give_up); then the packfile follows in the
// same response. Otherwise the section ends without a
// packfile and the client negotiates again with more
// haves (NAK when there is no common object at all).
if !done && len(haves) > 0 {
var common []plumbing.Hash
for _, h := range haves {
if _, err := st.EncodedObject(plumbing.AnyObject, h); err == nil {
common = append(common, h)
}
}
out.Acknowledgments = &packp.Acknowledgments{ACKs: common}
// "ready" is withheld until every want is reachable from the common
// haves (upstream's ok_to_give_up). Declaring it on the first common
// have would force single-round negotiation and a larger pack. When not
// ready (including no common object at all, which encodes as NAK), the
// acknowledgments section stands alone and the client refines its haves
// in the next request.
if len(common) == 0 || !wantsReachableFromHaves(st, wants, common) {
if err := out.Encode(w); err != nil {
return true, err
}
// Stateless (HTTP) carries one round per request: this response is
// complete and the client re-POSTs the next round. A stateful
// transport keeps the connection open so the client can send its
// next command=fetch with refined haves.
if opts.StatelessRPC {
return true, w.Close()
}
return false, nil
}
out.Acknowledgments.Ready = true
}
// shallow-info: a shallow fetch bounds the history sent. The boundary forms
// mirror upstream send_shallow_list (upload-pack.c):
// - deepen <n>: a depth boundary from the wants (getShallowCommits).
// - deepen-since / deepen-not: a date/ref boundary (getShallowCommitsByRevList,
// mirroring deepen_by_rev_list).
// Upstream forbids combining deepen with deepen-since/deepen-not, and so do we.
// deepen-relative only changes how the depth is counted: for a fresh fetch
// (no client shallows) relative and absolute depth coincide, and for an
// already-shallow client the depth is offset by the existing boundary's
// distance from the wants (see the deepen-relative handling below).
since := args.DeepenSince
notTips, err := resolveDeepenNot(st, args.DeepenNot)
if err != nil {
_ = w.Close()
return true, fmt.Errorf("resolving deepen-not: %w", err)
}
revList := !since.IsZero() || len(notTips) > 0
if depth > 0 && revList {
_ = w.Close()
return true, fmt.Errorf("deepen and deepen-since (or deepen-not) cannot be used together")
}
// A deepen was requested when depth > 0 or a rev-list bound was given.
// haveNewBoundary records that separately from len(newBoundary): a deepen
// that reaches full history yields an empty boundary, which still drives
// shallow-info and the unshallow lines and must not be mistaken for "no
// deepen requested". newBoundary is the grafting boundary for the deepened
// view (nil/empty means graft nothing: full history).
var newBoundary []plumbing.Hash
var haveNewBoundary bool
if depth > 0 || revList {
var shupd packp.ShallowUpdate
computed := true
if revList {
err = getShallowCommitsByRevList(st, wants, since, notTips, &shupd)
} else {
effectiveDepth := depth
if args.DeepenRelative && len(clientShallows) > 0 {
// deepen-relative counts depth from the client's existing
// shallow boundary, not from the wants. Mirror upstream
// get_shallow_commits (shallow.c): offset the absolute depth by
// the depth at which that boundary sits from the wants.
cur, derr := shallowFrontierDepth(st, wants, clientShallows)
if derr != nil {
_ = w.Close()
return true, fmt.Errorf("computing shallow frontier depth: %w", derr)
}
if cur == 0 {
// No client shallow is reachable from the wants; upstream
// computes no new boundary and leaves the client's view
// unchanged. Skip the deepen entirely.
computed = false
} else {
effectiveDepth = depth + cur
}
}
if computed {
err = getShallowCommits(st, wants, effectiveDepth, &shupd)
}
}
if err != nil {
_ = w.Close()
return true, fmt.Errorf("computing shallow commits: %w", err)
}
if computed {
haveNewBoundary = true
newBoundary = shupd.Shallows
}
}
var objs []plumbing.Hash
if len(clientShallows) > 0 {
// The client already has a shallow view (it sent "shallow" lines).
// A single object walk cannot graft the wanted history at the new
// boundary while also grafting the client's have-history at its existing
// boundary, so compute two views and send their difference:
// newView = objects reachable from the wants, grafted at the new
// boundary (the client's deepened view).
// clientView = objects the client already has, reachable from its haves
// grafted at its existing shallow boundary.
// newView \ clientView is exactly what the client is missing. It never
// omits a needed object; at worst it re-sends one the client has, which
// is harmless. This is what bounds a deepen of an already-shallow clone.
boundary := clientShallows
if haveNewBoundary {
// The deepened boundary, which may be empty: a deepen that reaches
// full history grafts nothing and unshallows the old boundary.
boundary = newBoundary
}
newView, nerr := objectsToUpload(&shallowBoundaryStorer{Storer: st, boundary: boundary}, wants, nil)
if nerr != nil {
_ = w.Close()
return true, fmt.Errorf("getting objects to upload: %w", nerr)
}
clientView, cerr := objectsToUpload(&shallowBoundaryStorer{Storer: st, boundary: clientShallows}, haves, nil)
if cerr != nil {
_ = w.Close()
return true, fmt.Errorf("getting client objects: %w", cerr)
}
objs = hashDifference(newView, clientView)
if haveNewBoundary {
out.ShallowInfo = &packp.ShallowInfo{
Shallows: newBoundary,
Unshallows: unshallowedCommits(clientShallows, newBoundary, newView),
}
}
} else {
packSt := st
if haveNewBoundary && len(newBoundary) > 0 {
out.ShallowInfo = &packp.ShallowInfo{Shallows: newBoundary}
packSt = &shallowBoundaryStorer{Storer: st, boundary: newBoundary}
}
objs, err = objectsToUpload(packSt, wants, haves)
if err != nil {
_ = w.Close()
return true, fmt.Errorf("getting objects to upload: %w", err)
}
}
// include-tag: add annotated tags whose target is in the pack (auto-tag
// following), mirroring upstream pack-objects --include-tag.
if args.IncludeTag {
objs, err = includeReachableTags(st, objs)
if err != nil {
_ = w.Close()
return true, fmt.Errorf("collecting include-tag objects: %w", err)
}
}
// Emit the metadata sections and the "packfile" section header. The client
// switches to sideband demux after seeing the header, matching reference git.
out.Packfile = true
if err := out.Encode(w); err != nil {
return true, err
}
// The packfile is muxed on sideband-64k band 1. This server never writes the
// progress band (band 2), so the client's no-progress request (args.NoProgress)
// is honored by construction; there is nothing to suppress.
writer := sideband.NewMuxer(sideband.Sideband64k, w)
var packWindow uint
if opts.SkipDeltaCompression {
packWindow = 0
} else if cfg, cerr := st.Config(); cerr == nil && cfg != nil {
packWindow = cfg.Pack.Window
} else {
packWindow = config.DefaultPackWindow
}
e := packfile.NewEncoder(writer, st, false)
if _, err := e.Encode(objs, packWindow); err != nil {
return true, fmt.Errorf("encoding packfile: %w", err)
}
// Terminate the sideband stream and the v2 fetch response.
if err := pktline.WriteFlush(w); err != nil {
return true, err
}
return true, w.Close()
}
// hashDifference returns the elements of a that are not in b, preserving a's
// order. It computes the objects a deepened client is missing (newView minus the
// client's existing view).
func hashDifference(a, b []plumbing.Hash) []plumbing.Hash {
set := make(map[plumbing.Hash]struct{}, len(b))
for _, h := range b {
set[h] = struct{}{}
}
var out []plumbing.Hash
for _, h := range a {
if _, ok := set[h]; !ok {
out = append(out, h)
}
}
return out
}
// unshallowedCommits returns the client's shallow commits that the deepened view
// now includes as interior commits (their parents are being sent), so the client
// can clear their shallow mark. Commits still on the new boundary stay shallow.
// Mirrors upstream send_unshallow (upload-pack.c).
func unshallowedCommits(clientShallows, newBoundary, newView []plumbing.Hash) []plumbing.Hash {
inView := make(map[plumbing.Hash]struct{}, len(newView))
for _, h := range newView {
inView[h] = struct{}{}
}
boundary := make(map[plumbing.Hash]struct{}, len(newBoundary))
for _, h := range newBoundary {
boundary[h] = struct{}{}
}
var out []plumbing.Hash
for _, cs := range clientShallows {
if _, ok := inView[cs]; !ok {
continue // not part of the deepened view
}
if _, ok := boundary[cs]; ok {
continue // still a boundary commit
}
out = append(out, cs)
}
return out
}
// resolveDeepenNot resolves each deepen-not argument (a ref name or an object
// id) to a commit hash, peeling annotated tags, mirroring how upstream feeds
// "--not <oid>" to rev-list (upload-pack.c send_shallow_list).
func resolveDeepenNot(st storage.Storer, refs []string) ([]plumbing.Hash, error) {
if len(refs) == 0 {
return nil, nil
}
out := make([]plumbing.Hash, 0, len(refs))
for _, r := range refs {
var h plumbing.Hash
if ref, err := storer.ResolveReference(st, plumbing.ReferenceName(r)); err == nil {
h = ref.Hash()
} else if oid, ok := plumbing.FromHex(r); ok {
if _, err := st.EncodedObject(plumbing.AnyObject, oid); err != nil {
return nil, fmt.Errorf("cannot resolve deepen-not %q", r)
}
h = oid
} else {
return nil, fmt.Errorf("cannot resolve deepen-not %q", r)
}
if peeled, ok := peelToNonTag(st, h); ok {
h = peeled
}
out = append(out, h)
}
return out, nil
}
// reachableCommits returns the set of commits reachable from tips (inclusive),
// used as the exclusion set for deepen-not.
func reachableCommits(st storage.Storer, tips []plumbing.Hash) (map[plumbing.Hash]struct{}, error) {
seen := make(map[plumbing.Hash]struct{})
stack := append([]plumbing.Hash(nil), tips...)
for len(stack) > 0 {
h := stack[len(stack)-1]
stack = stack[:len(stack)-1]
if _, ok := seen[h]; ok {
continue
}
seen[h] = struct{}{}
c, err := object.GetCommit(st, h)
if err != nil {
continue
}
stack = append(stack, c.ParentHashes...)
}
return seen, nil
}
// getShallowCommitsByRevList computes the shallow boundary for a deepen-since
// and/or deepen-not request, mirroring upstream's deepen_by_rev_list
// (upload-pack.c). The included set is every commit reachable from heads that is
// not older than since (when set) and not reachable from any notTips (when set);
// a commit in the set with a parent outside it is a shallow boundary.
//
// Unlike git's rev-list traversal it does not apply the date "slop" used to
// tolerate out-of-order committer timestamps, so under clock skew the boundary
// may differ by a few commits; the resulting shallow clone is still valid.
func getShallowCommitsByRevList(st storage.Storer, heads []plumbing.Hash, since time.Time, notTips []plumbing.Hash, upd *packp.ShallowUpdate) error {
exclude, err := reachableCommits(st, notTips)
if err != nil {
return err
}
included := make(map[plumbing.Hash]struct{})
parents := make(map[plumbing.Hash][]plumbing.Hash)
visited := make(map[plumbing.Hash]struct{})
stack := append([]plumbing.Hash(nil), heads...)
for len(stack) > 0 {
h := stack[len(stack)-1]
stack = stack[:len(stack)-1]
if _, ok := visited[h]; ok {
continue
}
visited[h] = struct{}{}
if _, ex := exclude[h]; ex {
continue
}
c, err := object.GetCommit(st, h)
if err != nil {
continue
}
if !since.IsZero() && c.Committer.When.Before(since) {
continue
}
included[h] = struct{}{}
parents[h] = c.ParentHashes
stack = append(stack, c.ParentHashes...)
}
for h := range included {
for _, p := range parents[h] {
if _, ok := included[p]; !ok {
upd.Shallows = append(upd.Shallows, h)
break
}
}
}
plumbing.HashesSort(upd.Shallows)
return nil
}
// includeReachableTags implements the fetch "include-tag" feature: for every
// annotated tag whose (peeled) target is already in objs, it adds the tag
// object and every tag object along the chain, mirroring upstream pack-objects
// --include-tag. Lightweight tags have no tag object and are skipped.
func includeReachableTags(st storage.Storer, objs []plumbing.Hash) ([]plumbing.Hash, error) {
have := make(map[plumbing.Hash]struct{}, len(objs))
for _, h := range objs {
have[h] = struct{}{}
}
iter, err := st.IterReferences()
if err != nil {
return objs, err
}
defer iter.Close()
added := objs
err = iter.ForEach(func(ref *plumbing.Reference) error {
if ref.Type() != plumbing.HashReference || !ref.Name().IsTag() {
return nil
}
var chain []plumbing.Hash
seen := make(map[plumbing.Hash]struct{})
cur := ref.Hash()
for {
if _, ok := have[cur]; ok {
// Reached an object already in the pack: include the tag
// objects that point at it.
for _, t := range chain {
if _, ok := have[t]; !ok {
have[t] = struct{}{}
added = append(added, t)
}
}
break
}
if _, ok := seen[cur]; ok {
break // defend against a tag cycle in a malformed repo
}
seen[cur] = struct{}{}
tag, terr := object.GetTag(st, cur)
if terr != nil {
break // non-tag object not in the pack: nothing to add
}
chain = append(chain, cur)
cur = tag.Target
}
return nil
})
if err != nil {
return objs, err
}
return added, nil
}
// shallowBoundaryStorer reports an additional set of shallow commits (the
// per-request boundary) on top of any the repository already has. revlist's
// object walk stops at shallow commits while still collecting their full trees,
// so wrapping the storer bounds a shallow fetch's packfile to the requested
// depth — the boundary commits ship complete, their ancestors are omitted —
// without the blob loss a plain have-exclusion would cause.
type shallowBoundaryStorer struct {
storage.Storer
boundary []plumbing.Hash
}
func (s *shallowBoundaryStorer) Shallow() ([]plumbing.Hash, error) {
base, err := s.Storer.Shallow()
if err != nil {
return nil, err
}
if len(s.boundary) == 0 {
return base, nil
}
return append(append([]plumbing.Hash(nil), base...), s.boundary...), nil
}
// wantsReachableFromHaves reports whether every want is reachable from the set
// of common haves — upstream's ok_to_give_up (upload-pack.c). A want is anchored
// when a common have is the want itself or one of its ancestors, i.e. the want
// can reach a have by walking parents. Tags are peeled to commits first, as the
// ancestry walk operates on commits. Returns false (keep negotiating) if any
// want cannot be resolved to a commit or is not yet anchored.
func wantsReachableFromHaves(st storage.Storer, wants, commonHaves []plumbing.Hash) bool {
haveSet := make(map[plumbing.Hash]struct{}, len(commonHaves))
haveCommits := make([]*object.Commit, 0, len(commonHaves))
for _, h := range commonHaves {
haveSet[h] = struct{}{}
if c, ok := peelToCommit(st, h); ok {
haveCommits = append(haveCommits, c)
}
}
for _, wHash := range wants {
wc, ok := peelToCommit(st, wHash)
if !ok {
return false
}
if _, ok := haveSet[wc.Hash]; ok {
continue
}
anchored := false
for _, hc := range haveCommits {
if hc.Hash == wc.Hash {
anchored = true
break
}
if isAnc, err := hc.IsAncestor(wc); err == nil && isAnc {
anchored = true
break
}
}
if !anchored {
return false
}
}
return true
}
// peelToCommit resolves h to a commit, following annotated tags. It returns
// false when h is missing or does not peel to a commit.
func peelToCommit(st storage.Storer, h plumbing.Hash) (*object.Commit, bool) {
obj, err := st.EncodedObject(plumbing.AnyObject, h)
if err != nil {
return nil, false
}
switch obj.Type() {
case plumbing.CommitObject:
c, err := object.GetCommit(st, h)
if err != nil {
return nil, false
}
return c, true
case plumbing.TagObject:
tag, err := object.GetTag(st, h)
if err != nil {
return nil, false
}
return peelToCommit(st, tag.Target)
default:
return nil, false
}
}
package transport
import (
"net/url"
giturl "github.com/go-git/go-git/v6/internal/url"
)
// ParseURL parses a remote URL string into a *url.URL. It handles:
// - Standard URLs (https://host/path, ssh://host/path, git://host/path)
// - SCP-like URLs (git@host:path) — normalized to ssh:// scheme
// - Local paths (/path/to/repo, C:\path) — normalized to file:// scheme
func ParseURL(endpoint string) (*url.URL, error) {
if u, ok := giturl.ParseSCP(endpoint); ok {
return u, nil
}
if u, ok := giturl.ParseFile(endpoint); ok {
return u, nil
}
return giturl.ParseURL(endpoint)
}
package transport
import (
"strings"
"github.com/go-git/go-git/v6/plumbing/format/pktline"
"github.com/go-git/go-git/v6/plumbing/protocol"
"github.com/go-git/go-git/v6/utils/ioutil"
)
// DiscoverVersion reads the first pktline from the reader to determine the
// protocol version. This is used by the client to determine the protocol
// version of the server.
//
// Note that the discovered version is not consumed from the reader, so the
// caller can read it again after discovering the version.
func DiscoverVersion(r ioutil.ReadPeeker) (protocol.Version, error) {
ver := protocol.V0
_, pktb, err := pktline.PeekLine(r)
if err != nil {
return ver, err
}
pkt := strings.TrimSpace(string(pktb))
if strings.HasPrefix(pkt, "version ") {
if v, _ := protocol.Parse(pkt[8:]); v > ver {
ver = protocol.Version(v)
}
}
return ver, nil
}
// ProtocolVersion tries to find the version parameter in the protocol string.
// This expects the protocol string from the GIT_PROTOCOL environment variable.
// This is used by the server to determine the protocol version requested by
// the client.
func ProtocolVersion(p string) protocol.Version {
var ver protocol.Version
for param := range strings.SplitSeq(p, ":") {
if strings.HasPrefix(param, "version=") {
if v, _ := protocol.Parse(param[8:]); v > ver {
ver = protocol.Version(v)
}
}
}
return ver
}
package worktree
import (
"bytes"
"errors"
"fmt"
"io"
"io/fs"
"path/filepath"
"regexp"
"strings"
"github.com/go-git/go-billy/v6"
"github.com/go-git/go-billy/v6/util"
"github.com/go-git/go-git/v6"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/cache"
"github.com/go-git/go-git/v6/storage"
"github.com/go-git/go-git/v6/storage/filesystem"
"github.com/go-git/go-git/v6/storage/filesystem/dotgit"
xstorage "github.com/go-git/go-git/v6/x/storage"
)
const (
// names for dir and files managed by worktrees.
dotgitDir = ".git"
worktrees = "worktrees"
commonDir = "commondir"
gitDir = "gitdir"
head = "HEAD"
originalHead = "ORIG_HEAD"
refs = "refs"
dirMode = 0o777
worktreeDotGitMaxSize = 1024
)
var (
worktreeNameRE = regexp.MustCompile(`^[a-zA-Z0-9\-]+$`)
// ErrWorktreeNotFound is returned when attempting to remove a worktree that does not exist.
ErrWorktreeNotFound = errors.New("worktree not found")
// ErrWorktreeAlreadyExists is returned when attempting to add a worktree with a name that already exists.
ErrWorktreeAlreadyExists = errors.New("worktree already exists")
)
// Worktree manages multiple working trees attached to a git repository.
// It provides functionality to add and remove linked worktrees, allowing
// multiple branches to be checked out simultaneously in different directories.
//
// A Worktree instance is tied to a specific repository through its storage
// backend, which must implement the WorktreeStorer interface.
type Worktree struct {
storer xstorage.WorktreeStorer
}
// New creates a new Worktree manager for the given storage backend.
//
// The storer must implement the WorktreeStorer interface, which provides
// access to the repository's filesystem for managing worktree metadata.
//
// Returns an error if storer is nil or does not implement WorktreeStorer.
func New(storer storage.Storer) (*Worktree, error) {
if storer == nil {
return nil, errors.New("storer is nil")
}
wts, ok := storer.(xstorage.WorktreeStorer)
if !ok {
return nil, errors.New("storer does not implement WorktreeStorer")
}
return &Worktree{
storer: wts,
}, nil
}
// Add creates a new linked worktree with the specified name and filesystem.
//
// This method sets up the necessary metadata and directory structure for a new
// worktree, similar to the `git worktree add` command. The worktree will be
// associated with the repository and can be used to work on a different commit
// or branch simultaneously.
func (w *Worktree) Add(wt billy.Filesystem, name string, opts ...Option) error {
if wt == nil {
return errors.New("cannot add worktree: fs is nil")
}
if !worktreeNameRE.MatchString(name) {
return fmt.Errorf("invalid worktree name %q", name)
}
o := &options{}
for _, opt := range opts {
opt(o)
}
if o.commit.IsZero() {
r, err := git.Open(w.storer.(storage.Storer), nil)
if err != nil {
return fmt.Errorf("unable to open repository: %w", err)
}
defer func() {
r.Storer = nil // avoid closing the storer, which is shared with the worktree
_ = r.Close()
}()
ref, err := r.Head()
if err != nil {
return fmt.Errorf("invalid reference: %w", err)
}
o.commit = ref.Hash()
}
err := o.Validate()
if err != nil {
return err
}
commonDir := w.storer.Filesystem()
path := filepath.Join(commonDir.Root(), worktrees, name)
_, err = commonDir.Lstat(path)
if err == nil {
return ErrWorktreeAlreadyExists
}
err = w.addDotGitDirs(commonDir, name)
if err != nil {
return err
}
err = w.addDotGitFiles(commonDir, wt, name, o)
if err != nil {
return err
}
err = w.addWorktreeDotGitFile(wt, path)
if err != nil {
return err
}
r, err := w.Open(wt)
if err != nil {
return err
}
defer func() { _ = r.Close() }()
work, err := r.Worktree()
if err != nil {
return err
}
opt := &git.CheckoutOptions{
Hash: o.commit,
}
if !o.detachedHead {
opt.Branch = plumbing.NewBranchReferenceName(name)
opt.Create = true
}
return work.Checkout(opt)
}
// Remove deletes a linked worktree by removing its metadata dir within .git.
//
// This method removes the metadata directory for the specified worktree from
// .git/worktrees/<name>, similar to the `git worktree remove` command. Note
// that this only removes the metadata; it does not delete the actual worktree
// filesystem or its files.
func (w *Worktree) Remove(name string) error {
if !worktreeNameRE.MatchString(name) {
return fmt.Errorf("invalid worktree name %q", name)
}
dotgit := w.storer.Filesystem()
path := filepath.Join(dotgit.Root(), worktrees, name)
fi, err := dotgit.Lstat(path)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return ErrWorktreeNotFound
}
return err
}
if !fi.IsDir() {
return errors.New("invalid worktree")
}
return util.RemoveAll(dotgit, path)
}
// List returns a list of all linked worktree names.
func (w *Worktree) List() ([]string, error) {
dotgit := w.storer.Filesystem()
_, err := dotgit.Lstat(worktrees)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return []string{}, nil
}
return nil, err
}
entries, err := dotgit.ReadDir(worktrees)
if err != nil {
return nil, err
}
var names []string
for _, entry := range entries {
if entry.IsDir() {
names = append(names, entry.Name())
}
}
return names, nil
}
// Open opens a repository that may be a linked worktree.
//
// When the target is not a linked worktree, it behaves just like git.Open.
// This logic is likely going to be moved to git.Open in the future.
func (w *Worktree) Open(wt billy.Filesystem) (*git.Repository, error) {
if wt == nil {
return nil, errors.New("worktree fs is nil")
}
fs := w.getDualFS(wt)
if fs == nil {
fs = w.storer.Filesystem()
}
stor := filesystem.NewStorage(fs, cache.NewObjectLRUDefault())
repo, err := git.Open(stor, wt)
if err != nil {
_ = stor.Close()
return nil, err
}
return repo, nil
}
// Init initialises a worktree filesystem, connecting it to an existing
// worktree metadata.
//
// This is a go-git concept, which adds flexibility to the way linked
// worktrees work. It enables a fs to be connected to an existing metadata,
// which is particularly useful cross-filesystem implementations.
// For example, in-memory worktrees that are connected pre-existing worktree
// metadata on disk - or vice versa.
func (w *Worktree) Init(wt billy.Filesystem, name string) error {
if wt == nil {
return errors.New("worktree fs is nil")
}
if !worktreeNameRE.MatchString(name) {
return fmt.Errorf("invalid worktree name %q", name)
}
commonDir := w.storer.Filesystem()
path := filepath.Join(commonDir.Root(), worktrees, name)
_, err := commonDir.Lstat(path)
if err != nil {
return ErrWorktreeNotFound
}
err = w.addWorktreeDotGitFile(wt, path)
if err != nil {
return fmt.Errorf("unable to create .git file: %w", err)
}
fs := w.getDualFS(wt)
if fs == nil {
return errors.New("unable to generate dual fs")
}
return nil
}
func (w *Worktree) getDualFS(wt billy.Filesystem) billy.Filesystem {
commonDir := w.storer.Filesystem()
f, err := wt.Open(dotgitDir)
if err != nil {
return nil
}
defer func() { _ = f.Close() }()
data, err := io.ReadAll(io.LimitReader(f, worktreeDotGitMaxSize))
if err != nil || len(data) < 9 {
return nil
}
// ensure it is reading gitdir data:
if !bytes.Equal(data[:len(gitDir)], []byte(gitDir)) {
return nil
}
path := strings.TrimSpace(string(data[8:]))
rel, err := filepath.Rel(commonDir.Root(), path)
if err != nil {
return nil
}
wtGitDir, err := commonDir.Chroot(rel)
if err != nil {
return nil
}
return dotgit.NewRepositoryFilesystem(wtGitDir, commonDir)
}
func (w *Worktree) addDotGitDirs(wt billy.Filesystem, name string) error {
return wt.MkdirAll(path(name, refs), dirMode)
}
func (w *Worktree) addWorktreeDotGitFile(wt billy.Filesystem, path string) error {
return writeFile(wt, dotgitDir, []byte("gitdir: "+path))
}
func (w *Worktree) addDotGitFiles(dotgit, wt billy.Filesystem, name string, opts *options) error {
err := writeFile(dotgit, path(name, commonDir), []byte("../.."))
if err != nil {
return err
}
err = writeFile(dotgit, path(name, gitDir), []byte(filepath.Join(wt.Root(), ".git")))
if err != nil {
return err
}
err = writeFile(dotgit, path(name, head), []byte(opts.commit.String()))
if err != nil {
return err
}
return writeFile(dotgit, path(name, originalHead), []byte(opts.commit.String()))
}
func writeFile(wt billy.Filesystem, fn string, data []byte) (err error) {
var f billy.File
f, err = wt.Create(fn)
if err != nil {
return err
}
defer func() {
err = f.Close()
}()
_, err = f.Write(append(data, []byte("\n")...))
return err
}
func path(wtn, fn string) string {
return filepath.Join(worktrees, wtn, fn)
}
package worktree
import (
"errors"
"github.com/go-git/go-git/v6/plumbing"
)
type options struct {
commit plumbing.Hash
detachedHead bool
}
func (o *options) Validate() error {
if o.commit.IsZero() {
return errors.New("commit hash is empty")
}
return nil
}
// Option is a functional option for configuring worktree operations.
// Options are passed to methods like Add to customize their behavior.
type Option func(*options)
// WithCommit specifies the commit hash to check out when adding a new worktree.
//
// The specified commit will be checked out in the new worktree, and both HEAD
// and ORIG_HEAD will be set to point to this commit.
func WithCommit(commit plumbing.Hash) Option {
return func(o *options) {
o.commit = commit
}
}
// WithDetachedHead creates the worktree with a detached HEAD at the specified commit.
//
// Use this option to create a detached HEAD instead, similar to `git worktree add --detach <path>`.
func WithDetachedHead() Option {
return func(o *options) {
o.detachedHead = true
}
}