Something went wrong. Try again.
Monorepo for Tangled tangled.org
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445// Package gitclient is a small git protocol v2 client over smart http, just// enough to read refs, trees and blobs from a knot without cloning.package gitclient
import ( "bytes" "context" "errors" "fmt" "io" "maps" "net/http" "path" "slices" "strings" "sync")
var ( ErrRefNotFound = errors.New("branch or tag not found") ErrPathNotFound = errors.New("path not found at the commit") ErrNotDir = fmt.Errorf("%w: not a directory", ErrPathNotFound) ErrTooLarge = errors.New("pack exceeds the read limit") ErrUnsupported = errors.New("server lacks a required v2 capability"))
// hex object id, sha1 or sha256 depending on the repo's object-formattype Oid string
type Client struct { http *http.Client url string // repo base, e.g. https://knot.example/did:plc:abc maxBytes int64 // cap on one fetch's pack
mu sync.Mutex caps *Caps}
func New(httpClient *http.Client, repoURL string, maxBytes int64) *Client { if httpClient == nil { httpClient = http.DefaultClient } return &Client{http: httpClient, url: strings.TrimSuffix(repoURL, "/"), maxBytes: maxBytes}}
type Caps struct { ObjectFormat string // "sha1" or "sha256" LsRefs bool Fetch []string // fetch= features: shallow, filter, ...}
func (c *Client) Caps(ctx context.Context) (Caps, error) { c.mu.Lock() defer c.mu.Unlock() if c.caps != nil { return *c.caps, nil } u := c.url + "/info/refs?service=git-upload-pack" req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) if err != nil { return Caps{}, err } req.Header.Set("Git-Protocol", "version=2") resp, err := c.http.Do(req) if err != nil { return Caps{}, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return Caps{}, httpError(resp) } caps := Caps{ObjectFormat: "sha1"} p := newPktReader(io.LimitReader(resp.Body, 1<<20)) v2 := false for { kind, line, err := p.next() if err != nil && !v2 { return Caps{}, fmt.Errorf("%w: %s doesn't speak protocol v2", ErrUnsupported, u) } if err != nil { return Caps{}, fmt.Errorf("reading %s: %w", u, err) } if kind != pktData { // a v0-style "# service=" banner ends in a flush before the v2 advert if v2 { break } continue } key, val, _ := strings.Cut(strings.TrimSuffix(string(line), "\n"), "=") switch key { case "version 2": v2 = true case "ls-refs": caps.LsRefs = true case "fetch": caps.Fetch = strings.Fields(val) case "object-format": caps.ObjectFormat = val } } if caps.ObjectFormat != "sha1" && caps.ObjectFormat != "sha256" { return Caps{}, fmt.Errorf("%w: object-format=%s", ErrUnsupported, caps.ObjectFormat) } c.caps = &caps return caps, nil}
type Ref struct { Name string Oid Oid Peeled Oid // set for annotated tags when Peel was asked Symref string // target of HEAD when Symrefs was asked}
type LsRefsOpts struct { Prefixes []string // ref-prefix lines Peel bool Symrefs bool}
// command=ls-refsfunc (c *Client) LsRefs(ctx context.Context, opts LsRefsOpts) ([]Ref, error) { caps, err := c.Caps(ctx) if err != nil { return nil, err } if !caps.LsRefs { return nil, fmt.Errorf("%w: ls-refs", ErrUnsupported) } var args []string if opts.Peel { args = append(args, "peel") } if opts.Symrefs { args = append(args, "symrefs") } for _, prefix := range opts.Prefixes { args = append(args, "ref-prefix "+prefix) } resp, err := c.command(ctx, caps, "ls-refs", args) if err != nil { return nil, err } defer resp.Body.Close() p := newPktReader(&capReader{r: resp.Body, n: c.maxBytes}) var refs []Ref for { kind, line, err := p.next() if err != nil { return nil, fmt.Errorf("ls-refs: %w", err) } if kind != pktData { return refs, nil } text := strings.TrimSuffix(string(line), "\n") if msg, ok := strings.CutPrefix(text, "ERR "); ok { return nil, fmt.Errorf("remote: %s", msg) } fields := strings.Split(text, " ") if len(fields) < 2 || !isOid(fields[0]) { return nil, fmt.Errorf("ls-refs: bad line %q", text) } ref := Ref{Oid: Oid(fields[0]), Name: fields[1]} for _, attr := range fields[2:] { if v, ok := strings.CutPrefix(attr, "peeled:"); ok { ref.Peeled = Oid(v) } else if v, ok := strings.CutPrefix(attr, "symref-target:"); ok { ref.Symref = v } } refs = append(refs, ref) }}
type FetchOpts struct { Wants []Oid Deepen int // 0 means no deepen line Filter string // e.g. "tree:0", "blob:none"; empty means none}
// command=fetch with done and no haves, since there's no local object storefunc (c *Client) Fetch(ctx context.Context, opts FetchOpts) (Objects, error) { caps, err := c.Caps(ctx) if err != nil { return nil, err } if opts.Filter != "" && !slices.Contains(caps.Fetch, "filter") { return nil, fmt.Errorf("%w: fetch=filter", ErrUnsupported) } if opts.Deepen > 0 && !slices.Contains(caps.Fetch, "shallow") { return nil, fmt.Errorf("%w: fetch=shallow", ErrUnsupported) } if len(opts.Wants) == 0 { return Objects{}, nil } args := []string{"no-progress", "ofs-delta"} for _, want := range opts.Wants { if len(want) != oidLen(caps.ObjectFormat) || !isOid(string(want)) { return nil, fmt.Errorf("bad object id %q", want) } args = append(args, "want "+string(want)) } if opts.Deepen > 0 { args = append(args, fmt.Sprintf("deepen %d", opts.Deepen)) } if opts.Filter != "" { args = append(args, "filter "+opts.Filter) } args = append(args, "done") resp, err := c.command(ctx, caps, "fetch", args) if err != nil { return nil, err } defer resp.Body.Close() pack, err := packfileSection(newPktReader(resp.Body)) if err != nil { return nil, fmt.Errorf("fetch: %w", err) } objs, err := readPack(&capReader{r: pack, n: c.maxBytes}, caps.ObjectFormat) if err != nil { return nil, fmt.Errorf("fetch: %w", err) } return objs, nil}
// branches before tags, like git; tags peeledfunc (c *Client) Resolve(ctx context.Context, ref string) (Oid, error) { if ref == "" || strings.ContainsAny(ref, "\x00\n ") { return "", ErrRefNotFound } candidates := []string{ref} if !strings.HasPrefix(ref, "refs/") { candidates = []string{"refs/heads/" + ref, "refs/tags/" + ref} } refs, err := c.LsRefs(ctx, LsRefsOpts{Prefixes: candidates, Peel: true}) if err != nil { return "", err } for _, name := range candidates { for _, r := range refs { if r.Name != name { continue } if r.Peeled != "" { return r.Peeled, nil } return r.Oid, nil } } return "", ErrRefNotFound}
type File struct { Name string Mode string // octal, as in the tree Oid Oid Data []byte // nil for subtrees and submodules}
// one level of dir, blobs filled infunc (c *Client) ReadDir(ctx context.Context, commit Oid, dir string) ([]File, error) { caps, objs, oid, err := c.walk(ctx, commit, dir) if err != nil { return nil, err } entries, err := c.tree(ctx, caps, objs, oid, "tree:1") if err != nil { return nil, err } files := make([]File, 0, len(entries)) for _, e := range entries { files = append(files, File{Name: e.Name, Mode: e.Mode, Oid: e.Oid}) } if err := c.fillBlobs(ctx, objs, files); err != nil { return nil, err } return files, nil}
// everything under dir, named relative to it; the dir tree is wanted unfiltered// so one pack brings it allfunc (c *Client) ReadTree(ctx context.Context, commit Oid, dir string) ([]File, error) { caps, objs, oid, err := c.walk(ctx, commit, dir) if err != nil { return nil, err } var files []File queue := []File{{Oid: oid}} for len(queue) > 0 { t := queue[0] queue = queue[1:] entries, err := c.tree(ctx, caps, objs, t.Oid, "") if err != nil { return nil, err } for _, e := range entries { f := File{Name: path.Join(t.Name, e.Name), Mode: e.Mode, Oid: e.Oid} if isTreeMode(e.Mode) { queue = append(queue, f) } files = append(files, f) } } if err := c.fillBlobs(ctx, objs, files); err != nil { return nil, err } return files, nil}
// middle trees come alone (tree:0) so sibling blobs stay on the server; servers// differ on filter depth for a wanted tree, hence fillBlobsfunc (c *Client) walk(ctx context.Context, commit Oid, dir string) (Caps, Objects, Oid, error) { caps, err := c.Caps(ctx) if err != nil { return Caps{}, nil, "", err } objs, err := c.Fetch(ctx, FetchOpts{Wants: []Oid{commit}, Deepen: 1, Filter: "tree:1"}) if err != nil { return Caps{}, nil, "", err } obj, ok := objs[commit] if !ok || obj.Kind != KindCommit { return Caps{}, nil, "", fmt.Errorf("fetch of commit %s didn't return it", commit) } tree, err := commitTree(obj.Data) if err != nil { return Caps{}, nil, "", err } for _, name := range strings.Split(dir, "/") { if name == "" { continue } entries, err := c.tree(ctx, caps, objs, tree, "tree:0") if err != nil { return Caps{}, nil, "", err } i := slices.IndexFunc(entries, func(e treeEntry) bool { return e.Name == name }) if i < 0 { return Caps{}, nil, "", fmt.Errorf("%s: %w", dir, ErrPathNotFound) } if !isTreeMode(entries[i].Mode) { return Caps{}, nil, "", fmt.Errorf("%s: %w", dir, ErrNotDir) } tree = entries[i].Oid } return caps, objs, tree, nil}
func (c *Client) fillBlobs(ctx context.Context, objs Objects, files []File) error { var missing []Oid for _, f := range files { if _, ok := objs[f.Oid]; isBlobMode(f.Mode) && !ok && !slices.Contains(missing, f.Oid) { missing = append(missing, f.Oid) } } if len(missing) > 0 { // wanted objects pass any filter; it keeps the server on its selective path more, err := c.Fetch(ctx, FetchOpts{Wants: missing, Filter: "tree:0"}) if err != nil { return err } maps.Copy(objs, more) } for i, f := range files { if !isBlobMode(f.Mode) { continue } obj, ok := objs[f.Oid] if !ok || obj.Kind != KindBlob { return fmt.Errorf("fetch of blob %s didn't return it", f.Oid) } files[i].Data = obj.Data } return nil}
func (c *Client) tree(ctx context.Context, caps Caps, have Objects, oid Oid, filter string) ([]treeEntry, error) { if _, ok := have[oid]; !ok { objs, err := c.Fetch(ctx, FetchOpts{Wants: []Oid{oid}, Filter: filter}) if err != nil { return nil, err } maps.Copy(have, objs) } obj, ok := have[oid] if !ok || obj.Kind != KindTree { return nil, fmt.Errorf("fetch of tree %s didn't return it", oid) } return parseTree(obj.Data, caps.ObjectFormat)}
func isBlobMode(mode string) bool { mode = strings.TrimLeft(mode, "0") return mode != "40000" && mode != "160000"}
func isTreeMode(mode string) bool { return strings.TrimLeft(mode, "0") == "40000"}
func (c *Client) command(ctx context.Context, caps Caps, command string, args []string) (*http.Response, error) { var body bytes.Buffer writePkt(&body, "command="+command+"\n") // git refuses non-sha1 unless the format is echoed back if caps.ObjectFormat != "sha1" { writePkt(&body, "object-format="+caps.ObjectFormat+"\n") } writeDelim(&body) for _, arg := range args { if err := writePkt(&body, arg+"\n"); err != nil { return nil, err } } writeFlush(&body)
u := c.url + "/git-upload-pack" req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, &body) if err != nil { return nil, err } req.Header.Set("Git-Protocol", "version=2") req.Header.Set("Content-Type", "application/x-git-upload-pack-request") req.Header.Set("Accept", "application/x-git-upload-pack-result") resp, err := c.http.Do(req) if err != nil { return nil, err } if resp.StatusCode != http.StatusOK { defer resp.Body.Close() return nil, httpError(resp) } return resp, nil}
// knot2 says why in a plain text body, like a want it won't servefunc httpError(resp *http.Response) error { body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) err := fmt.Errorf("%s %s: %s", resp.Request.Method, resp.Request.URL, resp.Status) if msg := strings.TrimSpace(string(body)); msg != "" && !strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") { err = fmt.Errorf("%w: %s", err, strings.ToValidUTF8(msg, "?")) } return err}