Something went wrong. Try again.
Monorepo for Tangled forked from tangled.org/core
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337package git
import ( "bytes" "context" "errors" "fmt" "io" "io/fs" "net/url" "os" "os/exec" "path/filepath" "regexp" "slices" "strings" "sync" "syscall" "time")
var ( ErrDiskLimitExceeded = errors.New("scratch disk limit exceeded") ErrPackLimitExceeded = errors.New("git pack limit exceeded") ErrAuthRequired = errors.New("authorization required") ErrRepoNotFound = errors.New("repository not found"))
type CommandRunner interface { Run(ctx context.Context, dir string, env []string, name string, args ...string) (string, error) LookPath(file string) (string, error)}
type RealCommandRunner struct { MaxOutputBytes int MaxDiskBytes int64 WatchDir string DiskPollInterval time.Duration}
func (r RealCommandRunner) LookPath(file string) (string, error) { return exec.LookPath(file)}
func (r RealCommandRunner) Run(ctx context.Context, dir string, env []string, name string, args ...string) (string, error) { cmdCtx, cmdCancel := context.WithCancel(ctx) defer cmdCancel()
cmd := exec.CommandContext(cmdCtx, name, args...) cmd.Dir = dir cmd.Env = env // git spawns child helpers like git-remote-https, kill the process group cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} cmd.Cancel = func() error { if cmd.Process == nil { return nil } return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) }
maxOut := r.MaxOutputBytes if maxOut <= 0 { maxOut = 256 * 1024 }
var stdoutBuf, stderrBuf bytes.Buffer cmd.Stdout = &boundedWriter{w: &stdoutBuf, limit: maxOut} cmd.Stderr = &boundedWriter{w: &stderrBuf, limit: maxOut}
var stopMonitor chan struct{} var monitorDone chan struct{} var diskMu sync.Mutex var diskErr error
if r.WatchDir != "" && r.MaxDiskBytes > 0 { stopMonitor = make(chan struct{}) monitorDone = make(chan struct{}) baseUsed, baseErr := fsUsedBytes(r.WatchDir) go func() { defer close(monitorDone) interval := r.DiskPollInterval if interval <= 0 { interval = time.Second } ticker := time.NewTicker(interval) defer ticker.Stop() ticks := diskSweepEvery - 1 for { select { case <-stopMonitor: return case <-cmdCtx.Done(): return case <-ticker.C: ticks++ grown := baseErr == nil if used, err := fsUsedBytes(r.WatchDir); err != nil || used-baseUsed <= r.MaxDiskBytes { grown = false } if !grown && ticks%diskSweepEvery != 0 { continue } size, err := dirSize(r.WatchDir) if err == nil && size > r.MaxDiskBytes { diskMu.Lock() diskErr = fmt.Errorf("%w: current %d bytes > max %d bytes", ErrDiskLimitExceeded, size, r.MaxDiskBytes) diskMu.Unlock() cmdCancel() return } } } }() }
err := cmd.Run()
if stopMonitor != nil { close(stopMonitor) <-monitorDone }
diskMu.Lock() capturedDiskErr := diskErr diskMu.Unlock()
outStr := stdoutBuf.String() if stderr := stderrBuf.String(); stderr != "" { outStr += stderr }
if capturedDiskErr != nil { return outStr, capturedDiskErr }
if err != nil { return outStr, fmt.Errorf("command %s failed: %w (output: %s)", name, err, sanitizeOutput(outStr)) }
return outStr, nil}
type boundedWriter struct { w io.Writer limit int total int}
func (b *boundedWriter) Write(p []byte) (int, error) { if b.total >= b.limit { return len(p), nil } remaining := b.limit - b.total toWrite := p if len(toWrite) > remaining { toWrite = toWrite[:remaining] } n, err := b.w.Write(toWrite) b.total += n return len(p), err}
const diskSweepEvery = 15
func fsUsedBytes(path string) (int64, error) { var st syscall.Statfs_t if err := syscall.Statfs(path, &st); err != nil { return 0, err } return int64(st.Blocks-st.Bavail) * int64(st.Bsize), nil}
func dirSize(path string) (int64, error) { var size int64 err := filepath.WalkDir(path, func(_ string, d fs.DirEntry, err error) error { if err != nil { return nil } if !d.IsDir() { info, err := d.Info() if err == nil { size += info.Size() } } return nil }) return size, err}
func RedactSecrets(s string, secrets ...string) string { res := s for _, sec := range secrets { if sec == "" || len(sec) < 4 { continue } res = strings.ReplaceAll(res, sec, "[REDACTED]") if encoded := url.QueryEscape(sec); encoded != sec { res = strings.ReplaceAll(res, encoded, "[REDACTED]") } if encoded := url.PathEscape(sec); encoded != sec { res = strings.ReplaceAll(res, encoded, "[REDACTED]") } } return res}
var credentialPattern = regexp.MustCompile(`(Bearer|Basic)\s+[A-Za-z0-9._~+/-]+=*`)
func sanitizeOutput(s string) string { return credentialPattern.ReplaceAllString(s, "$1 [REDACTED]")}
var authErrorMarkers = []string{ "401", "403", "authentication failed", "invalid username or password", "could not read username", "device not configured", "terminal prompts disabled",}
func ClassifyGitError(output string, exitErr error) error { outLower := strings.ToLower(output) // github answers a missing repository (and one the token cannot see) // with "repository not found"; that is not proof of a bad credential, // so it stays retryable instead of ending the job as authorization // required and scrubbing the batch token if strings.Contains(outLower, "repository not found") { return fmt.Errorf("%w: %s", ErrRepoNotFound, strings.TrimSpace(output)) } if slices.ContainsFunc(authErrorMarkers, func(m string) bool { return strings.Contains(outLower, m) }) { return fmt.Errorf("%w: %s", ErrAuthRequired, strings.TrimSpace(output)) } return exitErr}
func HardenedGitEnv(proxyAddr, askpassBin, homeDir string, askpassHost, askpassToken string) []string { env := []string{ "PATH=" + os.Getenv("PATH"), "HOME=" + homeDir, "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null", "GIT_TERMINAL_PROMPT=0", // without this a source url of file:///etc/passwd is a readable repository "GIT_ALLOW_PROTOCOL=https", "GIT_PROTOCOL_FROM_USER=0", }
if proxyAddr != "" { proxyURL := "http://" + proxyAddr env = append(env, "http_proxy="+proxyURL, "HTTP_PROXY="+proxyURL, "https_proxy="+proxyURL, "HTTPS_PROXY="+proxyURL, "all_proxy="+proxyURL, "ALL_PROXY="+proxyURL, "NO_PROXY=", "no_proxy=", ) }
// git gets none of our env, but a trust store set for the whole process is the operator's if ca := os.Getenv("SSL_CERT_FILE"); ca != "" { env = append(env, "GIT_SSL_CAINFO="+ca) }
if askpassBin != "" { env = append(env, "GIT_ASKPASS="+askpassBin, "MIGRATOR_ASKPASS_HOST="+askpassHost, "MIGRATOR_ASKPASS_TOKEN="+askpassToken, ) }
return env}
var quotedURLRegex = regexp.MustCompile(`'([^']+)'`)
func ExtractHostFromPrompt(prompt string) string { matches := quotedURLRegex.FindStringSubmatch(prompt) if len(matches) > 1 { u, err := url.Parse(matches[1]) if err == nil { return strings.ToLower(u.Hostname()) } } re := regexp.MustCompile(`https?://(?:[^@/\s]+@)?([^/':@\s]+)`) m := re.FindStringSubmatch(prompt) if len(m) > 1 { return strings.ToLower(m[1]) } return ""}
func HandleAskpass(prompt, expectedHost, token string) (string, error) { promptHost := ExtractHostFromPrompt(prompt) if promptHost == "" { return "", fmt.Errorf("no host found in askpass prompt %q", prompt) }
expectedHost = strings.ToLower(expectedHost) if !strings.EqualFold(promptHost, expectedHost) { return "", fmt.Errorf("host mismatch: prompt host %q != expected host %q", promptHost, expectedHost) }
promptLower := strings.ToLower(prompt) if strings.Contains(promptLower, "username") { return "x-access-token", nil } if strings.Contains(promptLower, "password") { return token, nil }
return "", fmt.Errorf("unknown askpass prompt: %q", prompt)}
func CreateAskpassScript(dir string) (string, error) { migratorBin, err := os.Executable() if err != nil { return "", fmt.Errorf("getting migrator executable path: %w", err) }
scriptPath := filepath.Join(dir, "askpass.sh") content := fmt.Sprintf(`#!/bin/shexec "%s" askpass "$@"`, migratorBin)
if err := os.WriteFile(scriptPath, []byte(content), 0700); err != nil { return "", fmt.Errorf("writing askpass script: %w", err) }
return scriptPath, nil}