Something went wrong. Try again.
[mirror] Scalable static site server for Git forges (like GitHub Pages)
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966package git_pages
import ( "cmp" "context" "crypto/tls" "encoding/json" "errors" "flag" "fmt" "io" "log" "log/slog" "net" "net/http" "net/http/httputil" "net/url" "os" "path" "runtime/debug" "slices" "strconv" "strings" "time"
"codeberg.org/git-pages/git-pages/src/names" sys "codeberg.org/git-pages/git-pages/src/sys" automemlimit "github.com/KimMachineGun/automemlimit/memlimit" "github.com/c2h5oh/datasize" "github.com/fatih/color" "github.com/kankanreno/go-snowflake" "github.com/prometheus/client_golang/prometheus/promhttp" "golang.org/x/crypto/acme" "google.golang.org/protobuf/proto")
var config *Configvar wildcards []*WildcardPatternvar fallback http.Handlervar backend Backendvar secrets Secretsvar existenceCache ExistenceCache
func configureFeatures(ctx context.Context) (err error) { if len(config.Features) > 0 { logc.Println(ctx, "features:", strings.Join(config.Features, ", ")) } for _, feature := range config.Features { switch feature { // Work-in-progress features: case "acme": // Permanently unstable features: case "codeberg-pages-compat", "relaxed-idna": // Stabilized features: case "archive-site", "audit", "compress", "patch", "serve-h2c", "existence-cache", "absolute-headers", "expiration", "preview": logc.Printf(ctx, "feature %s has been stabilized", feature) // Removed or renamed features: case "h2c", "sentry-telemetry-buffer", "site-existence-cache", "domain-existence-cache": logc.Printf(ctx, "feature %s has been removed", feature) // Invalid features: default: return fmt.Errorf("unknown feature %q", feature) } } return}
func configureMemLimit(ctx context.Context) (err error) { // Avoid being OOM killed by not garbage collecting early enough. memlimitBefore := datasize.ByteSize(debug.SetMemoryLimit(-1)) automemlimit.SetGoMemLimitWithOpts( automemlimit.WithLogger(slog.New(slog.DiscardHandler)), automemlimit.WithProvider( automemlimit.ApplyFallback( automemlimit.FromCgroup, automemlimit.FromSystem, ), ), automemlimit.WithRatio(float64(config.Limits.MaxHeapSizeRatio)), ) memlimitAfter := datasize.ByteSize(debug.SetMemoryLimit(-1)) if memlimitBefore == memlimitAfter { logc.Println(ctx, "memlimit: now", memlimitBefore.HR()) } else { logc.Println(ctx, "memlimit: was", memlimitBefore.HR(), "now", memlimitAfter.HR()) } return}
// Can only be safely called during initial configuration.func configureConcurrency(_ context.Context) (err error) { putBlobSemaphore = make(chan struct{}, config.Limits.ConcurrentUploads) return}
func configureWildcards(_ context.Context) (err error) { newWildcards, err := TranslateWildcards(config.Wildcard) if err != nil { return err } else { wildcards = newWildcards return nil }}
func configureFallback(_ context.Context) (err error) { if config.Fallback.ProxyTo != nil { fallbackURL := &config.Fallback.ProxyTo.URL fallback = &httputil.ReverseProxy{ Rewrite: func(r *httputil.ProxyRequest) { r.SetURL(fallbackURL) r.Out.Host = r.In.Host r.Out.Header["X-Forwarded-For"] = r.In.Header["X-Forwarded-For"] }, Transport: &http.Transport{ TLSClientConfig: &tls.Config{ InsecureSkipVerify: config.Fallback.Insecure, }, }, } } return}
// Thread-unsafe, must be called only during initial configuration.func configureAudit(_ context.Context) (err error) { snowflake.SetStartTime(AuditSnowflakeStartTime) snowflake.SetMachineID(config.Audit.NodeID) return}
func listen(ctx context.Context, name string, listen string, tlsConfig *tls.Config) net.Listener { if listen == "-" { return nil }
protocol, address, ok := strings.Cut(listen, "/") if !ok { logc.Fatalf(ctx, "%s: %s: malformed endpoint", name, listen) }
listener, err := net.Listen(protocol, address) if err != nil { logc.Fatalf(ctx, "%s: %s\n", name, err) }
return &tlsListener{listener, tlsConfig}}
func panicHandler(handler http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { defer func() { if err := recover(); err != nil { if err, ok := err.(error); ok && errors.Is(err, http.ErrAbortHandler) { panic(http.ErrAbortHandler) } logc.Printf(r.Context(), "panic: %s %s %s: %s\n%s", r.Method, r.Host, r.URL.Path, err, string(debug.Stack())) http.Error(w, fmt.Sprintf("internal server error: %s", err), http.StatusInternalServerError, ) } }() handler.ServeHTTP(w, r) })}
func serve(ctx context.Context, listener net.Listener, handler http.Handler) { if listener != nil { server := http.Server{Handler: handler} server.Protocols = new(http.Protocols) server.Protocols.SetHTTP1(true) server.Protocols.SetUnencryptedHTTP2(true) logc.Fatalln(ctx, server.Serve(listener)) }}
func domainArg(arg string) (domain names.Domain) { var err error if domain, err = names.ParseDomain(arg, names.ParseDomainRelaxed); err != nil { logc.Fatalf(context.Background(), "invalid site argument: %s\n", err) } return}
func siteArg(arg string) (site names.Site) { var err error switch strings.Count(arg, "/") { case 0: site.Domain = domainArg(arg) case 1: if site, err = names.ParseSite(arg); err != nil { logc.Fatalf(context.Background(), "invalid site argument: %s\n", err) } default: logc.Fatalln(context.Background(), "site argument must be either 'domain.tld' or 'domain.tld/dir") } return}
func fileOutputArg() (writer io.WriteCloser) { var err error if flag.NArg() == 0 { writer = os.Stdout } else { writer, err = os.Create(flag.Arg(0)) if err != nil { logc.Fatalln(context.Background(), err) } } return}
func usage() { fmt.Fprintf(os.Stderr, "Usage:\n") fmt.Fprintf(os.Stderr, "(server) "+ "git-pages [-config <file>|-no-config]\n") fmt.Fprintf(os.Stderr, "(info) "+ "git-pages {-version|-print-config-env-vars|-print-config}\n") fmt.Fprintf(os.Stderr, "(debug) "+ "git-pages {-list-blobs|-list-manifests}\n") fmt.Fprintf(os.Stderr, "(debug) "+ "git-pages {-get-blob|-get-manifest|-get-archive} <ref> [file]\n") fmt.Fprintf(os.Stderr, "(admin) "+ "git-pages {-update-site <ref> <file>|-delete-site <ref>}\n") fmt.Fprintf(os.Stderr, "(admin) "+ "git-pages {-freeze-domain|-unfreeze-domain|-purge-domain} <domain>\n") fmt.Fprintf(os.Stderr, "(audit) "+ "git-pages {-audit-log|-audit-read <id>|-audit-rollback <id>}\n") fmt.Fprintf(os.Stderr, "(audit) "+ "git-pages {-audit-expire <days>|-audit-detach <domain>/<project>}\n") fmt.Fprintf(os.Stderr, "(audit) "+ "git-pages -audit-server <endpoint> <program> [args...]\n") fmt.Fprintf(os.Stderr, "(maint) "+ "git-pages {-expire-sites|-scrub-storage} [-dry-run]\n") fmt.Fprintf(os.Stderr, "(maint) "+ "git-pages {-run-migration <name>|-trace-garbage|-analyze-storage {text|json}}\n") flag.PrintDefaults()}
func Main(versionInfo string) { ctx := context.Background()
flag.Usage = usage configTomlPath := flag.String("config", "", "load configuration from `filename` (default: 'config.toml')") secretTomlPath := flag.String("secrets", "", "load additional configuration values from `filename` (default: '$CREDENTIALS_DIRECTORY/secrets.toml' if it exists)") noConfig := flag.Bool("no-config", false, "run without configuration file (configure via environment variables)") printConfigEnvVars := flag.Bool("print-config-env-vars", false, "print every recognized configuration environment variable and exit") printConfig := flag.Bool("print-config", false, "print configuration as JSON and exit") listBlobs := flag.Bool("list-blobs", false, "enumerate every blob with its metadata") listManifests := flag.Bool("list-manifests", false, "enumerate every manifest with its metadata") getBlob := flag.String("get-blob", "", "write contents of `blob` ('sha256-xxxxxxx...xxx')") getManifest := flag.String("get-manifest", "", "write manifest for `site` (either 'domain.tld' or 'domain.tld/dir') as ProtoJSON") getArchive := flag.String("get-archive", "", "write archive for `site` (either 'domain.tld' or 'domain.tld/dir') in tar format") updateSite := flag.String("update-site", "", "update `site` (either 'domain.tld' or 'domain.tld/dir') from archive or repository URL") deleteSite := flag.String("delete-site", "", "delete `site` (either 'domain.tld' or 'domain.tld/dir')") freezeDomain := flag.String("freeze-domain", "", "prevent any site uploads to a given `domain`") unfreezeDomain := flag.String("unfreeze-domain", "", "allow site uploads to a `domain` again after it has been frozen") purgeDomain := flag.String("purge-domain", "", "delete all sites under `domain` (but not its subdomains)") auditLog := flag.Bool("audit-log", false, "display audit log") auditRead := flag.String("audit-read", "", "extract contents of audit record `id` to files '<id>-*'") auditRollback := flag.String("audit-rollback", "", "restore site from contents of audit record `id`") auditExpire := flag.String("audit-expire", "", "expire audit records older than `days` old") auditDetach := flag.String("audit-detach", "", "detach all blobs of audit records for a single `site` (or the entire domain with 'domain.tld/*')") auditServer := flag.String("audit-server", "", "listen for notifications on `endpoint` and spawn a process for each audit event") expireSites := flag.Bool("expire-sites", false, "expire sites according to their manifest") scrubStorage := flag.Bool("scrub-storage", false, "verify well-formedness of all data (CAUTION: will take offline sites with integrity issues)") runMigration := flag.String("run-migration", "", "run a store `migration` (one of: create-domain-markers)") analyzeStorage := flag.String("analyze-storage", "", "display aggregate storage used per domain (argument is output mode and one of: text, json)") traceGarbage := flag.Bool("trace-garbage", false, "estimate total size of unreachable blobs") reason := flag.String("reason", "", "specify a reason for administrative operations in the audit log") dryRun := flag.Bool("dry-run", false, "print what would be performed instead of executing it") version := flag.Bool("version", false, "display version") flag.Parse()
if *version { fmt.Printf("git-pages %s\n", versionInfo) os.Exit(0) }
var cliOperations int for _, selected := range []bool{ *listBlobs, *listManifests, *getBlob != "", *getManifest != "", *getArchive != "", *updateSite != "", *deleteSite != "", *freezeDomain != "", *unfreezeDomain != "", *purgeDomain != "", *auditLog, *auditRead != "", *auditRollback != "", *auditExpire != "", *auditDetach != "", *auditServer != "", *expireSites, *scrubStorage, *runMigration != "", *analyzeStorage != "", *traceGarbage, } { if selected { cliOperations++ } } if cliOperations > 1 { logc.Fatalln(ctx, "-list-blobs, -list-manifests, -get-blob, -get-manifest, "+ "-get-archive, -update-site, -delete-site, -freeze-domain, -unfreeze-domain, "+ "-purge-domain, -audit-log, -audit-read, -audit-rollback, -audit-expire, "+ "-audit-detach, -audit-server, -expire-sites, -scrub-storage, -run-migration, "+ "-analyze-storage, and -trace-garbage are mutually exclusive") } if *dryRun && !(*expireSites || *scrubStorage) { logc.Fatalln(ctx, "-dry-run is not applicable in this context") }
if *configTomlPath != "" && *noConfig { logc.Fatalln(ctx, "-no-config and -config are mutually exclusive") }
if *printConfigEnvVars { PrintConfigEnvVars() return }
var err error if *configTomlPath == "" && !*noConfig { *configTomlPath = "config.toml" }
if *secretTomlPath == "" && !*noConfig { // check for a second config file at $CREDENTIALS_DIRECTORY/secrets.toml, and use it if systemdCredentialsDir := os.Getenv("CREDENTIALS_DIRECTORY"); systemdCredentialsDir != "" { secretTomlTestPath := path.Join(systemdCredentialsDir, "secrets.toml") _, err := os.Stat(secretTomlTestPath) if !errors.Is(err, os.ErrNotExist) { *secretTomlPath = secretTomlTestPath } } }
if config, err = Configure(*configTomlPath, *secretTomlPath); err != nil { logc.Fatalln(ctx, "config:", err) }
if *printConfig { fmt.Println(config.TOML()) return }
InitObservability() defer FiniObservability()
if err = errors.Join( configureFeatures(ctx), configureMemLimit(ctx), configureConcurrency(ctx), configureWildcards(ctx), configureFallback(ctx), configureAudit(ctx), ); err != nil { logc.Fatalln(ctx, err) }
// The server has its own logic for creating the backend. if cliOperations > 0 { if backend, err = CreateBackend(ctx, &config.Storage); err != nil { logc.Fatalln(ctx, err) }
if existenceCache, err = CreateExistenceCache(ctx); err != nil { logc.Fatalln(ctx, err) }
ctx = WithPrincipal(ctx) GetPrincipal(ctx).CliAdmin = proto.Bool(true)
ctx = WithReason(ctx) *GetReason(ctx) = *reason }
switch { case *listBlobs: for metadata, err := range backend.EnumerateBlobs(ctx) { if err != nil { logc.Fatalln(ctx, err) } fmt.Fprintf(color.Output, "%s %s %s\n", metadata.Name, color.HiWhiteString(metadata.LastModified.UTC().Format(time.RFC3339)), color.HiGreenString(fmt.Sprint(metadata.Size)), ) }
case *listManifests: for item, err := range backend.GetAllManifests(ctx) { metadata, manifest := item.Splat() if err != nil { logc.Fatalln(ctx, err) } parts := []string{} if manifest.ExpiresAt != nil { parts = append(parts, color.HiYellowString("%s", metadata.Name), ) } else { parts = append(parts, color.HiMagentaString("%s", metadata.Name), ) } parts = append(parts, color.HiWhiteString(metadata.LastModified.UTC().Format(time.RFC3339)), color.HiGreenString(fmt.Sprint(metadata.Size)), ) if manifest.GetCorrupted() { parts = append(parts, color.HiRedString("(corrupted)"), ) } fmt.Fprintln(color.Output, strings.Join(parts, " ")) }
case *getBlob != "": reader, _, err := backend.GetBlob(ctx, *getBlob) if err != nil { logc.Fatalln(ctx, err) } io.Copy(fileOutputArg(), reader)
case *getManifest != "": webRoot := siteArg(*getManifest) manifest, _, err := backend.GetManifest(ctx, webRoot, GetManifestOptions{}) if err != nil { logc.Fatalln(ctx, err) } fmt.Fprintln(fileOutputArg(), string(ManifestJSON(manifest)))
case *getArchive != "": webRoot := siteArg(*getArchive) manifest, metadata, err := backend.GetManifest(ctx, webRoot, GetManifestOptions{}) if err != nil { logc.Fatalln(ctx, err) } if err = CollectTar(ctx, fileOutputArg(), manifest, metadata); err != nil { logc.Fatalln(ctx, err) }
case *updateSite != "": if flag.NArg() != 1 { logc.Fatalln(ctx, "update source must be provided as the argument") }
webRoot := siteArg(*updateSite)
sourceURL, err := url.Parse(flag.Arg(0)) if err != nil { logc.Fatalln(ctx, err) }
var result UpdateResult if sourceURL.Scheme == "" { file, err := os.Open(sourceURL.Path) if err != nil { logc.Fatalln(ctx, err) } defer file.Close()
var contentType string switch { case strings.HasSuffix(sourceURL.Path, ".zip"): contentType = "application/zip" case strings.HasSuffix(sourceURL.Path, ".tar"): contentType = "application/x-tar" case strings.HasSuffix(sourceURL.Path, ".tar.gz"): contentType = "application/x-tar+gzip" case strings.HasSuffix(sourceURL.Path, ".tar.zst"): contentType = "application/x-tar+zstd" default: log.Fatalf("cannot determine content type from filename %q\n", sourceURL) }
result = UpdateFromArchive(ctx, webRoot, "", contentType, file, UpdateOptions{}) } else { branch := "pages" if sourceURL.Fragment != "" { branch, sourceURL.Fragment = sourceURL.Fragment, "" }
result = UpdateFromRepository(ctx, webRoot, sourceURL.String(), branch, UpdateOptions{}) }
switch result.outcome { case UpdateError: logc.Println(ctx, result.err) os.Exit(2) case UpdateTimeout: logc.Printf(ctx, "admin: timeout %s\n", webRoot) os.Exit(1) case UpdateCreated: logc.Printf(ctx, "admin: created %s\n", webRoot) case UpdateReplaced: logc.Printf(ctx, "admin: replaced %s\n", webRoot) case UpdateDeleted: logc.Printf(ctx, "admin: deleted %s\n", webRoot) case UpdateNoChange: logc.Printf(ctx, "admin: no-change %s\n", webRoot) }
case *deleteSite != "": webRoot := siteArg(*deleteSite) err := backend.DeleteManifest(ctx, webRoot, ModifyManifestOptions{}) if err != nil { logc.Fatalln(ctx, err) }
logc.Println(ctx, "deleted")
case *freezeDomain != "" || *unfreezeDomain != "": var domain names.Domain var freeze bool if *freezeDomain != "" { domain = domainArg(*freezeDomain) freeze = true } else { domain = domainArg(*unfreezeDomain) freeze = false }
if freeze { if err = backend.FreezeDomain(ctx, domain); err != nil { logc.Fatalln(ctx, err) } logc.Printf(ctx, "admin: frozen %s\n", domain) } else { if err = backend.UnfreezeDomain(ctx, domain); err != nil { logc.Fatalln(ctx, err) } logc.Printf(ctx, "admin: thawed %s\n", domain) }
case *purgeDomain != "": domain := domainArg(*freezeDomain)
purgeCount := 0 for metadata, err := range backend.EnumerateManifests(ctx) { if err != nil { logc.Fatalln(ctx, err) }
if metadata.Name.Domain == domain { err := backend.DeleteManifest(ctx, metadata.Name, ModifyManifestOptions{}) if err != nil { logc.Fatalln(ctx, err) } purgeCount += 1 } }
logc.Printf(ctx, "admin: purged %s (%d sites)\n", domain, purgeCount)
case *auditLog: records := []*AuditRecord{} ids := backend.SearchAuditLog(ctx, SearchAuditLogOptions{}) for record, err := range backend.GetAuditLogRecords(ctx, ids) { if err != nil { logc.Fatalln(ctx, err) } records = append(records, record) }
slices.SortFunc(records, func(a, b *AuditRecord) int { return cmp.Compare(a.GetAuditID(), b.GetAuditID()) })
for _, record := range records { parts := []string{ record.GetAuditID().String(), color.HiWhiteString("%s", record.GetTimestamp().AsTime().UTC().Format(time.RFC3339)), } if record.GetEvent() == AuditEvent_CorruptedEvent { parts = append(parts, color.HiRedString("%s", fmt.Sprint(record.GetEvent())), ) } else { parts = append(parts, fmt.Sprint(record.GetEvent()), ) } if record.Manifest != nil && record.Manifest.ExpiresAt != nil { parts = append(parts, color.HiYellowString("%s", record.DescribeResource()), ) } else { parts = append(parts, color.HiMagentaString("%s", record.DescribeResource()), ) } parts = append(parts, color.HiGreenString("%s", record.DescribePrincipal()), ) if record.GetReason() != "" { parts = append(parts, color.HiCyanString("%q", record.GetReason()), ) } if record.IsDetached() { parts = append(parts, color.HiYellowString("(detached)"), ) } fmt.Fprintln(color.Output, strings.Join(parts, " ")) }
case *auditRead != "": id, err := ParseAuditID(*auditRead) if err != nil { logc.Fatalln(ctx, err) }
record, err := backend.QueryAuditLog(ctx, id) if err != nil { logc.Fatalln(ctx, err) }
if err = ExtractAuditRecord(ctx, id, record, "."); err != nil { logc.Fatalln(ctx, err) }
case *auditRollback != "": id, err := ParseAuditID(*auditRollback) if err != nil { logc.Fatalln(ctx, err) }
record, err := backend.QueryAuditLog(ctx, id) if err != nil { logc.Fatalln(ctx, err) }
if record.GetManifest() == nil || record.GetDomain() == "" || record.GetProject() == "" { logc.Fatalln(ctx, "no manifest in audit record") }
site := Must(names.ParseSite(fmt.Sprintf("%s/%s", record.GetDomain(), record.GetProject()))) err = backend.StageManifest(ctx, record.GetManifest()) if err != nil { logc.Fatalln(ctx, err) } err = backend.CommitManifest(ctx, site, record.GetManifest(), ModifyManifestOptions{}) if err != nil { logc.Fatalln(ctx, err) }
case *auditDetach != "": domain, project, found := strings.Cut(*auditDetach, "/") if !found || domain == "" || project == "" { logc.Fatalln(ctx, "argument to -audit-detach must be in the form of "+ "'domain.tld/project' or 'domain.tld/*'") }
if project != "*" { if _, err := names.ParseProject(project); err != nil { logc.Fatalf(ctx, "audit detach: project name: %v\n", err) } }
count := 0 ids := backend.SearchAuditLog(ctx, SearchAuditLogOptions{}) for record, err := range backend.GetAuditLogRecords(ctx, ids) { if err != nil { logc.Fatalln(ctx, err) } if record.GetDomain() == domain && (project == "*" || record.GetProject() == project) { if !record.IsDetachable() { continue } else if !record.IsDetached() { logc.Printf(ctx, "detaching audit record %s\n", record.GetAuditID()) err = backend.DetachAuditRecord(ctx, record.GetAuditID()) if err != nil { logc.Fatalln(ctx, err) } count++ } else { logc.Printf(ctx, "audit record %s already detached\n", record.GetAuditID()) } } }
if count == 0 { logc.Printf(ctx, "no detachable audit records found for %s/%s", domain, project) }
case *auditServer != "": if flag.NArg() < 1 { logc.Fatalln(ctx, "handler path not provided") }
processor, err := AuditEventProcessor(flag.Arg(0), flag.Args()[1:]) if err != nil { logc.Fatalln(ctx, err) }
go serve(ctx, listen(ctx, "audit", *auditServer, nil), ObserveHTTPHandler(processor))
sys.WaitForInterrupt() logc.Println(ctx, "audit: exiting")
case *auditExpire != "": days, err := strconv.ParseInt(*auditExpire, 10, 0) if err != nil { logc.Fatalln(ctx, err) }
ids := backend.SearchAuditLog(ctx, SearchAuditLogOptions{ Until: time.Now().AddDate(0, 0, int(-days)), })
count := 0 for id, err := range ids { if err != nil { logc.Fatalln(ctx, err) }
err = backend.ExpireAuditRecord(ctx, id) if err != nil { logc.Fatalln(ctx, err) } else { logc.Printf(ctx, "audit: expired record %s\n", id) count += 1 } }
logc.Printf(ctx, "audit: expired %d records\n", count)
case *expireSites: countExpired, countTransient := 0, 0 for item, err := range backend.GetAllManifests(ctx) { metadata, manifest := item.Splat() if err != nil { logc.Fatalln(ctx, err) } if manifest.ExpiresAt != nil { countTransient += 1 if manifest.ExpiresAt.AsTime().Before(time.Now()) { if !*dryRun { err = backend.ExpireManifest(ctx, metadata.Name) if err != nil { logc.Fatalln(ctx, err) } } logc.Printf(ctx, "expire: site %s expired at %s", metadata.Name, manifest.ExpiresAt.AsTime()) countExpired += 1 } } }
if *dryRun { logc.Printf(ctx, "expire: would expire %d out of %d transient sites (dry run)\n", countExpired, countTransient) } else { logc.Printf(ctx, "expire: expired %d out of %d transient sites\n", countExpired, countTransient) }
case *scrubStorage: if *GetReason(ctx) == "" { *GetReason(ctx) = "-scrub-storage" }
allGood, err := ScrubStorage(ctx, *dryRun) if err != nil { logc.Fatalln(ctx, err) } if !allGood { // Exit code 1 indicates an error during the scrubbing process; exit code 2 indicates // either that repair was successful, or that there is data to repair (if -dry-run). os.Exit(2) }
case *runMigration != "": if err = RunMigration(ctx, *runMigration); err != nil { logc.Fatalln(ctx, err) }
case *analyzeStorage == "text": // datasize.ByteSize.HR() is a little too wide for the 8-char column. formatSize := func(b datasize.ByteSize) string { switch { case b > datasize.GB: return fmt.Sprintf("%.1fG", b.GBytes()) case b > datasize.MB: return fmt.Sprintf("%.1fM", b.MBytes()) case b > datasize.KB: return fmt.Sprintf("%.1fK", b.KBytes()) default: return fmt.Sprintf("%dB", b) } }
analysis, err := AnalyzeStorage(ctx) if err != nil { logc.Fatalln(ctx, err) } slices.SortFunc(analysis, func(a *StorageSize, b *StorageSize) int { return cmp.Compare(a.TotalSize, b.TotalSize) })
for _, sizes := range analysis { var colorize func(string, ...interface{}) string fractionSize := float32(sizes.CurrentBlobSize) / float32(config.Limits.MaxSiteSize.Bytes()) switch { case fractionSize > 0.9: colorize = color.HiRedString case fractionSize > 0.7: colorize = color.HiYellowString case fractionSize > 0.1: colorize = color.HiGreenString default: colorize = color.HiWhiteString } if sizes.Domain != "*" { fmt.Fprintf(color.Output, "%s\t%s\t%s\t%s\t%s\n", colorize("%.0f%%", fractionSize*100.0), colorize("%s", formatSize(datasize.ByteSize(sizes.CurrentSize))), formatSize(datasize.ByteSize(sizes.NonCurrentSize)), formatSize(datasize.ByteSize(sizes.TotalSize)), color.HiMagentaString("%s", sizes.Domain), ) } else { fmt.Fprintf(color.Output, "---\t%s\t%s\t%s\t%s\n", color.HiCyanString("%s", formatSize(datasize.ByteSize(sizes.CurrentSize))), formatSize(datasize.ByteSize(sizes.NonCurrentSize)), formatSize(datasize.ByteSize(sizes.TotalSize)), color.HiMagentaString("%s", sizes.Domain), ) } } fmt.Fprintf(color.Output, "Quota%%\tCurrent\tNonCurr\tTotal\tDomain\n")
case *analyzeStorage == "json": analysis, err := AnalyzeStorage(ctx) if err != nil { logc.Fatalln(ctx, err) }
encoder := json.NewEncoder(os.Stdout) encoder.Encode(analysis)
case *analyzeStorage != "": logc.Fatalf(ctx, "unsupported -analyze-storage mode (must be text or json)")
case *traceGarbage: if err = TraceGarbage(ctx); err != nil { logc.Fatalln(ctx, err) }
default: // Start listening on all ports before initializing the backend, otherwise if the backend // spends some time initializing (which the S3 backend does) a proxy like Caddy can race // with git-pages on startup and return errors for requests that would have been served // just 0.5s later. pagesListener := listen(ctx, "pages", config.Server.Pages, nil) proxyListener := listen(ctx, "caddy", config.Server.Proxy, nil) metricsListener := listen(ctx, "metrics", config.Server.Metrics, nil) httpsListener := net.Listener(nil) if config.Feature("acme") { httpsListener = listen(ctx, "https", config.Server.Https, ACMEConfig(ctx)) }
if config.Feature("acme") { if secrets, err = CreateSecrets(ctx, &config.Secrets); err != nil { logc.Fatalln(ctx, err) } }
if backend, err = CreateBackend(ctx, &config.Storage); err != nil { logc.Fatalln(ctx, err) } backend = NewObservedBackend(backend)
if existenceCache, err = CreateExistenceCache(ctx); err != nil { logc.Fatalln(ctx, err) }
if config.Feature("acme") { // Once dns-persist-01 lands, we should add a `git-pages -acme-dns-persist-01` option // that computes and prints the CAA records for each of the configured wildcard domains. // For now, just print the account URI and the JWK thumbprint. This information can // be used to compute H(length_of_domain || domain_name || key || account_URL) by hand. go func() { if client, err := ACMEClient(ctx); err != nil { logc.Printf(ctx, "acme: %s\n", err) } else { if account, err := client.GetReg(ctx, ""); err != nil { logc.Printf(ctx, "acme: %s\n", err) } else { logc.Printf(ctx, "acme: account %s (%s)\n", account.URI, account.Status) } if thumbprint, err := acme.JWKThumbprint(client.Key.Public()); err != nil { logc.Printf(ctx, "acme: %s\n", err) } else { logc.Printf(ctx, "acme: public key %s\n", thumbprint) } } }() }
middleware := chainHTTPMiddleware( panicHandler, remoteAddrMiddleware, ObserveHTTPHandler, ) go serve(ctx, pagesListener, middleware(http.HandlerFunc(ServePages))) go serve(ctx, httpsListener, middleware(http.HandlerFunc(ServePages))) go serve(ctx, proxyListener, middleware(http.HandlerFunc(ServeProxy))) go serve(ctx, metricsListener, promhttp.Handler())
if config.Insecure { logc.Println(ctx, "serve: ready (INSECURE)") } else { logc.Println(ctx, "serve: ready") }
sys.WaitForInterrupt() logc.Println(ctx, "serve: exiting") }}