Something went wrong. Try again.
[mirror] Scalable static site server for Git forges (like GitHub Pages)
Something went wrong. Try again.
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084package git_pages
import ( "bytes" "context" "crypto/sha256" "fmt" "io" "iter" "net/http" "path" "strings" "sync" "time"
"codeberg.org/git-pages/git-pages/src/names" "github.com/c2h5oh/datasize" "github.com/maypok86/otter/v2" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto")
var ( blobsDedupedCount = promauto.NewCounter(prometheus.CounterOpts{ Name: "git_pages_blobs_deduped", Help: "Count of blobs deduplicated", }) blobsDedupedBytes = promauto.NewCounter(prometheus.CounterOpts{ Name: "git_pages_blobs_deduped_bytes", Help: "Total size in bytes of blobs deduplicated", })
blobCacheHitsCount = promauto.NewCounter(prometheus.CounterOpts{ Name: "git_pages_blob_cache_hits_count", Help: "Count of blobs that were retrieved from the cache", }) blobCacheHitsBytes = promauto.NewCounter(prometheus.CounterOpts{ Name: "git_pages_blob_cache_hits_bytes", Help: "Total size in bytes of blobs that were retrieved from the cache", }) blobCacheMissesCount = promauto.NewCounter(prometheus.CounterOpts{ Name: "git_pages_blob_cache_misses_count", Help: "Count of blobs that were not found in the cache (and were then successfully cached)", }) blobCacheMissesBytes = promauto.NewCounter(prometheus.CounterOpts{ Name: "git_pages_blob_cache_misses_bytes", Help: "Total size in bytes of blobs that were not found in the cache (and were then successfully cached)", }) blobCacheEvictionsCount = promauto.NewCounter(prometheus.CounterOpts{ Name: "git_pages_blob_cache_evictions_count", Help: "Count of blobs evicted from the cache", }) blobCacheEvictionsBytes = promauto.NewCounter(prometheus.CounterOpts{ Name: "git_pages_blob_cache_evictions_bytes", Help: "Total size in bytes of blobs evicted from the cache", })
manifestCacheHitsCount = promauto.NewCounter(prometheus.CounterOpts{ Name: "git_pages_manifest_cache_hits_count", Help: "Count of manifests that were retrieved from the cache", }) manifestCacheMissesCount = promauto.NewCounter(prometheus.CounterOpts{ Name: "git_pages_manifest_cache_misses_count", Help: "Count of manifests that were not found in the cache (and were then successfully cached)", }) manifestCacheEvictionsCount = promauto.NewCounter(prometheus.CounterOpts{ Name: "git_pages_manifest_cache_evictions_count", Help: "Count of manifests evicted from the cache", })
secretOperationCount = *promauto.NewCounterVec(prometheus.CounterOpts{ Name: "git_pages_secret_operation_count", Help: "Count of operations on secrets", }, []string{"kind", "oper"})
s3GetObjectDurationSeconds = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "git_pages_s3_get_object_duration_seconds", Help: "Time to read a whole object from S3", Buckets: []float64{.01, .025, .05, .1, .25, .5, .75, 1, 1.25, 1.5, 1.75, 2, 2.5, 5, 10},
NativeHistogramBucketFactor: 1.1, NativeHistogramMaxBucketNumber: 100, NativeHistogramMinResetDuration: 10 * time.Minute, }, []string{"kind"}) s3GetObjectResponseCount = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "git_pages_s3_get_object_responses_count", Help: "Count of s3:GetObject responses", }, []string{"kind", "code"}))
// Blobs can be safely cached indefinitely. They only need to be evicted to preserve memory.type CachedBlob struct { blob []byte mtime time.Time}
func (c *CachedBlob) Weight() uint32 { return uint32(len(c.blob)) }
// Manifests can only be cached for a short time to avoid serving stale content. Browser// page loads cause a large burst of manifest accesses that are essential for serving// `304 No Content` responses and these need to be handled very quickly, so both hits and// misses are cached.type CachedManifest struct { manifest *Manifest weight uint32 metadata ManifestMetadata err error}
func (c *CachedManifest) Weight() uint32 { return c.weight }
type S3Backend struct { client *minio.Client bucket string blobCache *observedCache[string, *CachedBlob] siteCache *observedCache[names.Site, *CachedManifest] featureCache *otter.Cache[BackendFeature, bool]}
var _ Backend = (*S3Backend)(nil)
func makeCacheOptions[K comparable, V any]( config *CacheConfig, weigher func(K, V) uint32,) *otter.Options[K, V] { options := &otter.Options[K, V]{} if config.MaxSize != 0 { options.MaximumWeight = config.MaxSize.Bytes() options.Weigher = weigher } if config.MaxStale != 0 { options.RefreshCalculator = otter.RefreshWriting[K, V]( time.Duration(config.MaxAge)) } if config.MaxAge != 0 || config.MaxStale != 0 { options.ExpiryCalculator = otter.ExpiryWriting[K, V]( time.Duration(config.MaxAge + config.MaxStale)) } return options}
func makeClientForBucket( ctx context.Context, config *S3Config, isStorage bool,) (client *minio.Client, err error) { var bucketLookup minio.BucketLookupType
switch config.BucketLookup { case "auto": bucketLookup = minio.BucketLookupAuto case "path": bucketLookup = minio.BucketLookupPath case "dns": bucketLookup = minio.BucketLookupDNS default: return nil, fmt.Errorf("unknown bucket lookup type: %s", config.BucketLookup) }
client, err = minio.New(config.Endpoint, &minio.Options{ Creds: credentials.NewStaticV4( config.AccessKeyID, config.SecretAccessKey, "", ), Secure: !config.Insecure, BucketLookup: bucketLookup, }) if err != nil { return nil, err }
exists, err := client.BucketExists(ctx, config.Bucket) if err != nil { return nil, err } else if !exists { logc.Printf(ctx, "s3: create bucket %s\n", config.Bucket)
err = client.MakeBucket(ctx, config.Bucket, minio.MakeBucketOptions{Region: config.Region}) if err != nil { return nil, err }
if isStorage { err = (&S3Backend{client: client, bucket: config.Bucket}). EnableFeature(ctx, FeatureCheckDomainMarker) if err != nil { return nil, err } } }
return client, nil}
func NewS3Backend(ctx context.Context, config *S3StorageConfig) (*S3Backend, error) { client, err := makeClientForBucket(ctx, &config.S3Config, true) if err != nil { return nil, err }
blobCacheMetrics := observedCacheMetrics{ HitNumberCounter: blobCacheHitsCount, HitWeightCounter: blobCacheHitsBytes, MissNumberCounter: blobCacheMissesCount, MissWeightCounter: blobCacheMissesBytes, EvictionNumberCounter: blobCacheEvictionsCount, EvictionWeightCounter: blobCacheEvictionsBytes, } blobCache, err := newObservedCache(makeCacheOptions(&config.BlobCache, func(key string, value *CachedBlob) uint32 { return uint32(len(value.blob)) }), blobCacheMetrics) if err != nil { return nil, err }
siteCacheMetrics := observedCacheMetrics{ HitNumberCounter: manifestCacheHitsCount, MissNumberCounter: manifestCacheMissesCount, EvictionNumberCounter: manifestCacheEvictionsCount, } siteCache, err := newObservedCache(makeCacheOptions(&config.SiteCache, func(key names.Site, value *CachedManifest) uint32 { return value.weight }), siteCacheMetrics) if err != nil { return nil, err }
featureCache, err := otter.New(&otter.Options[BackendFeature, bool]{ RefreshCalculator: otter.RefreshWriting[BackendFeature, bool](10 * time.Minute), }) if err != nil { return nil, err }
return &S3Backend{client, config.Bucket, blobCache, siteCache, featureCache}, nil}
func (s3 *S3Backend) Backend() Backend { return s3}
func blobObjectName(name string) string { return fmt.Sprintf("blob/%s", path.Join(splitBlobName(name)...))}
func storeFeatureObjectName(feature BackendFeature) string { return fmt.Sprintf("meta/feature/%s", feature)}
func (s3 *S3Backend) HasFeature(ctx context.Context, feature BackendFeature) bool { loader := func(ctx context.Context, feature BackendFeature) (bool, error) { _, err := s3.client.StatObject(ctx, s3.bucket, storeFeatureObjectName(feature), minio.StatObjectOptions{}) if err != nil { if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" { logc.Printf(ctx, "s3 feature %q: disabled", feature) return false, nil } else { return false, err } } logc.Printf(ctx, "s3 feature %q: enabled", feature) return true, nil }
isOn, err := s3.featureCache.Get(ctx, feature, otter.LoaderFunc[BackendFeature, bool](loader)) if err != nil { err = fmt.Errorf("getting s3 backend feature %q: %w", feature, err) ObserveError(err) logc.Println(ctx, err) return false } return isOn}
func (s3 *S3Backend) EnableFeature(ctx context.Context, feature BackendFeature) error { _, err := s3.client.PutObject(ctx, s3.bucket, storeFeatureObjectName(feature), &bytes.Reader{}, 0, minio.PutObjectOptions{}) return err}
func (s3 *S3Backend) GetBlob( ctx context.Context, name string,) ( reader io.ReadSeeker, metadata BlobMetadata, err error,) { loader := func(ctx context.Context, name string) (*CachedBlob, error) { logc.Printf(ctx, "s3: get blob %s\n", name)
startTime := time.Now()
object, err := s3.client.GetObject(ctx, s3.bucket, blobObjectName(name), minio.GetObjectOptions{}) // Note that many errors (e.g. NoSuchKey) will be reported only after this point. if err != nil { return nil, err } defer object.Close()
data, err := io.ReadAll(object) if err != nil { return nil, err }
stat, err := object.Stat() if err != nil { return nil, err }
s3GetObjectDurationSeconds. With(prometheus.Labels{"kind": "blob"}). Observe(time.Since(startTime).Seconds())
return &CachedBlob{data, stat.LastModified}, nil }
observer := func(ctx context.Context, name string) (*CachedBlob, error) { cached, err := loader(ctx, name) code := "OK" if errResp, ok := err.(minio.ErrorResponse); ok { code = errResp.Code } s3GetObjectResponseCount.With(prometheus.Labels{"kind": "blob", "code": code}).Inc() return cached, err }
var cached *CachedBlob cached, err = s3.blobCache.Get(ctx, name, otter.LoaderFunc[string, *CachedBlob](observer)) if err != nil { if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" { err = fmt.Errorf("%w: %s", ErrObjectNotFound, errResp.Key) } } else { reader = bytes.NewReader(cached.blob) metadata.Name = name metadata.Size = int64(len(cached.blob)) metadata.LastModified = cached.mtime } return}
func (s3 *S3Backend) PutBlob(ctx context.Context, name string, data []byte) error { logc.Printf(ctx, "s3: put blob %s (%s)\n", name, datasize.ByteSize(len(data)).HumanReadable())
_, err := s3.client.StatObject(ctx, s3.bucket, blobObjectName(name), minio.GetObjectOptions{}) if err != nil { if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" { _, err := s3.client.PutObject(ctx, s3.bucket, blobObjectName(name), bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{}) if err != nil { return err } else { ObserveData(ctx, "blob.status", "created") logc.Printf(ctx, "s3: put blob %s (created)\n", name) return nil } } else { return err } } else { ObserveData(ctx, "blob.status", "exists") logc.Printf(ctx, "s3: put blob %s (exists)\n", name) blobsDedupedCount.Inc() blobsDedupedBytes.Add(float64(len(data))) return nil }}
func (s3 *S3Backend) DeleteBlob(ctx context.Context, name string) error { logc.Printf(ctx, "s3: delete blob %s\n", name)
return s3.client.RemoveObject(ctx, s3.bucket, blobObjectName(name), minio.RemoveObjectOptions{})}
func (s3 *S3Backend) EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata, error] { return func(yield func(BlobMetadata, error) bool) { logc.Println(ctx, "s3: enumerate blobs")
ctx, cancel := context.WithCancel(ctx) defer cancel()
prefix := "blob/" for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{ Prefix: prefix, Recursive: true, }) { var metadata BlobMetadata var err error if err = object.Err; err == nil { key := strings.TrimPrefix(object.Key, prefix) if strings.HasSuffix(key, "/") { continue // directory; skip } else { metadata.Name = joinBlobName(strings.Split(key, "/")) metadata.Size = object.Size metadata.LastModified = object.LastModified } } if !yield(metadata, err) { break } } }}
func manifestObjectName(object string) string { return fmt.Sprintf("site/%s", object)}
func stagedManifestObjectName(manifestData []byte) string { return fmt.Sprintf("dirty/%x", sha256.Sum256(manifestData))}
type s3ManifestLoader struct { s3 *S3Backend}
func (l s3ManifestLoader) Load( ctx context.Context, key names.Site,) ( *CachedManifest, error,) { return l.load(ctx, key, nil)}
func (l s3ManifestLoader) Reload( ctx context.Context, key names.Site, oldValue *CachedManifest,) ( *CachedManifest, error,) { return l.load(ctx, key, oldValue)}
func (l s3ManifestLoader) load( ctx context.Context, site names.Site, oldManifest *CachedManifest,) ( *CachedManifest, error,) { logc.Printf(ctx, "s3: get manifest %s\n", site)
loader := func() (*CachedManifest, error) { opts := minio.GetObjectOptions{} if oldManifest != nil && oldManifest.metadata.ETag != "" { opts.SetMatchETagExcept(oldManifest.metadata.ETag) } object, err := l.s3.client.GetObject(ctx, l.s3.bucket, manifestObjectName(site.String()), opts) // Note that many errors (e.g. NoSuchKey) will be reported only after this point. if err != nil { return nil, err } defer object.Close()
data, err := io.ReadAll(object) if err != nil { return nil, err }
stat, err := object.Stat() if err != nil { return nil, err }
manifest, err := DecodeManifest(data) if err != nil { err = fmt.Errorf("decode %s: %w", site, err) return nil, err }
metadata := ManifestMetadata{ LastModified: stat.LastModified, ETag: stat.ETag, } return &CachedManifest{manifest, uint32(len(data)), metadata, nil}, nil }
observer := func() (*CachedManifest, error) { cached, err := loader() code := "OK" if errResp, ok := err.(minio.ErrorResponse); ok { code = errResp.Code } s3GetObjectResponseCount.With(prometheus.Labels{"kind": "manifest", "code": code}).Inc() return cached, err }
startTime := time.Now() cached, err := observer() s3GetObjectDurationSeconds. With(prometheus.Labels{"kind": "manifest"}). Observe(time.Since(startTime).Seconds())
if err != nil { errResp := minio.ToErrorResponse(err) if errResp.Code == "NoSuchKey" { err = fmt.Errorf("%w: %s", ErrObjectNotFound, errResp.Key) return &CachedManifest{nil, 1, ManifestMetadata{}, err}, nil } else if errResp.StatusCode == http.StatusNotModified && oldManifest != nil { return oldManifest, nil } else { return nil, err } } else { return cached, nil }}
func (s3 *S3Backend) GetManifest( ctx context.Context, site names.Site, opts GetManifestOptions,) ( manifest *Manifest, metadata ManifestMetadata, err error,) { if opts.BypassCache { entry, found := s3.siteCache.Cache.GetEntry(site) if found && entry.RefreshableAt().Before(time.Now()) { s3.siteCache.Cache.Invalidate(site) } }
var cached *CachedManifest cached, err = s3.siteCache.Get(ctx, site, s3ManifestLoader{s3}) if err != nil { return } else { // This could be `manifest, mtime, nil` or `nil, time.Time{}, ErrObjectNotFound`. manifest, metadata, err = cached.manifest, cached.metadata, cached.err return }}
func (s3 *S3Backend) StageManifest(ctx context.Context, manifest *Manifest) error { data := EncodeManifest(manifest) logc.Printf(ctx, "s3: stage manifest %x\n", sha256.Sum256(data))
_, err := s3.client.PutObject(ctx, s3.bucket, stagedManifestObjectName(data), bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{}) return err}
func domainFrozenObjectName(domain names.Domain) string { return manifestObjectName(fmt.Sprintf("%s/.frozen", domain.String()))}
func (s3 *S3Backend) checkDomainFrozen(ctx context.Context, domain names.Domain) error { _, err := s3.client.StatObject(ctx, s3.bucket, domainFrozenObjectName(domain), minio.GetObjectOptions{}) if err == nil { return ErrDomainFrozen } else if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" { return nil } else { return err }}
func (s3 *S3Backend) HasAtomicCAS(ctx context.Context) bool { // Support for `If-Unmodified-Since:` or `If-Match:` for PutObject requests is very spotty: // - AWS supports only `If-Match:`: // https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html // - Minio supports `If-Match:`: // https://blog.min.io/leading-the-way-minios-conditional-write-feature-for-modern-data-workloads/ // - Tigris supports `If-Unmodified-Since:` and `If-Match:`, but only with `X-Tigris-Consistent: true`; // https://www.tigrisdata.com/docs/objects/conditionals/ // Note that the `X-Tigris-Consistent: true` header must be present on *every* transaction // touching the object, not just on the CAS transactions. // - Wasabi does not support either one and docs seem to suggest that the headers are ignored; // - Garage does not support either one and source code suggests the headers are ignored. // It seems that the only safe option is to not claim support for atomic CAS, and only do // best-effort CAS implementation using HeadObject and PutObject/DeleteObject. return false}
func (s3 *S3Backend) checkManifestPrecondition( ctx context.Context, site names.Site, opts ModifyManifestOptions,) (exists bool, err error) { stat, err := s3.client.StatObject(ctx, s3.bucket, manifestObjectName(site.String()), minio.GetObjectOptions{}) if err != nil { errResp := minio.ToErrorResponse(err) if opts.IfUnmodifiedSince.IsZero() && opts.IfMatch == "" && errResp.Code == "NoSuchKey" { exists = false } else { return false, err } } else { exists = true }
if !opts.IfUnmodifiedSince.IsZero() && stat.LastModified.Compare(opts.IfUnmodifiedSince) > 0 { return exists, fmt.Errorf("%w: If-Unmodified-Since", ErrPreconditionFailed) } if opts.IfMatch != "" && stat.ETag != opts.IfMatch { return exists, fmt.Errorf("%w: If-Match", ErrPreconditionFailed) }
return exists, nil}
func (s3 *S3Backend) CommitManifest( ctx context.Context, site names.Site, manifest *Manifest, opts ModifyManifestOptions,) error { data := EncodeManifest(manifest) logc.Printf(ctx, "s3: commit manifest %x -> %s", sha256.Sum256(data), site.String())
if err := s3.checkDomainFrozen(ctx, site.Domain); err != nil { return err }
existed, err := s3.checkManifestPrecondition(ctx, site, opts) if err != nil { return err }
// Remove staged object unconditionally (whether commit succeeded or failed), since // the upper layer has to retry the complete operation anyway. putOptions := minio.PutObjectOptions{} putOptions.Header().Add("X-Tigris-Consistent", "true") if opts.IfMatch != "" { // Not guaranteed to do anything (see `HasAtomicCAS`), but let's try anyway; // this is a "belt and suspenders" approach, together with `checkManifestPrecondition`. // It does reliably work on MinIO at least. putOptions.SetMatchETag(opts.IfMatch) } _, putErr := s3.client.PutObject(ctx, s3.bucket, manifestObjectName(site.String()), bytes.NewReader(data), int64(len(data)), putOptions) removeErr := s3.client.RemoveObject(ctx, s3.bucket, stagedManifestObjectName(data), minio.RemoveObjectOptions{}) s3.siteCache.Cache.Invalidate(site) if putErr != nil { if errResp := minio.ToErrorResponse(putErr); errResp.Code == "PreconditionFailed" { return ErrPreconditionFailed } else { return putErr } } else if removeErr != nil { return removeErr }
if !existed { if err := s3.bumpLastSiteUpdateTimestamp(ctx); err != nil { return err } }
return nil}
func (s3 *S3Backend) DeleteManifest( ctx context.Context, site names.Site, opts ModifyManifestOptions,) error { logc.Printf(ctx, "s3: delete manifest %s\n", site)
if err := s3.checkDomainFrozen(ctx, site.Domain); err != nil { return err }
existed, err := s3.checkManifestPrecondition(ctx, site, opts) if err != nil { return err }
err = s3.client.RemoveObject(ctx, s3.bucket, manifestObjectName(site.String()), minio.RemoveObjectOptions{}) if err != nil { return err } s3.siteCache.Cache.Invalidate(site)
if existed { if err := s3.bumpLastSiteUpdateTimestamp(ctx); err != nil { return err } }
return nil}
func (s3 *S3Backend) ExpireManifest( ctx context.Context, site names.Site,) error { return s3.DeleteManifest(ctx, site, ModifyManifestOptions{})}
func (s3 *S3Backend) EnumerateManifests(ctx context.Context) iter.Seq2[*ManifestMetadata, error] { return func(yield func(*ManifestMetadata, error) bool) { logc.Println(ctx, "s3: enumerate manifests")
ctx, cancel := context.WithCancel(ctx) defer cancel()
prefix := "site/" for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{ Prefix: prefix, Recursive: true, }) { var metadata *ManifestMetadata var err error if err = object.Err; err == nil { key := strings.TrimPrefix(object.Key, prefix) if strings.HasSuffix(key, "/") { continue // directory; skip } else if site, err := names.ParseSite(key); err != nil { continue // not a site (marker, etc); skip } else { metadata = &ManifestMetadata{ Name: site, Size: object.Size, LastModified: object.LastModified, ETag: object.ETag, } } } if !yield(metadata, err) { break } } }}
// Limits the number of concurrent uploads, globally across the entire git-pages process.// Not currently configurable as there seems to be little need.var getAllManifestsSemaphore = make(chan struct{}, 64)
func (s3 *S3Backend) GetAllManifests(ctx context.Context) iter.Seq2[tuple[*ManifestMetadata, *Manifest], error] { type result struct { metadata *ManifestMetadata manifest *Manifest err error }
resultsChan := make(chan result) enumeratorCtx, cancel := context.WithCancel(ctx) go func(ctx context.Context) { wg := sync.WaitGroup{} for metadata, err := range s3.EnumerateManifests(ctx) { if err != nil { resultsChan <- result{nil, nil, err} } else { getAllManifestsSemaphore <- struct{}{} // acquire wg.Go(func() { defer func() { <-getAllManifestsSemaphore }() // release manifest, _, err := backend.GetManifest(ctx, metadata.Name, GetManifestOptions{}) resultsChan <- result{metadata, manifest, err} }) } } wg.Wait() close(resultsChan) }(enumeratorCtx)
return func(yield func(tuple[*ManifestMetadata, *Manifest], error) bool) { for result := range resultsChan { item := tuple[*ManifestMetadata, *Manifest]{result.metadata, result.manifest} if !yield(item, result.err) { cancel() break } } }}
func domainCheckObjectName(domain names.Domain) string { return manifestObjectName(fmt.Sprintf("%s/.exists", domain))}
func (s3 *S3Backend) CheckDomain(ctx context.Context, domain names.Domain) (exists bool, err error) { logc.Printf(ctx, "s3: check domain %s\n", domain)
_, err = s3.client.StatObject(ctx, s3.bucket, domainCheckObjectName(domain), minio.StatObjectOptions{}) if err != nil { if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" { exists, err = false, nil } } else { exists = true }
if !exists && !s3.HasFeature(ctx, FeatureCheckDomainMarker) { ctx, cancel := context.WithCancel(ctx) defer cancel()
for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{ Prefix: manifestObjectName(fmt.Sprintf("%s/", domain)), }) { if object.Err != nil { return false, object.Err } return true, nil } return false, nil }
return}
func (s3 *S3Backend) CreateDomain(ctx context.Context, domain names.Domain) error { logc.Printf(ctx, "s3: create domain %s\n", domain)
_, err := s3.client.PutObject(ctx, s3.bucket, domainCheckObjectName(domain), &bytes.Reader{}, 0, minio.PutObjectOptions{}) return err}
func (s3 *S3Backend) FreezeDomain(ctx context.Context, domain names.Domain) error { logc.Printf(ctx, "s3: freeze domain %s\n", domain)
_, err := s3.client.PutObject(ctx, s3.bucket, domainFrozenObjectName(domain), &bytes.Reader{}, 0, minio.PutObjectOptions{}) return err}
func (s3 *S3Backend) UnfreezeDomain(ctx context.Context, domain names.Domain) error { logc.Printf(ctx, "s3: unfreeze domain %s\n", domain)
err := s3.client.RemoveObject(ctx, s3.bucket, domainFrozenObjectName(domain), minio.RemoveObjectOptions{}) if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" { return nil } else { return err }}
const lastSiteUpdateObjectName = "meta/last-site-update"
func (s3 *S3Backend) HasSiteListChanged(ctx context.Context, since time.Time) (bool, time.Time, error) { opts := minio.GetObjectOptions{} if !since.IsZero() { if err := opts.SetModified(since); err != nil { return false, time.Time{}, err } }
info, err := s3.client.StatObject(ctx, s3.bucket, lastSiteUpdateObjectName, opts) if err != nil { if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" { // The marker has not been written yet, meaning no mutations have occurred. return false, time.Time{}, nil } else if errResp.StatusCode == http.StatusNotModified { // The marker has not changed. return false, time.Time{}, nil }
return false, time.Time{}, err }
return true, info.LastModified, nil}
func (s3 *S3Backend) bumpLastSiteUpdateTimestamp(ctx context.Context) error { logc.Println(ctx, "s3: bumping last site update timestamp") _, err := s3.client.PutObject(ctx, s3.bucket, lastSiteUpdateObjectName, &bytes.Reader{}, 0, minio.PutObjectOptions{}) return err}
func auditObjectName(id AuditID) string { return fmt.Sprintf("audit/%s", id)}
func auditDetachedObjectName(id AuditID) string { return fmt.Sprintf("audit/%s.detached", id)}
func (s3 *S3Backend) AppendAuditLog(ctx context.Context, id AuditID, record *AuditRecord) error { logc.Printf(ctx, "s3: append audit %s\n", id)
name := auditObjectName(id) data := EncodeAuditRecord(record)
options := minio.PutObjectOptions{} options.SetMatchETagExcept("*") // may or may not be supported _, err := s3.client.PutObject(ctx, s3.bucket, name, bytes.NewReader(data), int64(len(data)), options) if errResp := minio.ToErrorResponse(err); errResp.StatusCode == 412 { panic(fmt.Errorf("audit ID collision: %s", name)) } return err}
func (s3 *S3Backend) QueryAuditLog(ctx context.Context, id AuditID) (*AuditRecord, error) { logc.Printf(ctx, "s3: read audit %s\n", id)
object, err := s3.client.GetObject(ctx, s3.bucket, auditObjectName(id), minio.GetObjectOptions{}) if err != nil { return nil, err } defer object.Close()
data, err := io.ReadAll(object) if err != nil { return nil, err }
record, err := DecodeAuditRecord(data) if err != nil { err = fmt.Errorf("decode %s: %w", id, err) return nil, err }
_, err = s3.client.StatObject(ctx, s3.bucket, auditDetachedObjectName(id), minio.StatObjectOptions{}) if err == nil { record.Manifest = nil } else if errResp := minio.ToErrorResponse(err); errResp.Code != "NoSuchKey" { return nil, err }
return record, nil}
func (s3 *S3Backend) SearchAuditLog( ctx context.Context, opts SearchAuditLogOptions,) iter.Seq2[AuditID, error] { return func(yield func(AuditID, error) bool) { logc.Printf(ctx, "s3: search audit\n")
ctx, cancel := context.WithCancel(ctx) defer cancel()
prefix := "audit/" for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{ Prefix: prefix, }) { var id AuditID var err error if object.Err != nil { err = object.Err } else if strings.Contains(object.Key, ".") { continue } else if id, err = ParseAuditID(strings.TrimPrefix(object.Key, prefix)); err != nil { // report error } else if !opts.Since.IsZero() && id.CompareTime(opts.Since) < 0 { continue } else if !opts.Until.IsZero() && id.CompareTime(opts.Until) > 0 { continue } if !yield(id, err) { break } } }}
var getAuditLogRecordsSemaphore = make(chan struct{}, 64)
func (s3 *S3Backend) GetAuditLogRecords( ctx context.Context, ids iter.Seq2[AuditID, error],) iter.Seq2[*AuditRecord, error] { return func(yield func(*AuditRecord, error) bool) { resultsChan := make(chan tuple[*AuditRecord, error]) enumeratorCtx, cancel := context.WithCancel(ctx) defer cancel()
go func(ctx context.Context) { wg := sync.WaitGroup{} for id, err := range ids { if err != nil { resultsChan <- tuple[*AuditRecord, error]{nil, err} } else { getAuditLogRecordsSemaphore <- struct{}{} // acquire wg.Go(func() { defer func() { <-getAuditLogRecordsSemaphore }() // release record, err := s3.QueryAuditLog(ctx, id) resultsChan <- tuple[*AuditRecord, error]{record, err} }) } } wg.Wait() close(resultsChan) }(enumeratorCtx)
for result := range resultsChan { record, err := result.Splat() if !yield(record, err) { break } } }}
func (s3 *S3Backend) DetachAuditRecord(ctx context.Context, id AuditID) error { logc.Printf(ctx, "s3: detach audit record %s\n", id)
_, err := s3.client.PutObject(ctx, s3.bucket, auditDetachedObjectName(id), &bytes.Reader{}, 0, minio.PutObjectOptions{}) return err}
func (s3 *S3Backend) ExpireAuditRecord(ctx context.Context, id AuditID) error { logc.Printf(ctx, "s3: expire audit record %s\n", id)
return s3.client.RemoveObject(ctx, s3.bucket, auditObjectName(id), minio.RemoveObjectOptions{})}
type S3Secrets struct { client *minio.Client bucket string}
var _ Secrets = (*S3Secrets)(nil)
func NewS3Secrets(ctx context.Context, config *S3Config) (*S3Secrets, error) { client, err := makeClientForBucket(ctx, config, false) if err != nil { return nil, err } return &S3Secrets{client, config.Bucket}, nil}
func (s3 *S3Secrets) Secrets() Secrets { return s3}
func acmeObjectName(name string) string { return fmt.Sprintf("acme/%s", name)}
func (s3 *S3Secrets) GetACME( ctx context.Context, name string,) ( data []byte, err error,) { logc.Printf(ctx, "s3: get acme %s\n", name)
secretOperationCount.With(prometheus.Labels{"kind": "acme", "oper": "get"}).Inc()
loader := func(ctx context.Context, name string) ([]byte, error) { object, err := s3.client.GetObject(ctx, s3.bucket, acmeObjectName(name), minio.GetObjectOptions{}) // Note that many errors (e.g. NoSuchKey) will be reported only after this point. if err != nil { return nil, err } defer object.Close()
return io.ReadAll(object) }
startTime := time.Now() data, err = loader(ctx, name) s3GetObjectDurationSeconds. With(prometheus.Labels{"kind": "acme"}). Observe(time.Since(startTime).Seconds())
code := "OK" if errResp, ok := err.(minio.ErrorResponse); ok { code = errResp.Code } s3GetObjectResponseCount.With(prometheus.Labels{"kind": "acme", "code": code}).Inc()
if err != nil { if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" { err = fmt.Errorf("%w: %s", ErrObjectNotFound, errResp.Key) } } return}
func (s3 *S3Secrets) PutACME(ctx context.Context, name string, data []byte) error { logc.Printf(ctx, "s3: put acme %s\n", name)
secretOperationCount.With(prometheus.Labels{"kind": "acme", "oper": "put"}).Inc()
_, err := s3.client.PutObject(ctx, s3.bucket, acmeObjectName(name), bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{}) return err}
func (s3 *S3Secrets) DeleteACME(ctx context.Context, name string) error { logc.Printf(ctx, "s3: delete acme %s\n", name)
secretOperationCount.With(prometheus.Labels{"kind": "acme", "oper": "delete"}).Inc()
return s3.client.RemoveObject(ctx, s3.bucket, acmeObjectName(name), minio.RemoveObjectOptions{})}