Something went wrong. Try again.
Minimalistic automated (meta)data catalogue assembly and DCAT-AP compliant serving for static data files.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293package main
import ( "context" "io" "mime" "net/http" "slices" "strings"
"github.com/pkg/errors" "github.com/rs/zerolog/log")
// maxRedirects is how many redirects getTtl follows for one upstream URL.// Persistent identifiers (w3id.org, purl.org) and moved hosts redirect, often// to another domain, so cross-domain redirects are allowed.const maxRedirects = 5
// checkRedirect replaces Go's default redirect policy (10 redirects).func checkRedirect(req *http.Request, via []*http.Request) error { if len(via) > maxRedirects { return errors.Errorf("stopped after %d redirects (started at %q)", maxRedirects, via[0].URL) } prev := via[len(via)-1] if prev.URL.Scheme == "https" && req.URL.Scheme != "https" { return errors.Errorf("refusing redirect from https to %q", req.URL) } return nil}
// Accept header sent when fetching upstream metadata.var acceptContentTypes = "text/turtle, application/x-turtle;q=0.9"
// Content types tolerated in an upstream reply. Wider than what we ask for:// some servers label Turtle as plain text or an opaque download.var responseContentTypes = []string{"text/turtle", "application/x-turtle", "text/plain", "application/octet-stream"}
func getTtl(ctx context.Context, url string, httpClient *http.Client) (string, error) { req, err := http.NewRequestWithContext(ctx, "GET", url, nil) if err != nil { return "", errors.Wrapf(err, "construct request for %q", url) } req.Header.Add("Accept", acceptContentTypes)
resp, err := httpClient.Do(req) if err != nil { return "", errors.Wrapf(err, "get %q", url) } // close errors after reading are not actionable defer func() { _ = resp.Body.Close() }()
// resp.Request is the last request in the redirect chain if final := resp.Request.URL.String(); final != url { log.Info().Str("url", url).Str("final_url", final).Msg("upstream redirected") }
if resp.StatusCode >= 400 { return "", errors.Errorf("server returned wrong status (%s) for %q", resp.Status, url) }
contentTypeHeader := resp.Header.Get("Content-Type") if contentTypeHeader == "" { return "", errors.Errorf("server response for %q did not include Content-Type header", url) }
contentType, _, err := mime.ParseMediaType(contentTypeHeader) if err != nil { return "", errors.Wrapf(err, "parse Content-Type %q", contentTypeHeader) }
contains := slices.Contains(responseContentTypes, contentType) if !contains { return "", errors.Errorf("server returned unsupported ContentType: %s", contentType) }
const maxMetadataBytes = 32 << 20 // 32MB graphBytes, err := io.ReadAll(io.LimitReader(resp.Body, maxMetadataBytes+1)) if err != nil { return "", errors.Wrapf(err, "read body of %q", url) } if len(graphBytes) > maxMetadataBytes { return "", errors.Errorf("body of %q exceeds allowed size of %d", url, maxMetadataBytes) }
graphString := string(graphBytes) if strings.TrimSpace(graphString) == "" { return "", errors.Errorf("empty body for %q", url) }
return graphString, nil}