diff --git a/cmd/main.go b/cmd/main.go index baed4f9..9662048 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -18,6 +18,7 @@ import ( "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/certificates" "codeberg.org/codeberg/pages/server/database" + "codeberg.org/codeberg/pages/server/gitea" ) // AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed. @@ -81,9 +82,12 @@ func Serve(ctx *cli.Context) error { // TODO: make this an MRU cache with a size limit fileResponseCache := cache.NewKeyValueCache() + giteaClient := gitea.NewClient(giteaRoot, giteaAPIToken) + // Create handler based on settings handler := server.Handler(mainDomainSuffix, []byte(rawDomain), - giteaRoot, rawInfoPage, giteaAPIToken, + giteaClient, + giteaRoot, rawInfoPage, BlacklistedPaths, allowedCorsDomains, dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache) diff --git a/server/gitea/client.go b/server/gitea/client.go index 5bd1c62..8e245b0 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -4,6 +4,7 @@ import ( "fmt" "net/url" "path" + "strings" "time" "github.com/valyala/fasthttp" @@ -12,11 +13,21 @@ import ( "codeberg.org/codeberg/pages/server/shared" ) +const giteaAPIRepos = "/api/v1/repos/" + type Client struct { giteaRoot string giteaAPIToken string } +type FileResponse struct { + Exists bool + MimeType string + Body []byte +} + +func (f FileResponse) IsEmpty() bool { return len(f.Body) != 0 } + func NewClient(giteaRoot, giteaAPIToken string) *Client { return &Client{ giteaRoot: giteaRoot, @@ -24,8 +35,6 @@ func NewClient(giteaRoot, giteaAPIToken string) *Client { } } -const giteaAPIRepos = "/api/v1/repos/" - // TODOs: // * own client to store token & giteaRoot // * handle 404 -> page will show 500 atm @@ -46,6 +55,19 @@ func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource str return res.Body(), nil } +func (client *Client) ServeRawContent(uri string) (*fasthttp.Response, error) { + fastClient := shared.GetFastHTTPClient(10 * time.Second) + + req := fasthttp.AcquireRequest() + req.SetRequestURI(path.Join(client.giteaRoot, giteaAPIRepos, uri)) + req.Header.Set(fasthttp.HeaderAuthorization, client.giteaAPIToken) + + resp := fasthttp.AcquireResponse() + resp.SetBodyStream(&strings.Reader{}, -1) + + return resp, fastClient.Do(req, resp) +} + func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchName string) (time.Time, error) { fastClient := shared.GetFastHTTPClient(5 * time.Second) diff --git a/server/gitea/fasthttp.go b/server/gitea/fasthttp.go new file mode 100644 index 0000000..018b684 --- /dev/null +++ b/server/gitea/fasthttp.go @@ -0,0 +1,16 @@ +package gitea + +import ( + "time" + + "github.com/valyala/fasthttp" +) + +func getFastHTTPClient(timeout time.Duration) *fasthttp.Client { + return &fasthttp.Client{ + ReadTimeout: timeout, + MaxConnDuration: 60 * time.Second, + MaxConnWaitTimeout: 1000 * time.Millisecond, + MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea! + } +} diff --git a/server/handler.go b/server/handler.go index 2688893..2fb28a0 100644 --- a/server/handler.go +++ b/server/handler.go @@ -17,7 +17,8 @@ import ( // Handler handles a single HTTP request to the web server. func Handler(mainDomainSuffix, rawDomain []byte, - giteaRoot, rawInfoPage, giteaAPIToken string, + giteaClient *gitea.Client, + giteaRoot, rawInfoPage string, blacklistedPaths, allowedCorsDomains [][]byte, dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey, ) func(ctx *fasthttp.RequestCtx) { diff --git a/server/handler_test.go b/server/handler_test.go index 3b4d21a..73002a2 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -8,15 +8,16 @@ import ( "github.com/valyala/fasthttp" "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/gitea" ) func TestHandlerPerformance(t *testing.T) { + giteaRoot := "https://codeberg.org" + giteaClient := gitea.NewClient(giteaRoot, "") testHandler := Handler( - []byte("codeberg.page"), - []byte("raw.codeberg.org"), - "https://codeberg.org", - "https://docs.codeberg.org/pages/raw-content/", - "", + []byte("codeberg.page"), []byte("raw.codeberg.org"), + giteaClient, + giteaRoot, "https://docs.codeberg.org/pages/raw-content/", [][]byte{[]byte("/.well-known/acme-challenge/")}, [][]byte{[]byte("raw.codeberg.org"), []byte("fonts.codeberg.org"), []byte("design.codeberg.org")}, cache.NewKeyValueCache(), diff --git a/server/upstream/gitea.go b/server/upstream/gitea.go deleted file mode 100644 index d3f214a..0000000 --- a/server/upstream/gitea.go +++ /dev/null @@ -1,69 +0,0 @@ -package upstream - -import ( - "fmt" - "net/url" - "path" - "time" - - "github.com/valyala/fasthttp" - "github.com/valyala/fastjson" - - "codeberg.org/codeberg/pages/server/gitea" -) - -const giteaAPIRepos = "/api/v1/repos/" - -// TODOs: -// * own client to store token & giteaRoot -// * handle 404 -> page will show 500 atm - -func giteaRawContent(client *gitea.Client, targetOwner, targetRepo, ref, giteaRoot, giteaAPIToken, resource string) ([]byte, error) { - req := fasthttp.AcquireRequest() - - req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref))) - req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken) - res := fasthttp.AcquireResponse() - - if err := getFastHTTPClient(10*time.Second).Do(req, res); err != nil { - return nil, err - } - if res.StatusCode() != fasthttp.StatusOK { - return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode()) - } - return res.Body(), nil -} - -func giteaGetRepoBranchTimestamp(giteaRoot, repoOwner, repoName, branchName, giteaAPIToken string) (time.Time, error) { - client := getFastHTTPClient(5 * time.Second) - - req := fasthttp.AcquireRequest() - req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, repoOwner, repoName, "branches", branchName)) - req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken) - res := fasthttp.AcquireResponse() - - if err := client.Do(req, res); err != nil { - return time.Time{}, err - } - if res.StatusCode() != fasthttp.StatusOK { - return time.Time{}, fmt.Errorf("unexpected status code '%d'", res.StatusCode()) - } - return time.Parse(time.RFC3339, fastjson.GetString(res.Body(), "commit", "timestamp")) -} - -func giteaGetRepoDefaultBranch(giteaRoot, repoOwner, repoName, giteaAPIToken string) (string, error) { - client := getFastHTTPClient(5 * time.Second) - - req := fasthttp.AcquireRequest() - req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, repoOwner, repoName)) - req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken) - res := fasthttp.AcquireResponse() - - if err := client.Do(req, res); err != nil { - return "", err - } - if res.StatusCode() != fasthttp.StatusOK { - return "", fmt.Errorf("unexpected status code '%d'", res.StatusCode()) - } - return fastjson.GetString(res.Body(), "default_branch"), nil -} diff --git a/server/upstream/helper.go b/server/upstream/helper.go index 07b0477..446483f 100644 --- a/server/upstream/helper.go +++ b/server/upstream/helper.go @@ -1,6 +1,10 @@ package upstream import ( + "mime" + "path" + "strconv" + "strings" "time" "codeberg.org/codeberg/pages/server/cache" @@ -43,8 +47,23 @@ func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string, b return result } -type fileResponse struct { - exists bool - mimeType string - body []byte +func (o *Options) getMimeTypeByExtension() string { + mimeType := mime.TypeByExtension(path.Ext(o.TargetPath)) + mimeTypeSplit := strings.SplitN(mimeType, ";", 2) + if _, ok := o.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" { + if o.DefaultMimeType != "" { + mimeType = o.DefaultMimeType + } else { + mimeType = "application/octet-stream" + } + } + return mimeType +} + +func (o *Options) generateUri() string { + return path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath) +} + +func (o *Options) timestamp() string { + return strconv.FormatInt(o.BranchTimestamp.Unix(), 10) } diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index b9c5244..d184c2c 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -4,9 +4,6 @@ import ( "bytes" "fmt" "io" - "mime" - "path" - "strconv" "strings" "time" @@ -39,15 +36,6 @@ type Options struct { redirectIfExists string } -func getFastHTTPClient(timeout time.Duration) *fasthttp.Client { - return &fasthttp.Client{ - ReadTimeout: timeout, - MaxConnDuration: 60 * time.Second, - MaxConnWaitTimeout: 1000 * time.Millisecond, - MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea! - } -} - // Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context. func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) { log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger() @@ -83,25 +71,19 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, log.Debug().Msg("preparations") // Make a GET request to the upstream URL - uri := path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath) - var req *fasthttp.Request + uri := o.generateUri() var res *fasthttp.Response - var cachedResponse fileResponse + var cachedResponse gitea.FileResponse var err error - if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + strconv.FormatInt(o.BranchTimestamp.Unix(), 10)); ok && len(cachedValue.(fileResponse).body) > 0 { - cachedResponse = cachedValue.(fileResponse) + if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() { + cachedResponse = cachedValue.(gitea.FileResponse) } else { - req = fasthttp.AcquireRequest() - req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, uri)) - req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken) - res = fasthttp.AcquireResponse() - res.SetBodyStream(&strings.Reader{}, -1) - err = getFastHTTPClient(10*time.Second).Do(req, res) + res, err = giteaClient.ServeRawContent(uri) } log.Debug().Msg("acquisition") // Handle errors - if (res == nil && !cachedResponse.exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) { + if (res == nil && !cachedResponse.Exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) { if o.TryIndexPages { // copy the o struct & try if an index page exists optionsForIndexPages := *o @@ -110,8 +92,8 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, for _, indexPage := range upstreamIndexPages { optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) { - _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{ - exists: false, + _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{ + Exists: false, }, fileCacheTimeout) return true } @@ -121,8 +103,8 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, optionsForIndexPages.redirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html" optionsForIndexPages.TargetPath = o.TargetPath + ".html" if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) { - _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{ - exists: false, + _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{ + Exists: false, }, fileCacheTimeout) return true } @@ -130,14 +112,14 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, ctx.Response.SetStatusCode(fasthttp.StatusNotFound) if res != nil { // Update cache if the request is fresh - _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{ - exists: false, + _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{ + Exists: false, }, fileCacheTimeout) } return false } if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) { - fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", req.RequestURI(), err, res.StatusCode()) + fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", uri, err, res.StatusCode()) html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError) return true } @@ -159,15 +141,7 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, log.Debug().Msg("error handling") // Set the MIME type - mimeType := mime.TypeByExtension(path.Ext(o.TargetPath)) - mimeTypeSplit := strings.SplitN(mimeType, ";", 2) - if _, ok := o.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" { - if o.DefaultMimeType != "" { - mimeType = o.DefaultMimeType - } else { - mimeType = "application/octet-stream" - } - } + mimeType := o.getMimeTypeByExtension() ctx.Response.Header.SetContentType(mimeType) // Everything's okay so far @@ -186,20 +160,20 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter)) } } else { - _, err = ctx.Write(cachedResponse.body) + _, err = ctx.Write(cachedResponse.Body) } if err != nil { - fmt.Printf("Couldn't write body for \"%s\": %s\n", req.RequestURI(), err) + fmt.Printf("Couldn't write body for \"%s\": %s\n", uri, err) html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError) return true } log.Debug().Msg("response") if res != nil && ctx.Err() == nil { - cachedResponse.exists = true - cachedResponse.mimeType = mimeType - cachedResponse.body = cacheBodyWriter.Bytes() - _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), cachedResponse, fileCacheTimeout) + cachedResponse.Exists = true + cachedResponse.MimeType = mimeType + cachedResponse.Body = cacheBodyWriter.Bytes() + _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), cachedResponse, fileCacheTimeout) } return true