diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index f663eba..1d2fdbb 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -32,6 +32,7 @@ import ( "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/database" dnsutils "codeberg.org/codeberg/pages/server/dns" + "codeberg.org/codeberg/pages/server/gitea" "codeberg.org/codeberg/pages/server/upstream" ) @@ -81,7 +82,7 @@ func TLSConfig(mainDomainSuffix []byte, sni = string(sniBytes) } else { _, _ = targetRepo, targetBranch - _, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache) + _, valid := upstream.CheckCanonicalDomain(gitea.NewClient(giteaRoot, giteaAPIToken), targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), canonicalDomainCache) if !valid { sniBytes = mainDomainSuffix sni = string(sniBytes) diff --git a/server/gitea/client.go b/server/gitea/client.go new file mode 100644 index 0000000..5bd1c62 --- /dev/null +++ b/server/gitea/client.go @@ -0,0 +1,81 @@ +package gitea + +import ( + "fmt" + "net/url" + "path" + "time" + + "github.com/valyala/fasthttp" + "github.com/valyala/fastjson" + + "codeberg.org/codeberg/pages/server/shared" +) + +type Client struct { + giteaRoot string + giteaAPIToken string +} + +func NewClient(giteaRoot, giteaAPIToken string) *Client { + return &Client{ + giteaRoot: giteaRoot, + giteaAPIToken: giteaAPIToken, + } +} + +const giteaAPIRepos = "/api/v1/repos/" + +// TODOs: +// * own client to store token & giteaRoot +// * handle 404 -> page will show 500 atm + +func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource string) ([]byte, error) { + req := fasthttp.AcquireRequest() + + req.SetRequestURI(path.Join(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref))) + req.Header.Set(fasthttp.HeaderAuthorization, client.giteaAPIToken) + res := fasthttp.AcquireResponse() + + if err := shared.GetFastHTTPClient(10*time.Second).Do(req, res); err != nil { + return nil, err + } + if res.StatusCode() != fasthttp.StatusOK { + return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode()) + } + return res.Body(), nil +} + +func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchName string) (time.Time, error) { + fastClient := shared.GetFastHTTPClient(5 * time.Second) + + req := fasthttp.AcquireRequest() + req.SetRequestURI(path.Join(client.giteaRoot, giteaAPIRepos, repoOwner, repoName, "branches", branchName)) + req.Header.Set(fasthttp.HeaderAuthorization, client.giteaAPIToken) + res := fasthttp.AcquireResponse() + + if err := fastClient.Do(req, res); err != nil { + return time.Time{}, err + } + if res.StatusCode() != fasthttp.StatusOK { + return time.Time{}, fmt.Errorf("unexpected status code '%d'", res.StatusCode()) + } + return time.Parse(time.RFC3339, fastjson.GetString(res.Body(), "commit", "timestamp")) +} + +func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (string, error) { + fastClient := shared.GetFastHTTPClient(5 * time.Second) + + req := fasthttp.AcquireRequest() + req.SetRequestURI(path.Join(client.giteaRoot, giteaAPIRepos, repoOwner, repoName)) + req.Header.Set(fasthttp.HeaderAuthorization, client.giteaAPIToken) + res := fasthttp.AcquireResponse() + + if err := fastClient.Do(req, res); err != nil { + return "", err + } + if res.StatusCode() != fasthttp.StatusOK { + return "", fmt.Errorf("unexpected status code '%d'", res.StatusCode()) + } + return fastjson.GetString(res.Body(), "default_branch"), nil +} diff --git a/server/handler.go b/server/handler.go index 8445850..2688893 100644 --- a/server/handler.go +++ b/server/handler.go @@ -10,6 +10,7 @@ import ( "codeberg.org/codeberg/pages/html" "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/dns" + "codeberg.org/codeberg/pages/server/gitea" "codeberg.org/codeberg/pages/server/upstream" "codeberg.org/codeberg/pages/server/utils" ) @@ -86,7 +87,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, } // Check if the branch exists, otherwise treat it as a file path - branchTimestampResult := upstream.GetBranchTimestamp(targetOwner, repo, branch, giteaRoot, giteaAPIToken, branchTimestampCache) + branchTimestampResult := upstream.GetBranchTimestamp(giteaClient, targetOwner, repo, branch, branchTimestampCache) if branchTimestampResult == nil { // branch doesn't exist return false @@ -136,9 +137,8 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p", ) { log.Debug().Msg("tryBranch, now trying upstream") - tryUpstream(ctx, mainDomainSuffix, trimmedHost, + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - giteaRoot, giteaAPIToken, canonicalDomainCache, branchTimestampCache, fileResponseCache) return } @@ -152,9 +152,8 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p", ) log.Debug().Msg("tryBranch, now trying upstream") - tryUpstream(ctx, mainDomainSuffix, trimmedHost, + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - giteaRoot, giteaAPIToken, canonicalDomainCache, branchTimestampCache, fileResponseCache) return @@ -187,9 +186,8 @@ func Handler(mainDomainSuffix, rawDomain []byte, "/"+pathElements[0]+"/%p", ) { log.Debug().Msg("tryBranch, now trying upstream") - tryUpstream(ctx, mainDomainSuffix, trimmedHost, + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - giteaRoot, giteaAPIToken, canonicalDomainCache, branchTimestampCache, fileResponseCache) } else { html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) @@ -203,9 +201,8 @@ func Handler(mainDomainSuffix, rawDomain []byte, log.Debug().Msg("main domain preparations, now trying with specified branch") if tryBranch("pages", pathElements[0][1:], pathElements[1:], "/%p") { log.Debug().Msg("tryBranch, now trying upstream") - tryUpstream(ctx, mainDomainSuffix, trimmedHost, + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - giteaRoot, giteaAPIToken, canonicalDomainCache, branchTimestampCache, fileResponseCache) } else { html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) @@ -219,9 +216,8 @@ func Handler(mainDomainSuffix, rawDomain []byte, log.Debug().Msg("main domain preparations, now trying with specified repo") if pathElements[0] != "pages" && tryBranch(pathElements[0], "pages", pathElements[1:], "") { log.Debug().Msg("tryBranch, now trying upstream") - tryUpstream(ctx, mainDomainSuffix, trimmedHost, + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - giteaRoot, giteaAPIToken, canonicalDomainCache, branchTimestampCache, fileResponseCache) return } @@ -231,9 +227,8 @@ func Handler(mainDomainSuffix, rawDomain []byte, log.Debug().Msg("main domain preparations, now trying with default repo/branch") if tryBranch("pages", "", pathElements, "") { log.Debug().Msg("tryBranch, now trying upstream") - tryUpstream(ctx, mainDomainSuffix, trimmedHost, + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - giteaRoot, giteaAPIToken, canonicalDomainCache, branchTimestampCache, fileResponseCache) return } @@ -262,7 +257,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, // Try to use the given repo on the given branch or the default branch log.Debug().Msg("custom domain preparations, now trying with details from DNS") if tryBranch(targetRepo, targetBranch, pathElements, canonicalLink) { - canonicalDomain, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache) + canonicalDomain, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), canonicalDomainCache) if !valid { html.ReturnErrorPage(ctx, fasthttp.StatusMisdirectedRequest) return @@ -279,9 +274,8 @@ func Handler(mainDomainSuffix, rawDomain []byte, } log.Debug().Msg("tryBranch, now trying upstream") - tryUpstream(ctx, mainDomainSuffix, trimmedHost, + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - giteaRoot, giteaAPIToken, canonicalDomainCache, branchTimestampCache, fileResponseCache) return } diff --git a/server/shared/fast_http_client.go b/server/shared/fast_http_client.go new file mode 100644 index 0000000..0ab655e --- /dev/null +++ b/server/shared/fast_http_client.go @@ -0,0 +1,16 @@ +package shared + +import ( + "time" + + "github.com/valyala/fasthttp" +) + +func GetFastHTTPClient(timeout time.Duration) *fasthttp.Client { + return &fasthttp.Client{ + ReadTimeout: timeout, + MaxConnDuration: 60 * time.Second, + MaxConnWaitTimeout: 1000 * time.Millisecond, + MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea! + } +} diff --git a/server/try.go b/server/try.go index 4eda5b2..254d3ec 100644 --- a/server/try.go +++ b/server/try.go @@ -8,22 +8,22 @@ import ( "codeberg.org/codeberg/pages/html" "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/gitea" "codeberg.org/codeberg/pages/server/upstream" ) // tryUpstream forwards the target request to the Gitea API, and shows an error page on failure. -func tryUpstream(ctx *fasthttp.RequestCtx, +func tryUpstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, mainDomainSuffix, trimmedHost []byte, targetOptions *upstream.Options, - targetOwner, targetRepo, targetBranch, targetPath, + targetOwner, targetRepo, targetBranch, targetPath string, - giteaRoot, giteaAPIToken string, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey, ) { // check if a canonical domain exists on a request on MainDomain if bytes.HasSuffix(trimmedHost, mainDomainSuffix) { - canonicalDomain, _ := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache) + canonicalDomain, _ := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), canonicalDomainCache) if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) { canonicalPath := string(ctx.RequestURI()) if targetRepo != "pages" { @@ -43,7 +43,7 @@ func tryUpstream(ctx *fasthttp.RequestCtx, targetOptions.TargetPath = targetPath // Try to request the file from the Gitea API - if !targetOptions.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) { + if !targetOptions.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) { html.ReturnErrorPage(ctx, ctx.Response.StatusCode()) } } diff --git a/server/upstream/domains.go b/server/upstream/domains.go index 040451b..8d06630 100644 --- a/server/upstream/domains.go +++ b/server/upstream/domains.go @@ -4,10 +4,11 @@ import ( "strings" "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/gitea" ) // CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file). -func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaAPIToken string, canonicalDomainCache cache.SetGetKey) (string, bool) { +func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (string, bool) { var ( domains []string valid bool @@ -21,7 +22,7 @@ func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, m } } } else { - body, err := giteaRawContent(targetOwner, targetRepo, targetBranch, giteaRoot, giteaAPIToken, canonicalDomainConfig) + body, err := giteaClient.GiteaRawContent(targetOwner, targetRepo, targetBranch, canonicalDomainConfig) if err == nil { for _, domain := range strings.Split(string(body), "\n") { domain = strings.TrimPrefix( diff --git a/server/upstream/gitea.go b/server/upstream/gitea.go index eeeb0a6..d3f214a 100644 --- a/server/upstream/gitea.go +++ b/server/upstream/gitea.go @@ -8,6 +8,8 @@ import ( "github.com/valyala/fasthttp" "github.com/valyala/fastjson" + + "codeberg.org/codeberg/pages/server/gitea" ) const giteaAPIRepos = "/api/v1/repos/" @@ -16,7 +18,7 @@ const giteaAPIRepos = "/api/v1/repos/" // * own client to store token & giteaRoot // * handle 404 -> page will show 500 atm -func giteaRawContent(targetOwner, targetRepo, ref, giteaRoot, giteaAPIToken, resource string) ([]byte, error) { +func giteaRawContent(client *gitea.Client, targetOwner, targetRepo, ref, giteaRoot, giteaAPIToken, resource string) ([]byte, error) { req := fasthttp.AcquireRequest() req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref))) diff --git a/server/upstream/helper.go b/server/upstream/helper.go index 3b51479..07b0477 100644 --- a/server/upstream/helper.go +++ b/server/upstream/helper.go @@ -4,6 +4,7 @@ import ( "time" "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/gitea" ) type branchTimestamp struct { @@ -13,7 +14,7 @@ type branchTimestamp struct { // GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch // (or nil if the branch doesn't exist) -func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaAPIToken string, branchTimestampCache cache.SetGetKey) *branchTimestamp { +func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string, branchTimestampCache cache.SetGetKey) *branchTimestamp { if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok { if result == nil { return nil @@ -25,7 +26,7 @@ func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaAPIToken string, br } if len(branch) == 0 { // Get default branch - defaultBranch, err := giteaGetRepoDefaultBranch(giteaRoot, owner, repo, giteaAPIToken) + defaultBranch, err := giteaClient.GiteaGetRepoDefaultBranch(owner, repo) if err != nil { _ = branchTimestampCache.Set(owner+"/"+repo+"/", nil, defaultBranchCacheTimeout) return nil @@ -33,7 +34,7 @@ func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaAPIToken string, br result.Branch = defaultBranch } - timestamp, err := giteaGetRepoBranchTimestamp(giteaRoot, owner, repo, branch, giteaAPIToken) + timestamp, err := giteaClient.GiteaGetRepoBranchTimestamp(owner, repo, branch) if err != nil { return nil } diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index 33c80e9..b9c5244 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -15,6 +15,7 @@ import ( "codeberg.org/codeberg/pages/html" "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/gitea" ) // upstreamIndexPages lists pages that may be considered as index pages for directories. @@ -48,7 +49,7 @@ func getFastHTTPClient(timeout time.Duration) *fasthttp.Client { } // Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context. -func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken string, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) { +func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) { log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger() if o.ForbiddenMimeTypes == nil { @@ -57,7 +58,7 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st // Check if the branch exists and when it was modified if o.BranchTimestamp.IsZero() { - branch := GetBranchTimestamp(o.TargetOwner, o.TargetRepo, o.TargetBranch, giteaRoot, giteaAPIToken, branchTimestampCache) + branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch, branchTimestampCache) if branch == nil { html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) @@ -108,7 +109,7 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st optionsForIndexPages.appendTrailingSlash = true for _, indexPage := range upstreamIndexPages { optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage - if optionsForIndexPages.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) { + if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) { _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{ exists: false, }, fileCacheTimeout) @@ -119,7 +120,7 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st optionsForIndexPages.appendTrailingSlash = false optionsForIndexPages.redirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html" optionsForIndexPages.TargetPath = o.TargetPath + ".html" - if optionsForIndexPages.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) { + if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) { _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{ exists: false, }, fileCacheTimeout)