start moving

This commit is contained in:
6543 2022-05-15 23:12:13 +02:00
parent 176d4b86c2
commit 4f2c94c7d9
No known key found for this signature in database
GPG key ID: C99B82E40B027BAE
9 changed files with 129 additions and 32 deletions

View file

@ -32,6 +32,7 @@ import (
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/database"
dnsutils "codeberg.org/codeberg/pages/server/dns"
"codeberg.org/codeberg/pages/server/gitea"
"codeberg.org/codeberg/pages/server/upstream"
)
@ -81,7 +82,7 @@ func TLSConfig(mainDomainSuffix []byte,
sni = string(sniBytes)
} else {
_, _ = targetRepo, targetBranch
_, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)
_, valid := upstream.CheckCanonicalDomain(gitea.NewClient(giteaRoot, giteaAPIToken), targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), canonicalDomainCache)
if !valid {
sniBytes = mainDomainSuffix
sni = string(sniBytes)

81
server/gitea/client.go Normal file
View file

@ -0,0 +1,81 @@
package gitea
import (
"fmt"
"net/url"
"path"
"time"
"github.com/valyala/fasthttp"
"github.com/valyala/fastjson"
"codeberg.org/codeberg/pages/server/shared"
)
type Client struct {
giteaRoot string
giteaAPIToken string
}
func NewClient(giteaRoot, giteaAPIToken string) *Client {
return &Client{
giteaRoot: giteaRoot,
giteaAPIToken: giteaAPIToken,
}
}
const giteaAPIRepos = "/api/v1/repos/"
// TODOs:
// * own client to store token & giteaRoot
// * handle 404 -> page will show 500 atm
func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource string) ([]byte, error) {
req := fasthttp.AcquireRequest()
req.SetRequestURI(path.Join(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref)))
req.Header.Set(fasthttp.HeaderAuthorization, client.giteaAPIToken)
res := fasthttp.AcquireResponse()
if err := shared.GetFastHTTPClient(10*time.Second).Do(req, res); err != nil {
return nil, err
}
if res.StatusCode() != fasthttp.StatusOK {
return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
}
return res.Body(), nil
}
func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchName string) (time.Time, error) {
fastClient := shared.GetFastHTTPClient(5 * time.Second)
req := fasthttp.AcquireRequest()
req.SetRequestURI(path.Join(client.giteaRoot, giteaAPIRepos, repoOwner, repoName, "branches", branchName))
req.Header.Set(fasthttp.HeaderAuthorization, client.giteaAPIToken)
res := fasthttp.AcquireResponse()
if err := fastClient.Do(req, res); err != nil {
return time.Time{}, err
}
if res.StatusCode() != fasthttp.StatusOK {
return time.Time{}, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
}
return time.Parse(time.RFC3339, fastjson.GetString(res.Body(), "commit", "timestamp"))
}
func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (string, error) {
fastClient := shared.GetFastHTTPClient(5 * time.Second)
req := fasthttp.AcquireRequest()
req.SetRequestURI(path.Join(client.giteaRoot, giteaAPIRepos, repoOwner, repoName))
req.Header.Set(fasthttp.HeaderAuthorization, client.giteaAPIToken)
res := fasthttp.AcquireResponse()
if err := fastClient.Do(req, res); err != nil {
return "", err
}
if res.StatusCode() != fasthttp.StatusOK {
return "", fmt.Errorf("unexpected status code '%d'", res.StatusCode())
}
return fastjson.GetString(res.Body(), "default_branch"), nil
}

View file

@ -10,6 +10,7 @@ import (
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/dns"
"codeberg.org/codeberg/pages/server/gitea"
"codeberg.org/codeberg/pages/server/upstream"
"codeberg.org/codeberg/pages/server/utils"
)
@ -86,7 +87,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
}
// Check if the branch exists, otherwise treat it as a file path
branchTimestampResult := upstream.GetBranchTimestamp(targetOwner, repo, branch, giteaRoot, giteaAPIToken, branchTimestampCache)
branchTimestampResult := upstream.GetBranchTimestamp(giteaClient, targetOwner, repo, branch, branchTimestampCache)
if branchTimestampResult == nil {
// branch doesn't exist
return false
@ -136,9 +137,8 @@ func Handler(mainDomainSuffix, rawDomain []byte,
giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
) {
log.Debug().Msg("tryBranch, now trying upstream")
tryUpstream(ctx, mainDomainSuffix, trimmedHost,
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
return
}
@ -152,9 +152,8 @@ func Handler(mainDomainSuffix, rawDomain []byte,
giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
)
log.Debug().Msg("tryBranch, now trying upstream")
tryUpstream(ctx, mainDomainSuffix, trimmedHost,
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
return
@ -187,9 +186,8 @@ func Handler(mainDomainSuffix, rawDomain []byte,
"/"+pathElements[0]+"/%p",
) {
log.Debug().Msg("tryBranch, now trying upstream")
tryUpstream(ctx, mainDomainSuffix, trimmedHost,
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
} else {
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
@ -203,9 +201,8 @@ func Handler(mainDomainSuffix, rawDomain []byte,
log.Debug().Msg("main domain preparations, now trying with specified branch")
if tryBranch("pages", pathElements[0][1:], pathElements[1:], "/%p") {
log.Debug().Msg("tryBranch, now trying upstream")
tryUpstream(ctx, mainDomainSuffix, trimmedHost,
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
} else {
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
@ -219,9 +216,8 @@ func Handler(mainDomainSuffix, rawDomain []byte,
log.Debug().Msg("main domain preparations, now trying with specified repo")
if pathElements[0] != "pages" && tryBranch(pathElements[0], "pages", pathElements[1:], "") {
log.Debug().Msg("tryBranch, now trying upstream")
tryUpstream(ctx, mainDomainSuffix, trimmedHost,
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
return
}
@ -231,9 +227,8 @@ func Handler(mainDomainSuffix, rawDomain []byte,
log.Debug().Msg("main domain preparations, now trying with default repo/branch")
if tryBranch("pages", "", pathElements, "") {
log.Debug().Msg("tryBranch, now trying upstream")
tryUpstream(ctx, mainDomainSuffix, trimmedHost,
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
return
}
@ -262,7 +257,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
// Try to use the given repo on the given branch or the default branch
log.Debug().Msg("custom domain preparations, now trying with details from DNS")
if tryBranch(targetRepo, targetBranch, pathElements, canonicalLink) {
canonicalDomain, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)
canonicalDomain, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), canonicalDomainCache)
if !valid {
html.ReturnErrorPage(ctx, fasthttp.StatusMisdirectedRequest)
return
@ -279,9 +274,8 @@ func Handler(mainDomainSuffix, rawDomain []byte,
}
log.Debug().Msg("tryBranch, now trying upstream")
tryUpstream(ctx, mainDomainSuffix, trimmedHost,
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
return
}

View file

@ -0,0 +1,16 @@
package shared
import (
"time"
"github.com/valyala/fasthttp"
)
func GetFastHTTPClient(timeout time.Duration) *fasthttp.Client {
return &fasthttp.Client{
ReadTimeout: timeout,
MaxConnDuration: 60 * time.Second,
MaxConnWaitTimeout: 1000 * time.Millisecond,
MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
}
}

View file

@ -8,22 +8,22 @@ import (
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/gitea"
"codeberg.org/codeberg/pages/server/upstream"
)
// tryUpstream forwards the target request to the Gitea API, and shows an error page on failure.
func tryUpstream(ctx *fasthttp.RequestCtx,
func tryUpstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
mainDomainSuffix, trimmedHost []byte,
targetOptions *upstream.Options,
targetOwner, targetRepo, targetBranch, targetPath,
targetOwner, targetRepo, targetBranch, targetPath string,
giteaRoot, giteaAPIToken string,
canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
) {
// check if a canonical domain exists on a request on MainDomain
if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
canonicalDomain, _ := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)
canonicalDomain, _ := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), canonicalDomainCache)
if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) {
canonicalPath := string(ctx.RequestURI())
if targetRepo != "pages" {
@ -43,7 +43,7 @@ func tryUpstream(ctx *fasthttp.RequestCtx,
targetOptions.TargetPath = targetPath
// Try to request the file from the Gitea API
if !targetOptions.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
if !targetOptions.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
html.ReturnErrorPage(ctx, ctx.Response.StatusCode())
}
}

View file

@ -4,10 +4,11 @@ import (
"strings"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/gitea"
)
// CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file).
func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaAPIToken string, canonicalDomainCache cache.SetGetKey) (string, bool) {
func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (string, bool) {
var (
domains []string
valid bool
@ -21,7 +22,7 @@ func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, m
}
}
} else {
body, err := giteaRawContent(targetOwner, targetRepo, targetBranch, giteaRoot, giteaAPIToken, canonicalDomainConfig)
body, err := giteaClient.GiteaRawContent(targetOwner, targetRepo, targetBranch, canonicalDomainConfig)
if err == nil {
for _, domain := range strings.Split(string(body), "\n") {
domain = strings.TrimPrefix(

View file

@ -8,6 +8,8 @@ import (
"github.com/valyala/fasthttp"
"github.com/valyala/fastjson"
"codeberg.org/codeberg/pages/server/gitea"
)
const giteaAPIRepos = "/api/v1/repos/"
@ -16,7 +18,7 @@ const giteaAPIRepos = "/api/v1/repos/"
// * own client to store token & giteaRoot
// * handle 404 -> page will show 500 atm
func giteaRawContent(targetOwner, targetRepo, ref, giteaRoot, giteaAPIToken, resource string) ([]byte, error) {
func giteaRawContent(client *gitea.Client, targetOwner, targetRepo, ref, giteaRoot, giteaAPIToken, resource string) ([]byte, error) {
req := fasthttp.AcquireRequest()
req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref)))

View file

@ -4,6 +4,7 @@ import (
"time"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/gitea"
)
type branchTimestamp struct {
@ -13,7 +14,7 @@ type branchTimestamp struct {
// GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch
// (or nil if the branch doesn't exist)
func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaAPIToken string, branchTimestampCache cache.SetGetKey) *branchTimestamp {
func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string, branchTimestampCache cache.SetGetKey) *branchTimestamp {
if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok {
if result == nil {
return nil
@ -25,7 +26,7 @@ func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaAPIToken string, br
}
if len(branch) == 0 {
// Get default branch
defaultBranch, err := giteaGetRepoDefaultBranch(giteaRoot, owner, repo, giteaAPIToken)
defaultBranch, err := giteaClient.GiteaGetRepoDefaultBranch(owner, repo)
if err != nil {
_ = branchTimestampCache.Set(owner+"/"+repo+"/", nil, defaultBranchCacheTimeout)
return nil
@ -33,7 +34,7 @@ func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaAPIToken string, br
result.Branch = defaultBranch
}
timestamp, err := giteaGetRepoBranchTimestamp(giteaRoot, owner, repo, branch, giteaAPIToken)
timestamp, err := giteaClient.GiteaGetRepoBranchTimestamp(owner, repo, branch)
if err != nil {
return nil
}

View file

@ -15,6 +15,7 @@ import (
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/gitea"
)
// upstreamIndexPages lists pages that may be considered as index pages for directories.
@ -48,7 +49,7 @@ func getFastHTTPClient(timeout time.Duration) *fasthttp.Client {
}
// Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken string, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) {
func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) {
log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger()
if o.ForbiddenMimeTypes == nil {
@ -57,7 +58,7 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
// Check if the branch exists and when it was modified
if o.BranchTimestamp.IsZero() {
branch := GetBranchTimestamp(o.TargetOwner, o.TargetRepo, o.TargetBranch, giteaRoot, giteaAPIToken, branchTimestampCache)
branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch, branchTimestampCache)
if branch == nil {
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
@ -108,7 +109,7 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
optionsForIndexPages.appendTrailingSlash = true
for _, indexPage := range upstreamIndexPages {
optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage
if optionsForIndexPages.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
exists: false,
}, fileCacheTimeout)
@ -119,7 +120,7 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
optionsForIndexPages.appendTrailingSlash = false
optionsForIndexPages.redirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html"
optionsForIndexPages.TargetPath = o.TargetPath + ".html"
if optionsForIndexPages.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
exists: false,
}, fileCacheTimeout)