pages/pages-server
1
0
Fork 0
mirror of https://codeberg.org/Codeberg/pages-server.git synced 2025-04-25 06:16:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

wip

Browse source
This commit is contained in:
6543 2022-05-16 00:43:04 +02:00
parent 4f2c94c7d9
commit ec7fcca0f5
No known key found for this signature in database
GPG key ID: C99B82E40B027BAE
8 changed files with 96 additions and 128 deletions
Download patch file Download diff file Expand all files Collapse all files

6
cmd/main.go
View file

@ -18,6 +18,7 @@ import (
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/certificates"
"codeberg.org/codeberg/pages/server/database"
"codeberg.org/codeberg/pages/server/gitea"
)
// AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed.
@ -81,9 +82,12 @@ func Serve(ctx *cli.Context) error {
// TODO: make this an MRU cache with a size limit
fileResponseCache := cache.NewKeyValueCache()
giteaClient := gitea.NewClient(giteaRoot, giteaAPIToken)
// Create handler based on settings
handler := server.Handler(mainDomainSuffix, []byte(rawDomain),
giteaRoot, rawInfoPage, giteaAPIToken,
giteaClient,
giteaRoot, rawInfoPage,
BlacklistedPaths, allowedCorsDomains,
dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache)

26
server/gitea/client.go
View file

@ -4,6 +4,7 @@ import (
"fmt"
"net/url"
"path"
"strings"
"time"
"github.com/valyala/fasthttp"
@ -12,11 +13,21 @@ import (
"codeberg.org/codeberg/pages/server/shared"
)
const giteaAPIRepos = "/api/v1/repos/"
type Client struct {
giteaRoot string
giteaAPIToken string
}
type FileResponse struct {
Exists bool
MimeType string
Body []byte
}
func (f FileResponse) IsEmpty() bool { return len(f.Body) != 0 }
func NewClient(giteaRoot, giteaAPIToken string) *Client {
return &Client{
giteaRoot: giteaRoot,
@ -24,8 +35,6 @@ func NewClient(giteaRoot, giteaAPIToken string) *Client {
}
}
const giteaAPIRepos = "/api/v1/repos/"
// TODOs:
// * own client to store token & giteaRoot
// * handle 404 -> page will show 500 atm
@ -46,6 +55,19 @@ func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource str
return res.Body(), nil
}
func (client *Client) ServeRawContent(uri string) (*fasthttp.Response, error) {
fastClient := shared.GetFastHTTPClient(10 * time.Second)
req := fasthttp.AcquireRequest()
req.SetRequestURI(path.Join(client.giteaRoot, giteaAPIRepos, uri))
req.Header.Set(fasthttp.HeaderAuthorization, client.giteaAPIToken)
resp := fasthttp.AcquireResponse()
resp.SetBodyStream(&strings.Reader{}, -1)
return resp, fastClient.Do(req, resp)
}
func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchName string) (time.Time, error) {
fastClient := shared.GetFastHTTPClient(5 * time.Second)

16
server/gitea/fasthttp.go Normal file
View file

@ -0,0 +1,16 @@
package gitea
import (
"time"
"github.com/valyala/fasthttp"
)
func getFastHTTPClient(timeout time.Duration) *fasthttp.Client {
return &fasthttp.Client{
ReadTimeout: timeout,
MaxConnDuration: 60 * time.Second,
MaxConnWaitTimeout: 1000 * time.Millisecond,
MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
}
}

3
server/handler.go
View file

@ -17,7 +17,8 @@ import (
// Handler handles a single HTTP request to the web server.
func Handler(mainDomainSuffix, rawDomain []byte,
giteaRoot, rawInfoPage, giteaAPIToken string,
giteaClient *gitea.Client,
giteaRoot, rawInfoPage string,
blacklistedPaths, allowedCorsDomains [][]byte,
dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
) func(ctx *fasthttp.RequestCtx) {

11
server/handler_test.go
View file

@ -8,15 +8,16 @@ import (
"github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/gitea"
)
func TestHandlerPerformance(t *testing.T) {
giteaRoot := "https://codeberg.org"
giteaClient := gitea.NewClient(giteaRoot, "")
testHandler := Handler(
[]byte("codeberg.page"),
[]byte("raw.codeberg.org"),
"https://codeberg.org",
"https://docs.codeberg.org/pages/raw-content/",
"",
[]byte("codeberg.page"), []byte("raw.codeberg.org"),
giteaClient,
giteaRoot, "https://docs.codeberg.org/pages/raw-content/",
[][]byte{[]byte("/.well-known/acme-challenge/")},
[][]byte{[]byte("raw.codeberg.org"), []byte("fonts.codeberg.org"), []byte("design.codeberg.org")},
cache.NewKeyValueCache(),

69
server/upstream/gitea.go
View file

@ -1,69 +0,0 @@
package upstream
import (
"fmt"
"net/url"
"path"
"time"
"github.com/valyala/fasthttp"
"github.com/valyala/fastjson"
"codeberg.org/codeberg/pages/server/gitea"
)
const giteaAPIRepos = "/api/v1/repos/"
// TODOs:
// * own client to store token & giteaRoot
// * handle 404 -> page will show 500 atm
func giteaRawContent(client *gitea.Client, targetOwner, targetRepo, ref, giteaRoot, giteaAPIToken, resource string) ([]byte, error) {
req := fasthttp.AcquireRequest()
req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref)))
req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken)
res := fasthttp.AcquireResponse()
if err := getFastHTTPClient(10*time.Second).Do(req, res); err != nil {
return nil, err
}
if res.StatusCode() != fasthttp.StatusOK {
return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
}
return res.Body(), nil
}
func giteaGetRepoBranchTimestamp(giteaRoot, repoOwner, repoName, branchName, giteaAPIToken string) (time.Time, error) {
client := getFastHTTPClient(5 * time.Second)
req := fasthttp.AcquireRequest()
req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, repoOwner, repoName, "branches", branchName))
req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken)
res := fasthttp.AcquireResponse()
if err := client.Do(req, res); err != nil {
return time.Time{}, err
}
if res.StatusCode() != fasthttp.StatusOK {
return time.Time{}, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
}
return time.Parse(time.RFC3339, fastjson.GetString(res.Body(), "commit", "timestamp"))
}
func giteaGetRepoDefaultBranch(giteaRoot, repoOwner, repoName, giteaAPIToken string) (string, error) {
client := getFastHTTPClient(5 * time.Second)
req := fasthttp.AcquireRequest()
req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, repoOwner, repoName))
req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken)
res := fasthttp.AcquireResponse()
if err := client.Do(req, res); err != nil {
return "", err
}
if res.StatusCode() != fasthttp.StatusOK {
return "", fmt.Errorf("unexpected status code '%d'", res.StatusCode())
}
return fastjson.GetString(res.Body(), "default_branch"), nil
}

27
server/upstream/helper.go
View file

@ -1,6 +1,10 @@
package upstream
import (
"mime"
"path"
"strconv"
"strings"
"time"
"codeberg.org/codeberg/pages/server/cache"
@ -43,8 +47,23 @@ func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string, b
return result
}
type fileResponse struct {
exists bool
mimeType string
body []byte
func (o *Options) getMimeTypeByExtension() string {
mimeType := mime.TypeByExtension(path.Ext(o.TargetPath))
mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
if _, ok := o.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" {
if o.DefaultMimeType != "" {
mimeType = o.DefaultMimeType
} else {
mimeType = "application/octet-stream"
}
}
return mimeType
}
func (o *Options) generateUri() string {
return path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath)
}
func (o *Options) timestamp() string {
return strconv.FormatInt(o.BranchTimestamp.Unix(), 10)
}

66
server/upstream/upstream.go
View file

@ -4,9 +4,6 @@ import (
"bytes"
"fmt"
"io"
"mime"
"path"
"strconv"
"strings"
"time"
@ -39,15 +36,6 @@ type Options struct {
redirectIfExists string
}
func getFastHTTPClient(timeout time.Duration) *fasthttp.Client {
return &fasthttp.Client{
ReadTimeout: timeout,
MaxConnDuration: 60 * time.Second,
MaxConnWaitTimeout: 1000 * time.Millisecond,
MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
}
}
// Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) {
log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger()
@ -83,25 +71,19 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
log.Debug().Msg("preparations")
// Make a GET request to the upstream URL
uri := path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath)
var req *fasthttp.Request
uri := o.generateUri()
var res *fasthttp.Response
var cachedResponse fileResponse
var cachedResponse gitea.FileResponse
var err error
if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + strconv.FormatInt(o.BranchTimestamp.Unix(), 10)); ok && len(cachedValue.(fileResponse).body) > 0 {
cachedResponse = cachedValue.(fileResponse)
if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() {
cachedResponse = cachedValue.(gitea.FileResponse)
} else {
req = fasthttp.AcquireRequest()
req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, uri))
req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken)
res = fasthttp.AcquireResponse()
res.SetBodyStream(&strings.Reader{}, -1)
err = getFastHTTPClient(10*time.Second).Do(req, res)
res, err = giteaClient.ServeRawContent(uri)
}
log.Debug().Msg("acquisition")
// Handle errors
if (res == nil && !cachedResponse.exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) {
if (res == nil && !cachedResponse.Exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) {
if o.TryIndexPages {
// copy the o struct & try if an index page exists
optionsForIndexPages := *o
@ -110,8 +92,8 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
for _, indexPage := range upstreamIndexPages {
optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage
if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
exists: false,
_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
Exists: false,
}, fileCacheTimeout)
return true
}
@ -121,8 +103,8 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
optionsForIndexPages.redirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html"
optionsForIndexPages.TargetPath = o.TargetPath + ".html"
if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
exists: false,
_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
Exists: false,
}, fileCacheTimeout)
return true
}
@ -130,14 +112,14 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
ctx.Response.SetStatusCode(fasthttp.StatusNotFound)
if res != nil {
// Update cache if the request is fresh
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
exists: false,
_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
Exists: false,
}, fileCacheTimeout)
}
return false
}
if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) {
fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", req.RequestURI(), err, res.StatusCode())
fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", uri, err, res.StatusCode())
html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
return true
}
@ -159,15 +141,7 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
log.Debug().Msg("error handling")
// Set the MIME type
mimeType := mime.TypeByExtension(path.Ext(o.TargetPath))
mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
if _, ok := o.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" {
if o.DefaultMimeType != "" {
mimeType = o.DefaultMimeType
} else {
mimeType = "application/octet-stream"
}
}
mimeType := o.getMimeTypeByExtension()
ctx.Response.Header.SetContentType(mimeType)
// Everything's okay so far
@ -186,20 +160,20 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter))
}
} else {
_, err = ctx.Write(cachedResponse.body)
_, err = ctx.Write(cachedResponse.Body)
}
if err != nil {
fmt.Printf("Couldn't write body for \"%s\": %s\n", req.RequestURI(), err)
fmt.Printf("Couldn't write body for \"%s\": %s\n", uri, err)
html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
return true
}
log.Debug().Msg("response")
if res != nil && ctx.Err() == nil {
cachedResponse.exists = true
cachedResponse.mimeType = mimeType
cachedResponse.body = cacheBodyWriter.Bytes()
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), cachedResponse, fileCacheTimeout)
cachedResponse.Exists = true
cachedResponse.MimeType = mimeType
cachedResponse.Body = cacheBodyWriter.Bytes()
_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), cachedResponse, fileCacheTimeout)
}
return true