mirror of
https://codeberg.org/Codeberg/pages-server.git
synced 2025-04-19 03:26:57 +00:00
switch to std http implementation instead of fasthttp (#106)
close #100 close #109 close #113 close #28 close #63 Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/106
This commit is contained in:
6543
parent
69eabb248a
commit
b9966487f6
28 changed files with 827 additions and 584 deletions
|
|
@ -1,24 +0,0 @@
|
|||
package upstream
|
||||
|
||||
import "time"
|
||||
|
||||
// defaultBranchCacheTimeout specifies the timeout for the default branch cache. It can be quite long.
|
||||
var defaultBranchCacheTimeout = 15 * time.Minute
|
||||
|
||||
// branchExistenceCacheTimeout specifies the timeout for the branch timestamp & existence cache. It should be shorter
|
||||
// than fileCacheTimeout, as that gets invalidated if the branch timestamp has changed. That way, repo changes will be
|
||||
// picked up faster, while still allowing the content to be cached longer if nothing changes.
|
||||
var branchExistenceCacheTimeout = 5 * time.Minute
|
||||
|
||||
// fileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending
|
||||
// on your available memory.
|
||||
// TODO: move as option into cache interface
|
||||
var fileCacheTimeout = 5 * time.Minute
|
||||
|
||||
// fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default.
|
||||
var fileCacheSizeLimit = 1024 * 1024
|
||||
|
||||
// canonicalDomainCacheTimeout specifies the timeout for the canonical domain cache.
|
||||
var canonicalDomainCacheTimeout = 15 * time.Minute
|
||||
|
||||
const canonicalDomainConfig = ".domains"
|
||||
|
|
@ -2,11 +2,19 @@ package upstream
|
|||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
|
||||
"codeberg.org/codeberg/pages/server/cache"
|
||||
"codeberg.org/codeberg/pages/server/gitea"
|
||||
)
|
||||
|
||||
// canonicalDomainCacheTimeout specifies the timeout for the canonical domain cache.
|
||||
var canonicalDomainCacheTimeout = 15 * time.Minute
|
||||
|
||||
const canonicalDomainConfig = ".domains"
|
||||
|
||||
// CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file).
|
||||
func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (string, bool) {
|
||||
var (
|
||||
|
|
@ -36,6 +44,8 @@ func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, ta
|
|||
valid = true
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log.Info().Err(err).Msgf("could not read %s of %s/%s", canonicalDomainConfig, targetOwner, targetRepo)
|
||||
}
|
||||
domains = append(domains, targetOwner+mainDomainSuffix)
|
||||
if domains[len(domains)-1] == actualDomain {
|
||||
|
|
|
|||
|
|
@ -1,84 +1,36 @@
|
|||
package upstream
|
||||
|
||||
import (
|
||||
"mime"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"errors"
|
||||
|
||||
"codeberg.org/codeberg/pages/server/cache"
|
||||
"codeberg.org/codeberg/pages/server/gitea"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
type branchTimestamp struct {
|
||||
Branch string
|
||||
Timestamp time.Time
|
||||
}
|
||||
"codeberg.org/codeberg/pages/server/gitea"
|
||||
)
|
||||
|
||||
// GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch
|
||||
// (or nil if the branch doesn't exist)
|
||||
func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string, branchTimestampCache cache.SetGetKey) *branchTimestamp {
|
||||
func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string) *gitea.BranchTimestamp {
|
||||
log := log.With().Strs("BranchInfo", []string{owner, repo, branch}).Logger()
|
||||
if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok {
|
||||
if result == nil {
|
||||
log.Debug().Msg("branchTimestampCache found item, but result is empty")
|
||||
return nil
|
||||
}
|
||||
log.Debug().Msg("branchTimestampCache found item, returning result")
|
||||
return result.(*branchTimestamp)
|
||||
}
|
||||
result := &branchTimestamp{
|
||||
Branch: branch,
|
||||
}
|
||||
|
||||
if len(branch) == 0 {
|
||||
// Get default branch
|
||||
defaultBranch, err := giteaClient.GiteaGetRepoDefaultBranch(owner, repo)
|
||||
if err != nil {
|
||||
log.Err(err).Msg("Could't fetch default branch from repository")
|
||||
_ = branchTimestampCache.Set(owner+"/"+repo+"/", nil, defaultBranchCacheTimeout)
|
||||
return nil
|
||||
}
|
||||
log.Debug().Msg("Succesfully fetched default branch from Gitea")
|
||||
result.Branch = defaultBranch
|
||||
log.Debug().Msgf("Succesfully fetched default branch %q from Gitea", defaultBranch)
|
||||
branch = defaultBranch
|
||||
}
|
||||
|
||||
timestamp, err := giteaClient.GiteaGetRepoBranchTimestamp(owner, repo, result.Branch)
|
||||
timestamp, err := giteaClient.GiteaGetRepoBranchTimestamp(owner, repo, branch)
|
||||
if err != nil {
|
||||
log.Err(err).Msg("Could not get latest commit's timestamp from branch")
|
||||
if !errors.Is(err, gitea.ErrorNotFound) {
|
||||
log.Error().Err(err).Msg("Could not get latest commit's timestamp from branch")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
log.Debug().Msg("Succesfully fetched latest commit's timestamp from branch, adding to cache")
|
||||
result.Timestamp = timestamp
|
||||
_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, branchExistenceCacheTimeout)
|
||||
return result
|
||||
}
|
||||
|
||||
func (o *Options) getMimeTypeByExtension() string {
|
||||
if o.ForbiddenMimeTypes == nil {
|
||||
o.ForbiddenMimeTypes = make(map[string]bool)
|
||||
}
|
||||
mimeType := mime.TypeByExtension(path.Ext(o.TargetPath))
|
||||
mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
|
||||
if o.ForbiddenMimeTypes[mimeTypeSplit[0]] || mimeType == "" {
|
||||
if o.DefaultMimeType != "" {
|
||||
mimeType = o.DefaultMimeType
|
||||
} else {
|
||||
mimeType = "application/octet-stream"
|
||||
}
|
||||
}
|
||||
return mimeType
|
||||
}
|
||||
|
||||
func (o *Options) generateUri() string {
|
||||
return path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath)
|
||||
}
|
||||
|
||||
func (o *Options) generateUriClientArgs() (targetOwner, targetRepo, ref, resource string) {
|
||||
return o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath
|
||||
}
|
||||
|
||||
func (o *Options) timestamp() string {
|
||||
return strconv.FormatInt(o.BranchTimestamp.Unix(), 10)
|
||||
log.Debug().Msgf("Succesfully fetched latest commit's timestamp from branch: %#v", timestamp)
|
||||
return timestamp
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,20 +1,27 @@
|
|||
package upstream
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/valyala/fasthttp"
|
||||
|
||||
"codeberg.org/codeberg/pages/html"
|
||||
"codeberg.org/codeberg/pages/server/cache"
|
||||
"codeberg.org/codeberg/pages/server/context"
|
||||
"codeberg.org/codeberg/pages/server/gitea"
|
||||
)
|
||||
|
||||
const (
|
||||
headerLastModified = "Last-Modified"
|
||||
headerIfModifiedSince = "If-Modified-Since"
|
||||
|
||||
rawMime = "text/plain; charset=utf-8"
|
||||
)
|
||||
|
||||
// upstreamIndexPages lists pages that may be considered as index pages for directories.
|
||||
var upstreamIndexPages = []string{
|
||||
"index.html",
|
||||
|
|
@ -35,61 +42,61 @@ type Options struct {
|
|||
// Used for debugging purposes.
|
||||
Host string
|
||||
|
||||
DefaultMimeType string
|
||||
ForbiddenMimeTypes map[string]bool
|
||||
TryIndexPages bool
|
||||
BranchTimestamp time.Time
|
||||
TryIndexPages bool
|
||||
BranchTimestamp time.Time
|
||||
// internal
|
||||
appendTrailingSlash bool
|
||||
redirectIfExists string
|
||||
|
||||
ServeRaw bool
|
||||
}
|
||||
|
||||
// Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
|
||||
func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) {
|
||||
log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath, o.Host}).Logger()
|
||||
func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (final bool) {
|
||||
log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger()
|
||||
|
||||
if o.TargetOwner == "" || o.TargetRepo == "" {
|
||||
html.ReturnErrorPage(ctx, "either repo owner or name info is missing", http.StatusBadRequest)
|
||||
return true
|
||||
}
|
||||
|
||||
// Check if the branch exists and when it was modified
|
||||
if o.BranchTimestamp.IsZero() {
|
||||
branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch, branchTimestampCache)
|
||||
branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch)
|
||||
|
||||
if branch == nil {
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
if branch == nil || branch.Branch == "" {
|
||||
html.ReturnErrorPage(ctx,
|
||||
fmt.Sprintf("could not get timestamp of branch %q", o.TargetBranch),
|
||||
http.StatusFailedDependency)
|
||||
return true
|
||||
}
|
||||
o.TargetBranch = branch.Branch
|
||||
o.BranchTimestamp = branch.Timestamp
|
||||
}
|
||||
|
||||
if o.TargetOwner == "" || o.TargetRepo == "" || o.TargetBranch == "" {
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusBadRequest)
|
||||
return true
|
||||
}
|
||||
|
||||
// Check if the browser has a cached version
|
||||
if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Request.Header.Peek("If-Modified-Since"))); err == nil {
|
||||
if !ifModifiedSince.Before(o.BranchTimestamp) {
|
||||
ctx.Response.SetStatusCode(fasthttp.StatusNotModified)
|
||||
return true
|
||||
if ctx.Response() != nil {
|
||||
if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Response().Header.Get(headerIfModifiedSince))); err == nil {
|
||||
if !ifModifiedSince.Before(o.BranchTimestamp) {
|
||||
ctx.RespWriter.WriteHeader(http.StatusNotModified)
|
||||
log.Trace().Msg("check response against last modified: valid")
|
||||
return true
|
||||
}
|
||||
}
|
||||
log.Trace().Msg("check response against last modified: outdated")
|
||||
}
|
||||
|
||||
log.Debug().Msg("Preparing")
|
||||
|
||||
// Make a GET request to the upstream URL
|
||||
uri := o.generateUri()
|
||||
var res *fasthttp.Response
|
||||
var cachedResponse gitea.FileResponse
|
||||
var err error
|
||||
if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() {
|
||||
cachedResponse = cachedValue.(gitea.FileResponse)
|
||||
} else {
|
||||
res, err = giteaClient.ServeRawContent(o.generateUriClientArgs())
|
||||
reader, header, statusCode, err := giteaClient.ServeRawContent(o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath)
|
||||
if reader != nil {
|
||||
defer reader.Close()
|
||||
}
|
||||
|
||||
log.Debug().Msg("Aquisting")
|
||||
|
||||
// Handle errors
|
||||
if (err != nil && errors.Is(err, gitea.ErrorNotFound)) || (res == nil && !cachedResponse.Exists) {
|
||||
// Handle not found error
|
||||
if err != nil && errors.Is(err, gitea.ErrorNotFound) {
|
||||
if o.TryIndexPages {
|
||||
// copy the o struct & try if an index page exists
|
||||
optionsForIndexPages := *o
|
||||
|
|
@ -97,25 +104,20 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
|
|||
optionsForIndexPages.appendTrailingSlash = true
|
||||
for _, indexPage := range upstreamIndexPages {
|
||||
optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage
|
||||
if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
|
||||
_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
|
||||
Exists: false,
|
||||
}, fileCacheTimeout)
|
||||
if optionsForIndexPages.Upstream(ctx, giteaClient) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
// compatibility fix for GitHub Pages (/example → /example.html)
|
||||
optionsForIndexPages.appendTrailingSlash = false
|
||||
optionsForIndexPages.redirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html"
|
||||
optionsForIndexPages.redirectIfExists = strings.TrimSuffix(ctx.Path(), "/") + ".html"
|
||||
optionsForIndexPages.TargetPath = o.TargetPath + ".html"
|
||||
if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
|
||||
_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
|
||||
Exists: false,
|
||||
}, fileCacheTimeout)
|
||||
if optionsForIndexPages.Upstream(ctx, giteaClient) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
ctx.Response.SetStatusCode(fasthttp.StatusNotFound)
|
||||
|
||||
ctx.StatusCode = http.StatusNotFound
|
||||
if o.TryIndexPages {
|
||||
// copy the o struct & try if a not found page exists
|
||||
optionsForNotFoundPages := *o
|
||||
|
|
@ -123,94 +125,84 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
|
|||
optionsForNotFoundPages.appendTrailingSlash = false
|
||||
for _, notFoundPage := range upstreamNotFoundPages {
|
||||
optionsForNotFoundPages.TargetPath = "/" + notFoundPage
|
||||
if optionsForNotFoundPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
|
||||
_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
|
||||
Exists: false,
|
||||
}, fileCacheTimeout)
|
||||
if optionsForNotFoundPages.Upstream(ctx, giteaClient) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
if res != nil {
|
||||
// Update cache if the request is fresh
|
||||
_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
|
||||
Exists: false,
|
||||
}, fileCacheTimeout)
|
||||
}
|
||||
return false
|
||||
}
|
||||
if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) {
|
||||
log.Warn().Msgf("Couldn't fetch contents from %q: %v (status code %d)", uri, err, res.StatusCode())
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
|
||||
|
||||
// handle unexpected client errors
|
||||
if err != nil || reader == nil || statusCode != http.StatusOK {
|
||||
log.Debug().Msg("Handling error")
|
||||
var msg string
|
||||
|
||||
if err != nil {
|
||||
msg = "gitea client returned unexpected error"
|
||||
log.Error().Err(err).Msg(msg)
|
||||
msg = fmt.Sprintf("%s: %v", msg, err)
|
||||
}
|
||||
if reader == nil {
|
||||
msg = "gitea client returned no reader"
|
||||
log.Error().Msg(msg)
|
||||
}
|
||||
if statusCode != http.StatusOK {
|
||||
msg = fmt.Sprintf("Couldn't fetch contents (status code %d)", statusCode)
|
||||
log.Error().Msg(msg)
|
||||
}
|
||||
|
||||
html.ReturnErrorPage(ctx, msg, http.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
|
||||
// Append trailing slash if missing (for index files), and redirect to fix filenames in general
|
||||
// o.appendTrailingSlash is only true when looking for index pages
|
||||
if o.appendTrailingSlash && !bytes.HasSuffix(ctx.Request.URI().Path(), []byte{'/'}) {
|
||||
ctx.Redirect(string(ctx.Request.URI().Path())+"/", fasthttp.StatusTemporaryRedirect)
|
||||
if o.appendTrailingSlash && !strings.HasSuffix(ctx.Path(), "/") {
|
||||
ctx.Redirect(ctx.Path()+"/", http.StatusTemporaryRedirect)
|
||||
return true
|
||||
}
|
||||
if bytes.HasSuffix(ctx.Request.URI().Path(), []byte("/index.html")) {
|
||||
ctx.Redirect(strings.TrimSuffix(string(ctx.Request.URI().Path()), "index.html"), fasthttp.StatusTemporaryRedirect)
|
||||
if strings.HasSuffix(ctx.Path(), "/index.html") {
|
||||
ctx.Redirect(strings.TrimSuffix(ctx.Path(), "index.html"), http.StatusTemporaryRedirect)
|
||||
return true
|
||||
}
|
||||
if o.redirectIfExists != "" {
|
||||
ctx.Redirect(o.redirectIfExists, fasthttp.StatusTemporaryRedirect)
|
||||
ctx.Redirect(o.redirectIfExists, http.StatusTemporaryRedirect)
|
||||
return true
|
||||
}
|
||||
|
||||
log.Debug().Msg("Handling error")
|
||||
|
||||
// Set the MIME type
|
||||
mimeType := o.getMimeTypeByExtension()
|
||||
ctx.Response.Header.SetContentType(mimeType)
|
||||
|
||||
// Set ETag
|
||||
if cachedResponse.Exists {
|
||||
ctx.Response.Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag)
|
||||
} else if res != nil {
|
||||
cachedResponse.ETag = res.Header.Peek(fasthttp.HeaderETag)
|
||||
ctx.Response.Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag)
|
||||
// Set ETag & MIME
|
||||
if eTag := header.Get(gitea.ETagHeader); eTag != "" {
|
||||
ctx.RespWriter.Header().Set(gitea.ETagHeader, eTag)
|
||||
}
|
||||
|
||||
if ctx.Response.StatusCode() != fasthttp.StatusNotFound {
|
||||
// Everything's okay so far
|
||||
ctx.Response.SetStatusCode(fasthttp.StatusOK)
|
||||
if cacheIndicator := header.Get(gitea.PagesCacheIndicatorHeader); cacheIndicator != "" {
|
||||
ctx.RespWriter.Header().Set(gitea.PagesCacheIndicatorHeader, cacheIndicator)
|
||||
}
|
||||
ctx.Response.Header.SetLastModified(o.BranchTimestamp)
|
||||
if length := header.Get(gitea.ContentLengthHeader); length != "" {
|
||||
ctx.RespWriter.Header().Set(gitea.ContentLengthHeader, length)
|
||||
}
|
||||
if mime := header.Get(gitea.ContentTypeHeader); mime == "" || o.ServeRaw {
|
||||
ctx.RespWriter.Header().Set(gitea.ContentTypeHeader, rawMime)
|
||||
} else {
|
||||
ctx.RespWriter.Header().Set(gitea.ContentTypeHeader, mime)
|
||||
}
|
||||
ctx.RespWriter.Header().Set(headerLastModified, o.BranchTimestamp.In(time.UTC).Format(time.RFC1123))
|
||||
|
||||
log.Debug().Msg("Prepare response")
|
||||
|
||||
// Write the response body to the original request
|
||||
var cacheBodyWriter bytes.Buffer
|
||||
if res != nil {
|
||||
if res.Header.ContentLength() > fileCacheSizeLimit {
|
||||
// fasthttp else will set "Content-Length: 0"
|
||||
ctx.Response.SetBodyStream(&strings.Reader{}, -1)
|
||||
ctx.RespWriter.WriteHeader(ctx.StatusCode)
|
||||
|
||||
err = res.BodyWriteTo(ctx.Response.BodyWriter())
|
||||
} else {
|
||||
// TODO: cache is half-empty if request is cancelled - does the ctx.Err() below do the trick?
|
||||
err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter))
|
||||
// Write the response body to the original request
|
||||
if reader != nil {
|
||||
_, err := io.Copy(ctx.RespWriter, reader)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msgf("Couldn't write body for %q", o.TargetPath)
|
||||
html.ReturnErrorPage(ctx, "", http.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
} else {
|
||||
_, err = ctx.Write(cachedResponse.Body)
|
||||
}
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msgf("Couldn't write body for %q", uri)
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
|
||||
log.Debug().Msg("Sending response")
|
||||
|
||||
if res != nil && res.Header.ContentLength() <= fileCacheSizeLimit && ctx.Err() == nil {
|
||||
cachedResponse.Exists = true
|
||||
cachedResponse.MimeType = mimeType
|
||||
cachedResponse.Body = cacheBodyWriter.Bytes()
|
||||
_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), cachedResponse, fileCacheTimeout)
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue