pages/pages-server
1
0
Fork 0
mirror of https://codeberg.org/Codeberg/pages-server.git synced 2024-09-30 20:13:39 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

rm cache from fasthttp

Browse source
This commit is contained in:
6543 2022-07-27 17:25:08 +02:00
parent 33298aa8ff
commit 6fd9cbfafb
No known key found for this signature in database
GPG key ID: C99B82E40B027BAE
12 changed files with 425 additions and 250 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
html/error_std.go
View file

@ -5,14 +5,15 @@ package html
import (
"bytes"
"io"
"net/http"
"codeberg.org/codeberg/pages/server/context"
)
// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced
// with the provided status code.
func ReturnErrorPage(resp *http.Response, code int) {
resp.StatusCode = code
resp.Header.Set("Content-Type", "text/html; charset=utf-8")
func ReturnErrorPage(ctx *context.Context, code int) {
ctx.RespWriter.Header().Set("Content-Type", "text/html; charset=utf-8")
ctx.RespWriter.WriteHeader(code)
resp.Body = io.NopCloser(bytes.NewReader(errorBody(code)))
io.Copy(ctx.RespWriter, bytes.NewReader(errorBody(code)))
}

25
server/context/context.go Normal file
View file

@ -0,0 +1,25 @@
package context
import (
stdContext "context"
"net/http"
)
type Context struct {
RespWriter http.ResponseWriter
Req *http.Request
}
func (c *Context) Context() stdContext.Context {
if c.Req != nil {
return c.Req.Context()
}
return stdContext.Background()
}
func (c *Context) Response() *http.Response {
if c.Req != nil {
return c.Req.Response
}
return nil
}

8
server/gitea/cache.go
View file

@ -26,4 +26,12 @@ var (
// than fileCacheTimeout, as that gets invalidated if the branch timestamp has changed. That way, repo changes will be
// picked up faster, while still allowing the content to be cached longer if nothing changes.
branchExistenceCacheTimeout = 5 * time.Minute
// fileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending
// on your available memory.
// TODO: move as option into cache interface
fileCacheTimeout = 5 * time.Minute
// fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default.
fileCacheSizeLimit = 1024 * 1024
)

68
server/gitea/client_std.go
View file

@ -37,35 +37,61 @@ func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, follo
}
func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource string) ([]byte, error) {
rawBytes, resp, err := client.sdkClient.GetFile(targetOwner, targetRepo, ref, resource, client.supportLFS)
reader, _, err := client.ServeRawContent(targetOwner, targetRepo, ref, resource)
if err != nil {
return nil, err
}
switch resp.StatusCode {
case http.StatusOK:
return rawBytes, nil
case http.StatusNotFound:
return nil, ErrorNotFound
default:
return nil, fmt.Errorf("unexpected status code '%d'", resp.StatusCode)
}
defer reader.Close()
return io.ReadAll(reader)
}
func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource string) (io.ReadCloser, error) {
func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource string) (io.ReadCloser, *http.Response, error) {
// if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() {
// cachedResponse = cachedValue.(gitea.FileResponse)
reader, resp, err := client.sdkClient.GetFileReader(targetOwner, targetRepo, ref, resource, client.supportLFS)
if err != nil {
return nil, err
}
if resp != nil {
switch resp.StatusCode {
case http.StatusOK:
switch resp.StatusCode {
case http.StatusOK:
return reader, nil
case http.StatusNotFound:
return nil, ErrorNotFound
default:
return nil, fmt.Errorf("unexpected status code '%d'", resp.StatusCode)
// add caching
// Write the response body to the original request
// var cacheBodyWriter bytes.Buffer
// if res != nil {
// if res.Header.ContentLength() > fileCacheSizeLimit {
// // fasthttp else will set "Content-Length: 0"
// ctx.Response().SetBodyStream(&strings.Reader{}, -1)
//
// err = res.BodyWriteTo(ctx.Response.BodyWriter())
// } else {
// // TODO: cache is half-empty if request is cancelled - does the ctx.Err() below do the trick?
// err = res.BodyWriteTo(io.MultiWriter(ctx.Response().BodyWriter(), &cacheBodyWriter))
// }
// } else {
// _, err = ctx.Write(cachedResponse.Body)
// }
// if res != nil && res.Header.ContentLength() <= fileCacheSizeLimit && ctx.Err() == nil {
// cachedResponse.Exists = true
// cachedResponse.MimeType = mimeType
// cachedResponse.Body = cacheBodyWriter.Bytes()
// _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), cachedResponse, fileCacheTimeout)
// }
return reader, resp.Response, err
case http.StatusNotFound:
// add not exist caching
// _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
// Exists: false,
// }, fileCacheTimeout)
return nil, resp.Response, ErrorNotFound
default:
return nil, resp.Response, fmt.Errorf("unexpected status code '%d'", resp.StatusCode)
}
}
return nil, nil, err
}
func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchName string) (*BranchTimestamp, error) {

50
server/handler.go
View file

@ -4,6 +4,7 @@ package server
import (
"bytes"
"net/http"
"strings"
"github.com/rs/zerolog"
@ -25,23 +26,23 @@ func Handler(mainDomainSuffix, rawDomain []byte,
giteaRoot, rawInfoPage string,
blacklistedPaths, allowedCorsDomains [][]byte,
dnsLookupCache, canonicalDomainCache cache.SetGetKey,
) func(ctx *fasthttp.RequestCtx) {
return func(ctx *fasthttp.RequestCtx) {
log := log.With().Str("Handler", string(ctx.Request.Header.RequestURI())).Logger()
) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
log := log.With().Str("Handler", req.RequestURI).Logger()
ctx.Response.Header.Set("Server", "CodebergPages/"+version.Version)
w.Header().Set("Server", "CodebergPages/"+version.Version)
// Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin
ctx.Response.Header.Set("Referrer-Policy", "strict-origin-when-cross-origin")
w.Header().Set("Referrer-Policy", "strict-origin-when-cross-origin")
// Enable browser caching for up to 10 minutes
ctx.Response.Header.Set("Cache-Control", "public, max-age=600")
w.Header().Set("Cache-Control", "public, max-age=600")
trimmedHost := utils.TrimHostPort(ctx.Request.Host())
trimmedHost := utils.TrimHostPort([]byte(req.Host))
// Add HSTS for RawDomain and MainDomainSuffix
if hsts := GetHSTSHeader(trimmedHost, mainDomainSuffix, rawDomain); hsts != "" {
ctx.Response.Header.Set("Strict-Transport-Security", hsts)
w.Header().Set("Strict-Transport-Security", hsts)
}
// Block all methods not required for static pages
@ -68,12 +69,13 @@ func Handler(mainDomainSuffix, rawDomain []byte,
}
}
if allowCors {
ctx.Response.Header.Set("Access-Control-Allow-Origin", "*")
ctx.Response.Header.Set("Access-Control-Allow-Methods", "GET, HEAD")
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Access-Control-Allow-Methods", "GET, HEAD")
}
ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS")
w.Header().Set("Allow", "GET, HEAD, OPTIONS")
if ctx.IsOptions() {
ctx.Response.Header.SetStatusCode(fasthttp.StatusNoContent)
ctx.Response.Header.SetStatusCode(http.StatusNoContent)
return
}
@ -137,7 +139,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
if len(pathElements) < 2 {
// https://{RawDomain}/{owner}/{repo}[/@{branch}]/{path} is required
ctx.Redirect(rawInfoPage, fasthttp.StatusTemporaryRedirect)
ctx.Redirect(rawInfoPage, http.StatusTemporaryRedirect)
return
}
targetOwner = pathElements[0]
@ -157,7 +159,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
return
}
log.Debug().Msg("missing branch")
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
html.ReturnErrorPage(ctx, http.StatusFailedDependency)
return
}
@ -183,7 +185,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
if targetOwner == "www" {
// www.codeberg.page redirects to codeberg.page // TODO: rm hardcoded - use cname?
ctx.Redirect("https://"+string(mainDomainSuffix[1:])+string(ctx.Path()), fasthttp.StatusPermanentRedirect)
ctx.Redirect("https://"+string(mainDomainSuffix[1:])+string(ctx.Path()), http.StatusPermanentRedirect)
return
}
@ -192,7 +194,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
if len(pathElements) > 1 && strings.HasPrefix(pathElements[1], "@") {
if targetRepo == "pages" {
// example.codeberg.org/pages/@... redirects to example.codeberg.org/@...
ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), fasthttp.StatusTemporaryRedirect)
ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), http.StatusTemporaryRedirect)
return
}
@ -206,7 +208,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
canonicalDomainCache)
} else {
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
html.ReturnErrorPage(ctx, http.StatusFailedDependency)
}
return
}
@ -222,7 +224,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
canonicalDomainCache)
} else {
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
html.ReturnErrorPage(ctx, http.StatusFailedDependency)
}
return
}
@ -253,7 +255,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
}
// Couldn't find a valid repo/branch
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
html.ReturnErrorPage(ctx, http.StatusFailedDependency)
return
} else {
trimmedHostStr := string(trimmedHost)
@ -261,7 +263,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
// Serve pages from external domains
targetOwner, targetRepo, targetBranch = dns.GetTargetFromDNS(trimmedHostStr, string(mainDomainSuffix), dnsLookupCache)
if targetOwner == "" {
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
html.ReturnErrorPage(ctx, http.StatusFailedDependency)
return
}
@ -279,17 +281,17 @@ func Handler(mainDomainSuffix, rawDomain []byte,
targetRepo, targetBranch, pathElements, canonicalLink) {
canonicalDomain, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), canonicalDomainCache)
if !valid {
html.ReturnErrorPage(ctx, fasthttp.StatusMisdirectedRequest)
html.ReturnErrorPage(ctx, http.StatusMisdirectedRequest)
return
} else if canonicalDomain != trimmedHostStr {
// only redirect if the target is also a codeberg page!
targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix), dnsLookupCache)
if targetOwner != "" {
ctx.Redirect("https://"+canonicalDomain+string(ctx.RequestURI()), fasthttp.StatusTemporaryRedirect)
ctx.Redirect("https://"+canonicalDomain+string(ctx.RequestURI()), http.StatusTemporaryRedirect)
return
}
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
html.ReturnErrorPage(ctx, http.StatusFailedDependency)
return
}
@ -300,7 +302,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
return
}
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
html.ReturnErrorPage(ctx, http.StatusFailedDependency)
return
}
}

18
server/handler_fasthttp.go
View file

@ -24,7 +24,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
giteaClient *gitea.Client,
giteaRoot, rawInfoPage string,
blacklistedPaths, allowedCorsDomains [][]byte,
dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
dnsLookupCache, canonicalDomainCache cache.SetGetKey,
) func(ctx *fasthttp.RequestCtx) {
return func(ctx *fasthttp.RequestCtx) {
log := log.With().Str("Handler", string(ctx.Request.Header.RequestURI())).Logger()
@ -96,7 +96,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
branch = strings.ReplaceAll(branch, "~", "/")
// Check if the branch exists, otherwise treat it as a file path
branchTimestampResult := upstream.GetBranchTimestamp(giteaClient, targetOwner, repo, branch, branchTimestampCache)
branchTimestampResult := upstream.GetBranchTimestamp(giteaClient, targetOwner, repo, branch)
if branchTimestampResult == nil {
log.Debug().Msg("tryBranch: branch doesn't exist")
return false
@ -153,7 +153,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
log.Debug().Msg("tryBranch, now trying upstream 1")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
canonicalDomainCache)
return
}
log.Debug().Msg("missing branch")
@ -169,7 +169,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
log.Debug().Msg("tryBranch, now trying upstream 2")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
canonicalDomainCache)
return
} else if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
@ -204,7 +204,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
log.Debug().Msg("tryBranch, now trying upstream 3")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
canonicalDomainCache)
} else {
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
}
@ -220,7 +220,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
log.Debug().Msg("tryBranch, now trying upstream 4")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
canonicalDomainCache)
} else {
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
}
@ -236,7 +236,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
log.Debug().Msg("tryBranch, now trying upstream 5")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
canonicalDomainCache)
return
}
@ -248,7 +248,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
log.Debug().Msg("tryBranch, now trying upstream 6")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
canonicalDomainCache)
return
}
@ -296,7 +296,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
log.Debug().Msg("tryBranch, now trying upstream 7")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
canonicalDomainCache)
return
}

12
server/try.go
View file

@ -4,18 +4,18 @@ package server
import (
"bytes"
"net/http"
"strings"
"github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/context"
"codeberg.org/codeberg/pages/server/gitea"
"codeberg.org/codeberg/pages/server/upstream"
)
// tryUpstream forwards the target request to the Gitea API, and shows an error page on failure.
func tryUpstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
func tryUpstream(ctx *context.Context, giteaClient *gitea.Client,
mainDomainSuffix, trimmedHost []byte,
targetOptions *upstream.Options,
@ -27,14 +27,14 @@ func tryUpstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
canonicalDomain, _ := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), canonicalDomainCache)
if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) {
canonicalPath := string(ctx.RequestURI())
canonicalPath := ctx.Req.RequestURI
if targetRepo != "pages" {
path := strings.SplitN(canonicalPath, "/", 3)
if len(path) >= 3 {
canonicalPath = "/" + path[2]
}
}
ctx.Redirect("https://"+canonicalDomain+canonicalPath, fasthttp.StatusTemporaryRedirect)
ctx.Redirect("https://"+canonicalDomain+canonicalPath, http.StatusTemporaryRedirect)
return
}
}
@ -46,6 +46,6 @@ func tryUpstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
// Try to request the file from the Gitea API
if !targetOptions.Upstream(ctx, giteaClient) {
html.ReturnErrorPage(ctx, ctx.Response.StatusCode())
html.ReturnErrorPage(w, ctx.Response().StatusCode)
}
}

4
server/try_fasthttp.go
View file

@ -21,7 +21,7 @@ func tryUpstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
targetOptions *upstream.Options,
targetOwner, targetRepo, targetBranch, targetPath string,
canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
canonicalDomainCache cache.SetGetKey,
) {
// check if a canonical domain exists on a request on MainDomain
if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
@ -45,7 +45,7 @@ func tryUpstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
targetOptions.TargetPath = targetPath
// Try to request the file from the Gitea API
if !targetOptions.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
if !targetOptions.Upstream(ctx, giteaClient) {
html.ReturnErrorPage(ctx, ctx.Response.StatusCode())
}
}

8
server/upstream/const.go
View file

@ -2,14 +2,6 @@ package upstream
import "time"
// fileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending
// on your available memory.
// TODO: move as option into cache interface
var fileCacheTimeout = 5 * time.Minute
// fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default.
var fileCacheSizeLimit = 1024 * 1024
// canonicalDomainCacheTimeout specifies the timeout for the canonical domain cache.
var canonicalDomainCacheTimeout = 15 * time.Minute

175
server/upstream/upstream.go
View file

@ -1,18 +1,7 @@
package upstream
import (
"bytes"
"errors"
"fmt"
"io"
"strings"
"time"
"github.com/rs/zerolog/log"
"github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/gitea"
)
// upstreamIndexPages lists pages that may be considered as index pages for directories.
@ -40,167 +29,3 @@ type Options struct {
appendTrailingSlash bool
redirectIfExists string
}
// Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client) (final bool) {
log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger()
// Check if the branch exists and when it was modified
if o.BranchTimestamp.IsZero() {
branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch)
if branch == nil {
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
return true
}
o.TargetBranch = branch.Branch
o.BranchTimestamp = branch.Timestamp
}
if o.TargetOwner == "" || o.TargetRepo == "" || o.TargetBranch == "" {
html.ReturnErrorPage(ctx, fasthttp.StatusBadRequest)
return true
}
// Check if the browser has a cached version
if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Request.Header.Peek("If-Modified-Since"))); err == nil {
if !ifModifiedSince.Before(o.BranchTimestamp) {
ctx.Response.SetStatusCode(fasthttp.StatusNotModified)
return true
}
}
log.Debug().Msg("preparations")
// Make a GET request to the upstream URL
uri := o.generateUri()
var cachedResponse gitea.FileResponse
// if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() {
// cachedResponse = cachedValue.(gitea.FileResponse)
// } else {
res, err := giteaClient.ServeRawContent(o.generateUriClientArgs())
log.Debug().Msg("acquisition")
// Handle errors
if (err != nil && errors.Is(err, gitea.ErrorNotFound)) || (res == nil && !cachedResponse.Exists) {
if o.TryIndexPages {
// copy the o struct & try if an index page exists
optionsForIndexPages := *o
optionsForIndexPages.TryIndexPages = false
optionsForIndexPages.appendTrailingSlash = true
for _, indexPage := range upstreamIndexPages {
optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage
if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache) {
_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
Exists: false,
}, fileCacheTimeout)
return true
}
}
// compatibility fix for GitHub Pages (/example → /example.html)
optionsForIndexPages.appendTrailingSlash = false
optionsForIndexPages.redirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html"
optionsForIndexPages.TargetPath = o.TargetPath + ".html"
if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache) {
_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
Exists: false,
}, fileCacheTimeout)
return true
}
}
ctx.Response.SetStatusCode(fasthttp.StatusNotFound)
if o.TryIndexPages {
// copy the o struct & try if a not found page exists
optionsForNotFoundPages := *o
optionsForNotFoundPages.TryIndexPages = false
optionsForNotFoundPages.appendTrailingSlash = false
for _, notFoundPage := range upstreamNotFoundPages {
optionsForNotFoundPages.TargetPath = "/" + notFoundPage
if optionsForNotFoundPages.Upstream(ctx, giteaClient, branchTimestampCache) {
_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
Exists: false,
}, fileCacheTimeout)
return true
}
}
}
if res != nil {
// Update cache if the request is fresh
_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
Exists: false,
}, fileCacheTimeout)
}
return false
}
if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) {
fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", uri, err, res.StatusCode())
html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
return true
}
// Append trailing slash if missing (for index files), and redirect to fix filenames in general
// o.appendTrailingSlash is only true when looking for index pages
if o.appendTrailingSlash && !bytes.HasSuffix(ctx.Request.URI().Path(), []byte{'/'}) {
ctx.Redirect(string(ctx.Request.URI().Path())+"/", fasthttp.StatusTemporaryRedirect)
return true
}
if bytes.HasSuffix(ctx.Request.URI().Path(), []byte("/index.html")) {
ctx.Redirect(strings.TrimSuffix(string(ctx.Request.URI().Path()), "index.html"), fasthttp.StatusTemporaryRedirect)
return true
}
if o.redirectIfExists != "" {
ctx.Redirect(o.redirectIfExists, fasthttp.StatusTemporaryRedirect)
return true
}
log.Debug().Msg("error handling")
// Set the MIME type
mimeType := o.getMimeTypeByExtension()
ctx.Response.Header.SetContentType(mimeType)
// Set ETag
if cachedResponse.Exists {
ctx.Response.Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag)
} else if res != nil {
cachedResponse.ETag = res.Header.Peek(fasthttp.HeaderETag)
ctx.Response.Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag)
}
if ctx.Response.StatusCode() != fasthttp.StatusNotFound {
// Everything's okay so far
ctx.Response.SetStatusCode(fasthttp.StatusOK)
}
ctx.Response.Header.SetLastModified(o.BranchTimestamp)
log.Debug().Msg("response preparations")
// Write the response body to the original request
var cacheBodyWriter bytes.Buffer
if res != nil {
if res.Header.ContentLength() > fileCacheSizeLimit {
// fasthttp else will set "Content-Length: 0"
ctx.Response.SetBodyStream(&strings.Reader{}, -1)
err = res.BodyWriteTo(ctx.Response.BodyWriter())
} else {
// TODO: cache is half-empty if request is cancelled - does the ctx.Err() below do the trick?
err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter))
}
} else {
_, err = ctx.Write(cachedResponse.Body)
}
if err != nil {
fmt.Printf("Couldn't write body for \"%s\": %s\n", uri, err)
html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
return true
}
log.Debug().Msg("response")
if res != nil && res.Header.ContentLength() <= fileCacheSizeLimit && ctx.Err() == nil {
cachedResponse.Exists = true
cachedResponse.MimeType = mimeType
cachedResponse.Body = cacheBodyWriter.Bytes()
_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), cachedResponse, fileCacheTimeout)
}
return true
}

150
server/upstream/upstream_fasthttp.go Normal file
View file

@ -0,0 +1,150 @@
//go:build fasthttp
package upstream
import (
"bytes"
"errors"
"fmt"
"strings"
"time"
"github.com/rs/zerolog/log"
"github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/gitea"
)
// Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client) (final bool) {
log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger()
// Check if the branch exists and when it was modified
if o.BranchTimestamp.IsZero() {
branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch)
if branch == nil {
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
return true
}
o.TargetBranch = branch.Branch
o.BranchTimestamp = branch.Timestamp
}
if o.TargetOwner == "" || o.TargetRepo == "" || o.TargetBranch == "" {
html.ReturnErrorPage(ctx, fasthttp.StatusBadRequest)
return true
}
// Check if the browser has a cached version
if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Request.Header.Peek("If-Modified-Since"))); err == nil {
if !ifModifiedSince.Before(o.BranchTimestamp) {
ctx.Response.SetStatusCode(fasthttp.StatusNotModified)
return true
}
}
log.Debug().Msg("preparations")
// Make a GET request to the upstream URL
uri := o.generateUri()
var cachedResponse gitea.FileResponse
// if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() {
// cachedResponse = cachedValue.(gitea.FileResponse)
// } else {
res, err := giteaClient.ServeRawContent(o.generateUriClientArgs())
log.Debug().Msg("acquisition")
// Handle errors
if (err != nil && errors.Is(err, gitea.ErrorNotFound)) || (res == nil && !cachedResponse.Exists) {
if o.TryIndexPages {
// copy the o struct & try if an index page exists
optionsForIndexPages := *o
optionsForIndexPages.TryIndexPages = false
optionsForIndexPages.appendTrailingSlash = true
for _, indexPage := range upstreamIndexPages {
optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage
if optionsForIndexPages.Upstream(ctx, giteaClient) {
return true
}
}
// compatibility fix for GitHub Pages (/example → /example.html)
optionsForIndexPages.appendTrailingSlash = false
optionsForIndexPages.redirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html"
optionsForIndexPages.TargetPath = o.TargetPath + ".html"
if optionsForIndexPages.Upstream(ctx, giteaClient) {
return true
}
}
ctx.Response.SetStatusCode(fasthttp.StatusNotFound)
if o.TryIndexPages {
// copy the o struct & try if a not found page exists
optionsForNotFoundPages := *o
optionsForNotFoundPages.TryIndexPages = false
optionsForNotFoundPages.appendTrailingSlash = false
for _, notFoundPage := range upstreamNotFoundPages {
optionsForNotFoundPages.TargetPath = "/" + notFoundPage
if optionsForNotFoundPages.Upstream(ctx, giteaClient) {
return true
}
}
}
return false
}
if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) {
fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", uri, err, res.StatusCode())
html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
return true
}
// Append trailing slash if missing (for index files), and redirect to fix filenames in general
// o.appendTrailingSlash is only true when looking for index pages
if o.appendTrailingSlash && !bytes.HasSuffix(ctx.Request.URI().Path(), []byte{'/'}) {
ctx.Redirect(string(ctx.Request.URI().Path())+"/", fasthttp.StatusTemporaryRedirect)
return true
}
if bytes.HasSuffix(ctx.Request.URI().Path(), []byte("/index.html")) {
ctx.Redirect(strings.TrimSuffix(string(ctx.Request.URI().Path()), "index.html"), fasthttp.StatusTemporaryRedirect)
return true
}
if o.redirectIfExists != "" {
ctx.Redirect(o.redirectIfExists, fasthttp.StatusTemporaryRedirect)
return true
}
log.Debug().Msg("error handling")
// Set the MIME type
mimeType := o.getMimeTypeByExtension()
ctx.Response.Header.SetContentType(mimeType)
// Set ETag
if cachedResponse.Exists {
ctx.Response.Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag)
} else if res != nil {
cachedResponse.ETag = res.Header.Peek(fasthttp.HeaderETag)
ctx.Response.Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag)
}
if ctx.Response.StatusCode() != fasthttp.StatusNotFound {
// Everything's okay so far
ctx.Response.SetStatusCode(fasthttp.StatusOK)
}
ctx.Response.Header.SetLastModified(o.BranchTimestamp)
log.Debug().Msg("response preparations")
// Write the response body to the original request
// fasthttp else will set "Content-Length: 0"
ctx.Response.SetBodyStream(&strings.Reader{}, -1)
err = res.BodyWriteTo(ctx.Response.BodyWriter())
if err != nil {
fmt.Printf("Couldn't write body for \"%s\": %s\n", uri, err)
html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
return true
}
log.Debug().Msg("response")
return true
}

146
server/upstream/upstream_std.go Normal file
View file

@ -0,0 +1,146 @@
//go:build !fasthttp
package upstream
import (
"bytes"
"errors"
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/rs/zerolog/log"
"github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/context"
"codeberg.org/codeberg/pages/server/gitea"
)
// Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (final bool) {
log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger()
// Check if the branch exists and when it was modified
if o.BranchTimestamp.IsZero() {
branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch)
if branch == nil {
html.ReturnErrorPage(ctx, http.StatusFailedDependency)
return true
}
o.TargetBranch = branch.Branch
o.BranchTimestamp = branch.Timestamp
}
if o.TargetOwner == "" || o.TargetRepo == "" || o.TargetBranch == "" {
html.ReturnErrorPage(ctx, http.StatusBadRequest)
return true
}
// Check if the browser has a cached version
if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Req.Response.Header.Get("If-Modified-Since"))); err == nil {
if !ifModifiedSince.Before(o.BranchTimestamp) {
ctx.RespWriter.WriteHeader(http.StatusNotModified)
return true
}
}
log.Debug().Msg("preparations")
reader, res, err := giteaClient.ServeRawContent(o.generateUriClientArgs())
log.Debug().Msg("acquisition")
// Handle errors
if (err != nil && errors.Is(err, gitea.ErrorNotFound)) || (res == nil) {
if o.TryIndexPages {
// copy the o struct & try if an index page exists
optionsForIndexPages := *o
optionsForIndexPages.TryIndexPages = false
optionsForIndexPages.appendTrailingSlash = true
for _, indexPage := range upstreamIndexPages {
optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage
if optionsForIndexPages.Upstream(ctx, giteaClient) {
return true
}
}
// compatibility fix for GitHub Pages (/example → /example.html)
optionsForIndexPages.appendTrailingSlash = false
optionsForIndexPages.redirectIfExists = strings.TrimSuffix(ctx.Req.URL.Path, "/") + ".html"
optionsForIndexPages.TargetPath = o.TargetPath + ".html"
if optionsForIndexPages.Upstream(ctx, giteaClient) {
return true
}
}
ctx.Response().SetStatusCode(http.StatusNotFound)
if o.TryIndexPages {
// copy the o struct & try if a not found page exists
optionsForNotFoundPages := *o
optionsForNotFoundPages.TryIndexPages = false
optionsForNotFoundPages.appendTrailingSlash = false
for _, notFoundPage := range upstreamNotFoundPages {
optionsForNotFoundPages.TargetPath = "/" + notFoundPage
if optionsForNotFoundPages.Upstream(ctx, giteaClient) {
return true
}
}
}
return false
}
if res != nil && (err != nil || res.StatusCode() != http.StatusOK) {
fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", uri, err, res.StatusCode())
html.ReturnErrorPage(ctx, http.StatusInternalServerError)
return true
}
// Append trailing slash if missing (for index files), and redirect to fix filenames in general
// o.appendTrailingSlash is only true when looking for index pages
if o.appendTrailingSlash && !bytes.HasSuffix(ctx.Request().URI().Path(), []byte{'/'}) {
ctx.Redirect(string(ctx.Request.URI().Path())+"/", http.StatusTemporaryRedirect)
return true
}
if bytes.HasSuffix(ctx.Request.URI().Path(), []byte("/index.html")) {
ctx.Redirect(strings.TrimSuffix(string(ctx.Request.URI().Path()), "index.html"), http.StatusTemporaryRedirect)
return true
}
if o.redirectIfExists != "" {
ctx.Redirect(o.redirectIfExists, http.StatusTemporaryRedirect)
return true
}
log.Debug().Msg("error handling")
// Set the MIME type
mimeType := o.getMimeTypeByExtension()
ctx.Response.Header.SetContentType(mimeType)
// Set ETag
if cachedResponse.Exists {
ctx.Response().Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag)
} else if res != nil {
cachedResponse.ETag = res.Header.Peek(fasthttp.HeaderETag)
ctx.Response().Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag)
}
if ctx.Response().StatusCode() != http.StatusNotFound {
// Everything's okay so far
ctx.Response().SetStatusCode(http.StatusOK)
}
ctx.Response().Header.SetLastModified(o.BranchTimestamp)
log.Debug().Msg("response preparations")
// Write the response body to the original request
if reader != nil {
_, err := io.Copy(ctx.RespWriter, reader)
if err != nil {
fmt.Printf("Couldn't write body for \"%s\": %s\n", uri, err)
html.ReturnErrorPage(ctx, http.StatusInternalServerError)
return true
}
}
log.Debug().Msg("response")
return true
}