mirror of
https://codeberg.org/Codeberg/pages-server.git
synced 2024-11-18 10:29:43 +00:00
move upstream into own package
This commit is contained in:
6543
parent
f35c4d0f66
commit
38426c26db
7 changed files with 321 additions and 288 deletions
23
html/error.go
Normal file
23
html/error.go
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
package html
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"github.com/valyala/fasthttp"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced
|
||||
// with the provided status code.
|
||||
func ReturnErrorPage(ctx *fasthttp.RequestCtx, code int) {
|
||||
ctx.Response.SetStatusCode(code)
|
||||
ctx.Response.Header.SetContentType("text/html; charset=utf-8")
|
||||
message := fasthttp.StatusMessage(code)
|
||||
if code == fasthttp.StatusMisdirectedRequest {
|
||||
message += " - domain not specified in <code>.domains</code> file"
|
||||
}
|
||||
if code == fasthttp.StatusFailedDependency {
|
||||
message += " - target repo/branch doesn't exist or is private"
|
||||
}
|
||||
// TODO: use template engine?
|
||||
ctx.Response.SetBody(bytes.ReplaceAll(NotFoundPage, []byte("%status"), []byte(strconv.Itoa(code)+" "+message)))
|
||||
}
|
||||
|
|
@ -1,11 +1,14 @@
|
|||
package server
|
||||
|
||||
import (
|
||||
"github.com/OrlovEvgeny/go-mcache"
|
||||
"github.com/valyala/fasthttp"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/OrlovEvgeny/go-mcache"
|
||||
"github.com/valyala/fasthttp"
|
||||
|
||||
"codeberg.org/codeberg/pages/server/upstream"
|
||||
)
|
||||
|
||||
// DnsLookupCacheTimeout specifies the timeout for the DNS lookup cache.
|
||||
|
|
@ -84,7 +87,7 @@ func checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, m
|
|||
req.SetRequestURI(giteaRoot + "/api/v1/repos/" + targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/.domains" + "?access_token=" + giteaApiToken)
|
||||
res := fasthttp.AcquireResponse()
|
||||
|
||||
err := upstreamClient.Do(req, res)
|
||||
err := upstream.Client.Do(req, res)
|
||||
if err == nil && res.StatusCode() == fasthttp.StatusOK {
|
||||
for _, domain := range strings.Split(string(res.Body()), "\n") {
|
||||
domain = strings.ToLower(domain)
|
||||
|
|
|
|||
|
|
@ -2,21 +2,13 @@ package server
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/OrlovEvgeny/go-mcache"
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/valyala/fasthttp"
|
||||
"github.com/valyala/fastjson"
|
||||
|
||||
"codeberg.org/codeberg/pages/html"
|
||||
"codeberg.org/codeberg/pages/server/upstream"
|
||||
"codeberg.org/codeberg/pages/server/utils"
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/valyala/fasthttp"
|
||||
)
|
||||
|
||||
// Handler handles a single HTTP request to the web server.
|
||||
|
|
@ -49,7 +41,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp
|
|||
// Block blacklisted paths (like ACME challenges)
|
||||
for _, blacklistedPath := range blacklistedPaths {
|
||||
if bytes.HasPrefix(ctx.Path(), blacklistedPath) {
|
||||
returnErrorPage(ctx, fasthttp.StatusForbidden)
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusForbidden)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
|
@ -74,7 +66,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp
|
|||
|
||||
// Prepare request information to Gitea
|
||||
var targetOwner, targetRepo, targetBranch, targetPath string
|
||||
var targetOptions = &upstreamOptions{
|
||||
var targetOptions = &upstream.Options{
|
||||
ForbiddenMimeTypes: map[string]struct{}{},
|
||||
TryIndexPages: true,
|
||||
}
|
||||
|
|
@ -87,7 +79,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp
|
|||
}
|
||||
|
||||
// Check if the branch exists, otherwise treat it as a file path
|
||||
branchTimestampResult := getBranchTimestamp(targetOwner, repo, branch, giteaRoot, giteaApiToken)
|
||||
branchTimestampResult := upstream.GetBranchTimestamp(targetOwner, repo, branch, giteaRoot, giteaApiToken)
|
||||
if branchTimestampResult == nil {
|
||||
// branch doesn't exist
|
||||
return false
|
||||
|
|
@ -96,9 +88,9 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp
|
|||
// Branch exists, use it
|
||||
targetRepo = repo
|
||||
targetPath = strings.Trim(strings.Join(path, "/"), "/")
|
||||
targetBranch = branchTimestampResult.branch
|
||||
targetBranch = branchTimestampResult.Branch
|
||||
|
||||
targetOptions.BranchTimestamp = branchTimestampResult.timestamp
|
||||
targetOptions.BranchTimestamp = branchTimestampResult.Timestamp
|
||||
|
||||
if canonicalLink != "" {
|
||||
// Hide from search machines & add canonical link
|
||||
|
|
@ -128,8 +120,8 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp
|
|||
}
|
||||
|
||||
// Try to request the file from the Gitea API
|
||||
if !upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaApiToken, targetOptions) {
|
||||
returnErrorPage(ctx, ctx.Response.StatusCode())
|
||||
if !upstream.Upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaApiToken, targetOptions) {
|
||||
html.ReturnErrorPage(ctx, ctx.Response.StatusCode())
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -163,7 +155,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp
|
|||
return
|
||||
}
|
||||
log.Debug().Msg("missing branch")
|
||||
returnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
return
|
||||
} else {
|
||||
log.Debug().Msg("raw domain preparations, now trying with default branch")
|
||||
|
|
@ -206,7 +198,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp
|
|||
log.Debug().Msg("tryBranch, now trying upstream")
|
||||
tryUpstream()
|
||||
} else {
|
||||
returnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
|
@ -219,7 +211,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp
|
|||
log.Debug().Msg("tryBranch, now trying upstream")
|
||||
tryUpstream()
|
||||
} else {
|
||||
returnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
|
@ -244,7 +236,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp
|
|||
}
|
||||
|
||||
// Couldn't find a valid repo/branch
|
||||
returnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
return
|
||||
} else {
|
||||
trimmedHostStr := string(trimmedHost)
|
||||
|
|
@ -252,7 +244,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp
|
|||
// Serve pages from external domains
|
||||
targetOwner, targetRepo, targetBranch = getTargetFromDNS(trimmedHostStr, string(mainDomainSuffix))
|
||||
if targetOwner == "" {
|
||||
returnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -269,7 +261,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp
|
|||
if tryBranch(targetRepo, targetBranch, pathElements, canonicalLink) {
|
||||
canonicalDomain, valid := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), giteaRoot, giteaApiToken)
|
||||
if !valid {
|
||||
returnErrorPage(ctx, fasthttp.StatusMisdirectedRequest)
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusMisdirectedRequest)
|
||||
return
|
||||
} else if canonicalDomain != trimmedHostStr {
|
||||
// only redirect if the target is also a codeberg page!
|
||||
|
|
@ -278,7 +270,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp
|
|||
ctx.Redirect("https://"+canonicalDomain+string(ctx.RequestURI()), fasthttp.StatusTemporaryRedirect)
|
||||
return
|
||||
} else {
|
||||
returnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
|
@ -287,268 +279,9 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp
|
|||
tryUpstream()
|
||||
return
|
||||
} else {
|
||||
returnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// returnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced
|
||||
// with the provided status code.
|
||||
func returnErrorPage(ctx *fasthttp.RequestCtx, code int) {
|
||||
ctx.Response.SetStatusCode(code)
|
||||
ctx.Response.Header.SetContentType("text/html; charset=utf-8")
|
||||
message := fasthttp.StatusMessage(code)
|
||||
if code == fasthttp.StatusMisdirectedRequest {
|
||||
message += " - domain not specified in <code>.domains</code> file"
|
||||
}
|
||||
if code == fasthttp.StatusFailedDependency {
|
||||
message += " - target repo/branch doesn't exist or is private"
|
||||
}
|
||||
// TODO: use template engine?
|
||||
ctx.Response.SetBody(bytes.ReplaceAll(html.NotFoundPage, []byte("%status"), []byte(strconv.Itoa(code)+" "+message)))
|
||||
}
|
||||
|
||||
// DefaultBranchCacheTimeout specifies the timeout for the default branch cache. It can be quite long.
|
||||
var DefaultBranchCacheTimeout = 15 * time.Minute
|
||||
|
||||
// BranchExistanceCacheTimeout specifies the timeout for the branch timestamp & existance cache. It should be shorter
|
||||
// than FileCacheTimeout, as that gets invalidated if the branch timestamp has changed. That way, repo changes will be
|
||||
// picked up faster, while still allowing the content to be cached longer if nothing changes.
|
||||
var BranchExistanceCacheTimeout = 5 * time.Minute
|
||||
|
||||
// branchTimestampCache stores branch timestamps for faster cache checking
|
||||
var branchTimestampCache = mcache.New()
|
||||
|
||||
type branchTimestamp struct {
|
||||
branch string
|
||||
timestamp time.Time
|
||||
}
|
||||
|
||||
// FileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending
|
||||
// on your available memory.
|
||||
var FileCacheTimeout = 5 * time.Minute
|
||||
|
||||
// FileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default.
|
||||
var FileCacheSizeLimit = 1024 * 1024
|
||||
|
||||
// fileResponseCache stores responses from the Gitea server
|
||||
// TODO: make this an MRU cache with a size limit
|
||||
var fileResponseCache = mcache.New()
|
||||
|
||||
type fileResponse struct {
|
||||
exists bool
|
||||
mimeType string
|
||||
body []byte
|
||||
}
|
||||
|
||||
// getBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch
|
||||
// (or nil if the branch doesn't exist)
|
||||
func getBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string) *branchTimestamp {
|
||||
if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok {
|
||||
if result == nil {
|
||||
return nil
|
||||
}
|
||||
return result.(*branchTimestamp)
|
||||
}
|
||||
result := &branchTimestamp{}
|
||||
result.branch = branch
|
||||
if branch == "" {
|
||||
// Get default branch
|
||||
var body = make([]byte, 0)
|
||||
// TODO: use header for API key?
|
||||
status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"?access_token="+giteaApiToken, 5*time.Second)
|
||||
if err != nil || status != 200 {
|
||||
_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, nil, DefaultBranchCacheTimeout)
|
||||
return nil
|
||||
}
|
||||
result.branch = fastjson.GetString(body, "default_branch")
|
||||
}
|
||||
|
||||
var body = make([]byte, 0)
|
||||
status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"/branches/"+branch+"?access_token="+giteaApiToken, 5*time.Second)
|
||||
if err != nil || status != 200 {
|
||||
return nil
|
||||
}
|
||||
|
||||
result.timestamp, _ = time.Parse(time.RFC3339, fastjson.GetString(body, "commit", "timestamp"))
|
||||
_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, BranchExistanceCacheTimeout)
|
||||
return result
|
||||
}
|
||||
|
||||
var upstreamClient = fasthttp.Client{
|
||||
ReadTimeout: 10 * time.Second,
|
||||
MaxConnDuration: 60 * time.Second,
|
||||
MaxConnWaitTimeout: 1000 * time.Millisecond,
|
||||
MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
|
||||
}
|
||||
|
||||
// upstreamIndexPages lists pages that may be considered as index pages for directories.
|
||||
var upstreamIndexPages = []string{
|
||||
"index.html",
|
||||
}
|
||||
|
||||
// upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
|
||||
func upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaApiToken string, options *upstreamOptions) (final bool) {
|
||||
log := log.With().Strs("upstream", []string{targetOwner, targetRepo, targetBranch, targetPath}).Logger()
|
||||
|
||||
if options.ForbiddenMimeTypes == nil {
|
||||
options.ForbiddenMimeTypes = map[string]struct{}{}
|
||||
}
|
||||
|
||||
// Check if the branch exists and when it was modified
|
||||
if options.BranchTimestamp == (time.Time{}) {
|
||||
branch := getBranchTimestamp(targetOwner, targetRepo, targetBranch, giteaRoot, giteaApiToken)
|
||||
|
||||
if branch == nil {
|
||||
returnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
return true
|
||||
}
|
||||
targetBranch = branch.branch
|
||||
options.BranchTimestamp = branch.timestamp
|
||||
}
|
||||
|
||||
if targetOwner == "" || targetRepo == "" || targetBranch == "" {
|
||||
returnErrorPage(ctx, fasthttp.StatusBadRequest)
|
||||
return true
|
||||
}
|
||||
|
||||
// Check if the browser has a cached version
|
||||
if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Request.Header.Peek("If-Modified-Since"))); err == nil {
|
||||
if !ifModifiedSince.Before(options.BranchTimestamp) {
|
||||
ctx.Response.SetStatusCode(fasthttp.StatusNotModified)
|
||||
return true
|
||||
}
|
||||
}
|
||||
log.Debug().Msg("preparations")
|
||||
|
||||
// Make a GET request to the upstream URL
|
||||
uri := targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/" + targetPath
|
||||
var req *fasthttp.Request
|
||||
var res *fasthttp.Response
|
||||
var cachedResponse fileResponse
|
||||
var err error
|
||||
if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + strconv.FormatInt(options.BranchTimestamp.Unix(), 10)); ok && len(cachedValue.(fileResponse).body) > 0 {
|
||||
cachedResponse = cachedValue.(fileResponse)
|
||||
} else {
|
||||
req = fasthttp.AcquireRequest()
|
||||
req.SetRequestURI(giteaRoot + "/api/v1/repos/" + uri + "?access_token=" + giteaApiToken)
|
||||
res = fasthttp.AcquireResponse()
|
||||
res.SetBodyStream(&strings.Reader{}, -1)
|
||||
err = upstreamClient.Do(req, res)
|
||||
}
|
||||
log.Debug().Msg("acquisition")
|
||||
|
||||
// Handle errors
|
||||
if (res == nil && !cachedResponse.exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) {
|
||||
if options.TryIndexPages {
|
||||
// copy the options struct & try if an index page exists
|
||||
optionsForIndexPages := *options
|
||||
optionsForIndexPages.TryIndexPages = false
|
||||
optionsForIndexPages.AppendTrailingSlash = true
|
||||
for _, indexPage := range upstreamIndexPages {
|
||||
if upstream(ctx, targetOwner, targetRepo, targetBranch, strings.TrimSuffix(targetPath, "/")+"/"+indexPage, giteaRoot, giteaApiToken, &optionsForIndexPages) {
|
||||
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{
|
||||
exists: false,
|
||||
}, FileCacheTimeout)
|
||||
return true
|
||||
}
|
||||
}
|
||||
// compatibility fix for GitHub Pages (/example → /example.html)
|
||||
optionsForIndexPages.AppendTrailingSlash = false
|
||||
optionsForIndexPages.RedirectIfExists = string(ctx.Request.URI().Path()) + ".html"
|
||||
if upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath+".html", giteaRoot, giteaApiToken, &optionsForIndexPages) {
|
||||
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{
|
||||
exists: false,
|
||||
}, FileCacheTimeout)
|
||||
return true
|
||||
}
|
||||
}
|
||||
ctx.Response.SetStatusCode(fasthttp.StatusNotFound)
|
||||
if res != nil {
|
||||
// Update cache if the request is fresh
|
||||
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{
|
||||
exists: false,
|
||||
}, FileCacheTimeout)
|
||||
}
|
||||
return false
|
||||
}
|
||||
if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) {
|
||||
fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", req.RequestURI(), err, res.StatusCode())
|
||||
returnErrorPage(ctx, fasthttp.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
|
||||
// Append trailing slash if missing (for index files), and redirect to fix filenames in general
|
||||
// options.AppendTrailingSlash is only true when looking for index pages
|
||||
if options.AppendTrailingSlash && !bytes.HasSuffix(ctx.Request.URI().Path(), []byte{'/'}) {
|
||||
ctx.Redirect(string(ctx.Request.URI().Path())+"/", fasthttp.StatusTemporaryRedirect)
|
||||
return true
|
||||
}
|
||||
if bytes.HasSuffix(ctx.Request.URI().Path(), []byte("/index.html")) {
|
||||
ctx.Redirect(strings.TrimSuffix(string(ctx.Request.URI().Path()), "index.html"), fasthttp.StatusTemporaryRedirect)
|
||||
return true
|
||||
}
|
||||
if options.RedirectIfExists != "" {
|
||||
ctx.Redirect(options.RedirectIfExists, fasthttp.StatusTemporaryRedirect)
|
||||
return true
|
||||
}
|
||||
log.Debug().Msg("error handling")
|
||||
|
||||
// Set the MIME type
|
||||
mimeType := mime.TypeByExtension(path.Ext(targetPath))
|
||||
mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
|
||||
if _, ok := options.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" {
|
||||
if options.DefaultMimeType != "" {
|
||||
mimeType = options.DefaultMimeType
|
||||
} else {
|
||||
mimeType = "application/octet-stream"
|
||||
}
|
||||
}
|
||||
ctx.Response.Header.SetContentType(mimeType)
|
||||
|
||||
// Everything's okay so far
|
||||
ctx.Response.SetStatusCode(fasthttp.StatusOK)
|
||||
ctx.Response.Header.SetLastModified(options.BranchTimestamp)
|
||||
|
||||
log.Debug().Msg("response preparations")
|
||||
|
||||
// Write the response body to the original request
|
||||
var cacheBodyWriter bytes.Buffer
|
||||
if res != nil {
|
||||
if res.Header.ContentLength() > FileCacheSizeLimit {
|
||||
err = res.BodyWriteTo(ctx.Response.BodyWriter())
|
||||
} else {
|
||||
// TODO: cache is half-empty if request is cancelled - does the ctx.Err() below do the trick?
|
||||
err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter))
|
||||
}
|
||||
} else {
|
||||
_, err = ctx.Write(cachedResponse.body)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Printf("Couldn't write body for \"%s\": %s\n", req.RequestURI(), err)
|
||||
returnErrorPage(ctx, fasthttp.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
log.Debug().Msg("response")
|
||||
|
||||
if res != nil && ctx.Err() == nil {
|
||||
cachedResponse.exists = true
|
||||
cachedResponse.mimeType = mimeType
|
||||
cachedResponse.body = cacheBodyWriter.Bytes()
|
||||
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), cachedResponse, FileCacheTimeout)
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// upstreamOptions provides various options for the upstream request.
|
||||
type upstreamOptions struct {
|
||||
DefaultMimeType string
|
||||
ForbiddenMimeTypes map[string]struct{}
|
||||
TryIndexPages bool
|
||||
AppendTrailingSlash bool
|
||||
RedirectIfExists string
|
||||
BranchTimestamp time.Time
|
||||
}
|
||||
|
|
|
|||
10
server/upstream/2rm.go
Normal file
10
server/upstream/2rm.go
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
package upstream
|
||||
|
||||
import "github.com/OrlovEvgeny/go-mcache"
|
||||
|
||||
// branchTimestampCache stores branch timestamps for faster cache checking
|
||||
var branchTimestampCache = mcache.New()
|
||||
|
||||
// fileResponseCache stores responses from the Gitea server
|
||||
// TODO: make this an MRU cache with a size limit
|
||||
var fileResponseCache = mcache.New()
|
||||
18
server/upstream/const.go
Normal file
18
server/upstream/const.go
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
package upstream
|
||||
|
||||
import "time"
|
||||
|
||||
// DefaultBranchCacheTimeout specifies the timeout for the default branch cache. It can be quite long.
|
||||
var DefaultBranchCacheTimeout = 15 * time.Minute
|
||||
|
||||
// BranchExistanceCacheTimeout specifies the timeout for the branch timestamp & existance cache. It should be shorter
|
||||
// than FileCacheTimeout, as that gets invalidated if the branch timestamp has changed. That way, repo changes will be
|
||||
// picked up faster, while still allowing the content to be cached longer if nothing changes.
|
||||
var BranchExistanceCacheTimeout = 5 * time.Minute
|
||||
|
||||
// FileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending
|
||||
// on your available memory.
|
||||
var FileCacheTimeout = 5 * time.Minute
|
||||
|
||||
// FileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default.
|
||||
var FileCacheSizeLimit = 1024 * 1024
|
||||
53
server/upstream/helper.go
Normal file
53
server/upstream/helper.go
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
package upstream
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/valyala/fasthttp"
|
||||
"github.com/valyala/fastjson"
|
||||
)
|
||||
|
||||
type branchTimestamp struct {
|
||||
Branch string
|
||||
Timestamp time.Time
|
||||
}
|
||||
|
||||
// GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch
|
||||
// (or nil if the branch doesn't exist)
|
||||
func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string) *branchTimestamp {
|
||||
if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok {
|
||||
if result == nil {
|
||||
return nil
|
||||
}
|
||||
return result.(*branchTimestamp)
|
||||
}
|
||||
result := &branchTimestamp{}
|
||||
result.Branch = branch
|
||||
if branch == "" {
|
||||
// Get default branch
|
||||
var body = make([]byte, 0)
|
||||
// TODO: use header for API key?
|
||||
status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"?access_token="+giteaApiToken, 5*time.Second)
|
||||
if err != nil || status != 200 {
|
||||
_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, nil, DefaultBranchCacheTimeout)
|
||||
return nil
|
||||
}
|
||||
result.Branch = fastjson.GetString(body, "default_branch")
|
||||
}
|
||||
|
||||
var body = make([]byte, 0)
|
||||
status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"/branches/"+branch+"?access_token="+giteaApiToken, 5*time.Second)
|
||||
if err != nil || status != 200 {
|
||||
return nil
|
||||
}
|
||||
|
||||
result.Timestamp, _ = time.Parse(time.RFC3339, fastjson.GetString(body, "commit", "timestamp"))
|
||||
_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, BranchExistanceCacheTimeout)
|
||||
return result
|
||||
}
|
||||
|
||||
type fileResponse struct {
|
||||
exists bool
|
||||
mimeType string
|
||||
body []byte
|
||||
}
|
||||
193
server/upstream/upstream.go
Normal file
193
server/upstream/upstream.go
Normal file
|
|
@ -0,0 +1,193 @@
|
|||
package upstream
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"codeberg.org/codeberg/pages/html"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/valyala/fasthttp"
|
||||
)
|
||||
|
||||
// upstreamIndexPages lists pages that may be considered as index pages for directories.
|
||||
var upstreamIndexPages = []string{
|
||||
"index.html",
|
||||
}
|
||||
|
||||
// Options provides various options for the upstream request.
|
||||
type Options struct {
|
||||
DefaultMimeType string
|
||||
ForbiddenMimeTypes map[string]struct{}
|
||||
TryIndexPages bool
|
||||
AppendTrailingSlash bool
|
||||
RedirectIfExists string
|
||||
BranchTimestamp time.Time
|
||||
}
|
||||
|
||||
var Client = fasthttp.Client{
|
||||
ReadTimeout: 10 * time.Second,
|
||||
MaxConnDuration: 60 * time.Second,
|
||||
MaxConnWaitTimeout: 1000 * time.Millisecond,
|
||||
MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
|
||||
}
|
||||
|
||||
// Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
|
||||
func Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaApiToken string, options *Options) (final bool) {
|
||||
log := log.With().Strs("upstream", []string{targetOwner, targetRepo, targetBranch, targetPath}).Logger()
|
||||
|
||||
if options.ForbiddenMimeTypes == nil {
|
||||
options.ForbiddenMimeTypes = map[string]struct{}{}
|
||||
}
|
||||
|
||||
// Check if the branch exists and when it was modified
|
||||
if options.BranchTimestamp == (time.Time{}) {
|
||||
branch := GetBranchTimestamp(targetOwner, targetRepo, targetBranch, giteaRoot, giteaApiToken)
|
||||
|
||||
if branch == nil {
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
return true
|
||||
}
|
||||
targetBranch = branch.Branch
|
||||
options.BranchTimestamp = branch.Timestamp
|
||||
}
|
||||
|
||||
if targetOwner == "" || targetRepo == "" || targetBranch == "" {
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusBadRequest)
|
||||
return true
|
||||
}
|
||||
|
||||
// Check if the browser has a cached version
|
||||
if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Request.Header.Peek("If-Modified-Since"))); err == nil {
|
||||
if !ifModifiedSince.Before(options.BranchTimestamp) {
|
||||
ctx.Response.SetStatusCode(fasthttp.StatusNotModified)
|
||||
return true
|
||||
}
|
||||
}
|
||||
log.Debug().Msg("preparations")
|
||||
|
||||
// Make a GET request to the upstream URL
|
||||
uri := targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/" + targetPath
|
||||
var req *fasthttp.Request
|
||||
var res *fasthttp.Response
|
||||
var cachedResponse fileResponse
|
||||
var err error
|
||||
if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + strconv.FormatInt(options.BranchTimestamp.Unix(), 10)); ok && len(cachedValue.(fileResponse).body) > 0 {
|
||||
cachedResponse = cachedValue.(fileResponse)
|
||||
} else {
|
||||
req = fasthttp.AcquireRequest()
|
||||
req.SetRequestURI(giteaRoot + "/api/v1/repos/" + uri + "?access_token=" + giteaApiToken)
|
||||
res = fasthttp.AcquireResponse()
|
||||
res.SetBodyStream(&strings.Reader{}, -1)
|
||||
err = Client.Do(req, res)
|
||||
}
|
||||
log.Debug().Msg("acquisition")
|
||||
|
||||
// Handle errors
|
||||
if (res == nil && !cachedResponse.exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) {
|
||||
if options.TryIndexPages {
|
||||
// copy the options struct & try if an index page exists
|
||||
optionsForIndexPages := *options
|
||||
optionsForIndexPages.TryIndexPages = false
|
||||
optionsForIndexPages.AppendTrailingSlash = true
|
||||
for _, indexPage := range upstreamIndexPages {
|
||||
if Upstream(ctx, targetOwner, targetRepo, targetBranch, strings.TrimSuffix(targetPath, "/")+"/"+indexPage, giteaRoot, giteaApiToken, &optionsForIndexPages) {
|
||||
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{
|
||||
exists: false,
|
||||
}, FileCacheTimeout)
|
||||
return true
|
||||
}
|
||||
}
|
||||
// compatibility fix for GitHub Pages (/example → /example.html)
|
||||
optionsForIndexPages.AppendTrailingSlash = false
|
||||
optionsForIndexPages.RedirectIfExists = string(ctx.Request.URI().Path()) + ".html"
|
||||
if Upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath+".html", giteaRoot, giteaApiToken, &optionsForIndexPages) {
|
||||
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{
|
||||
exists: false,
|
||||
}, FileCacheTimeout)
|
||||
return true
|
||||
}
|
||||
}
|
||||
ctx.Response.SetStatusCode(fasthttp.StatusNotFound)
|
||||
if res != nil {
|
||||
// Update cache if the request is fresh
|
||||
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{
|
||||
exists: false,
|
||||
}, FileCacheTimeout)
|
||||
}
|
||||
return false
|
||||
}
|
||||
if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) {
|
||||
fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", req.RequestURI(), err, res.StatusCode())
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
|
||||
// Append trailing slash if missing (for index files), and redirect to fix filenames in general
|
||||
// options.AppendTrailingSlash is only true when looking for index pages
|
||||
if options.AppendTrailingSlash && !bytes.HasSuffix(ctx.Request.URI().Path(), []byte{'/'}) {
|
||||
ctx.Redirect(string(ctx.Request.URI().Path())+"/", fasthttp.StatusTemporaryRedirect)
|
||||
return true
|
||||
}
|
||||
if bytes.HasSuffix(ctx.Request.URI().Path(), []byte("/index.html")) {
|
||||
ctx.Redirect(strings.TrimSuffix(string(ctx.Request.URI().Path()), "index.html"), fasthttp.StatusTemporaryRedirect)
|
||||
return true
|
||||
}
|
||||
if options.RedirectIfExists != "" {
|
||||
ctx.Redirect(options.RedirectIfExists, fasthttp.StatusTemporaryRedirect)
|
||||
return true
|
||||
}
|
||||
log.Debug().Msg("error handling")
|
||||
|
||||
// Set the MIME type
|
||||
mimeType := mime.TypeByExtension(path.Ext(targetPath))
|
||||
mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
|
||||
if _, ok := options.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" {
|
||||
if options.DefaultMimeType != "" {
|
||||
mimeType = options.DefaultMimeType
|
||||
} else {
|
||||
mimeType = "application/octet-stream"
|
||||
}
|
||||
}
|
||||
ctx.Response.Header.SetContentType(mimeType)
|
||||
|
||||
// Everything's okay so far
|
||||
ctx.Response.SetStatusCode(fasthttp.StatusOK)
|
||||
ctx.Response.Header.SetLastModified(options.BranchTimestamp)
|
||||
|
||||
log.Debug().Msg("response preparations")
|
||||
|
||||
// Write the response body to the original request
|
||||
var cacheBodyWriter bytes.Buffer
|
||||
if res != nil {
|
||||
if res.Header.ContentLength() > FileCacheSizeLimit {
|
||||
err = res.BodyWriteTo(ctx.Response.BodyWriter())
|
||||
} else {
|
||||
// TODO: cache is half-empty if request is cancelled - does the ctx.Err() below do the trick?
|
||||
err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter))
|
||||
}
|
||||
} else {
|
||||
_, err = ctx.Write(cachedResponse.body)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Printf("Couldn't write body for \"%s\": %s\n", req.RequestURI(), err)
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
log.Debug().Msg("response")
|
||||
|
||||
if res != nil && ctx.Err() == nil {
|
||||
cachedResponse.exists = true
|
||||
cachedResponse.mimeType = mimeType
|
||||
cachedResponse.body = cacheBodyWriter.Bytes()
|
||||
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), cachedResponse, FileCacheTimeout)
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
Loading…
Reference in a new issue