pages-server/server/upstream/upstream.go

206 lines
7.1 KiB
Go
Raw Normal View History

2021-12-05 13:47:33 +00:00
package upstream
import (
"bytes"
"fmt"
"io"
"mime"
"path"
"strconv"
"strings"
"time"
"github.com/rs/zerolog/log"
"github.com/valyala/fasthttp"
2021-12-05 14:53:46 +00:00
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache"
2021-12-05 13:47:33 +00:00
)
// upstreamIndexPages lists pages that may be considered as index pages for directories.
var upstreamIndexPages = []string{
"index.html",
}
// Options provides various options for the upstream request.
type Options struct {
TargetOwner,
TargetRepo,
TargetBranch,
TargetPath,
DefaultMimeType string
2021-12-05 16:57:54 +00:00
ForbiddenMimeTypes map[string]struct{}
TryIndexPages bool
BranchTimestamp time.Time
// internal
appendTrailingSlash bool
redirectIfExists string
2021-12-05 13:47:33 +00:00
}
func getFastHTTPClient(timeout time.Duration) *fasthttp.Client {
return &fasthttp.Client{
ReadTimeout: timeout,
MaxConnDuration: 60 * time.Second,
MaxConnWaitTimeout: 1000 * time.Millisecond,
MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
}
2021-12-05 13:47:33 +00:00
}
// Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken string, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) {
log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger()
2021-12-05 13:47:33 +00:00
2021-12-05 16:57:54 +00:00
if o.ForbiddenMimeTypes == nil {
o.ForbiddenMimeTypes = map[string]struct{}{}
2021-12-05 13:47:33 +00:00
}
// Check if the branch exists and when it was modified
if o.BranchTimestamp.IsZero() {
branch := GetBranchTimestamp(o.TargetOwner, o.TargetRepo, o.TargetBranch, giteaRoot, giteaAPIToken, branchTimestampCache)
2021-12-05 13:47:33 +00:00
if branch == nil {
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
return true
}
o.TargetBranch = branch.Branch
2021-12-05 16:57:54 +00:00
o.BranchTimestamp = branch.Timestamp
2021-12-05 13:47:33 +00:00
}
if o.TargetOwner == "" || o.TargetRepo == "" || o.TargetBranch == "" {
2021-12-05 13:47:33 +00:00
html.ReturnErrorPage(ctx, fasthttp.StatusBadRequest)
return true
}
// Check if the browser has a cached version
if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Request.Header.Peek("If-Modified-Since"))); err == nil {
2021-12-05 16:57:54 +00:00
if !ifModifiedSince.Before(o.BranchTimestamp) {
2021-12-05 13:47:33 +00:00
ctx.Response.SetStatusCode(fasthttp.StatusNotModified)
return true
}
}
log.Debug().Msg("preparations")
// Make a GET request to the upstream URL
uri := path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath)
2021-12-05 13:47:33 +00:00
var req *fasthttp.Request
var res *fasthttp.Response
var cachedResponse fileResponse
var err error
2021-12-05 16:57:54 +00:00
if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + strconv.FormatInt(o.BranchTimestamp.Unix(), 10)); ok && len(cachedValue.(fileResponse).body) > 0 {
2021-12-05 13:47:33 +00:00
cachedResponse = cachedValue.(fileResponse)
} else {
req = fasthttp.AcquireRequest()
req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, uri))
req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken)
2021-12-05 13:47:33 +00:00
res = fasthttp.AcquireResponse()
res.SetBodyStream(&strings.Reader{}, -1)
err = getFastHTTPClient(10*time.Second).Do(req, res)
2021-12-05 13:47:33 +00:00
}
log.Debug().Msg("acquisition")
// Handle errors
if (res == nil && !cachedResponse.exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) {
2021-12-05 16:57:54 +00:00
if o.TryIndexPages {
// copy the o struct & try if an index page exists
optionsForIndexPages := *o
2021-12-05 13:47:33 +00:00
optionsForIndexPages.TryIndexPages = false
2021-12-05 16:57:54 +00:00
optionsForIndexPages.appendTrailingSlash = true
2021-12-05 13:47:33 +00:00
for _, indexPage := range upstreamIndexPages {
optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage
if optionsForIndexPages.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
2021-12-05 16:57:54 +00:00
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
2021-12-05 13:47:33 +00:00
exists: false,
2021-12-05 15:24:26 +00:00
}, fileCacheTimeout)
2021-12-05 13:47:33 +00:00
return true
}
}
// compatibility fix for GitHub Pages (/example → /example.html)
2021-12-05 16:57:54 +00:00
optionsForIndexPages.appendTrailingSlash = false
2021-12-05 21:50:46 +00:00
optionsForIndexPages.redirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html"
optionsForIndexPages.TargetPath = o.TargetPath + ".html"
if optionsForIndexPages.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
2021-12-05 16:57:54 +00:00
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
2021-12-05 13:47:33 +00:00
exists: false,
2021-12-05 15:24:26 +00:00
}, fileCacheTimeout)
2021-12-05 13:47:33 +00:00
return true
}
}
ctx.Response.SetStatusCode(fasthttp.StatusNotFound)
if res != nil {
// Update cache if the request is fresh
2021-12-05 16:57:54 +00:00
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
2021-12-05 13:47:33 +00:00
exists: false,
2021-12-05 15:24:26 +00:00
}, fileCacheTimeout)
2021-12-05 13:47:33 +00:00
}
return false
}
if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) {
fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", req.RequestURI(), err, res.StatusCode())
html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
return true
}
// Append trailing slash if missing (for index files), and redirect to fix filenames in general
2021-12-05 16:57:54 +00:00
// o.appendTrailingSlash is only true when looking for index pages
if o.appendTrailingSlash && !bytes.HasSuffix(ctx.Request.URI().Path(), []byte{'/'}) {
2021-12-05 13:47:33 +00:00
ctx.Redirect(string(ctx.Request.URI().Path())+"/", fasthttp.StatusTemporaryRedirect)
return true
}
if bytes.HasSuffix(ctx.Request.URI().Path(), []byte("/index.html")) {
ctx.Redirect(strings.TrimSuffix(string(ctx.Request.URI().Path()), "index.html"), fasthttp.StatusTemporaryRedirect)
return true
}
2021-12-05 16:57:54 +00:00
if o.redirectIfExists != "" {
ctx.Redirect(o.redirectIfExists, fasthttp.StatusTemporaryRedirect)
2021-12-05 13:47:33 +00:00
return true
}
log.Debug().Msg("error handling")
// Set the MIME type
mimeType := mime.TypeByExtension(path.Ext(o.TargetPath))
2021-12-05 13:47:33 +00:00
mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
2021-12-05 16:57:54 +00:00
if _, ok := o.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" {
if o.DefaultMimeType != "" {
mimeType = o.DefaultMimeType
2021-12-05 13:47:33 +00:00
} else {
mimeType = "application/octet-stream"
}
}
ctx.Response.Header.SetContentType(mimeType)
// Everything's okay so far
ctx.Response.SetStatusCode(fasthttp.StatusOK)
2021-12-05 16:57:54 +00:00
ctx.Response.Header.SetLastModified(o.BranchTimestamp)
2021-12-05 13:47:33 +00:00
log.Debug().Msg("response preparations")
// Write the response body to the original request
var cacheBodyWriter bytes.Buffer
if res != nil {
2021-12-05 15:24:26 +00:00
if res.Header.ContentLength() > fileCacheSizeLimit {
2021-12-05 13:47:33 +00:00
err = res.BodyWriteTo(ctx.Response.BodyWriter())
} else {
// TODO: cache is half-empty if request is cancelled - does the ctx.Err() below do the trick?
err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter))
}
} else {
_, err = ctx.Write(cachedResponse.body)
}
if err != nil {
fmt.Printf("Couldn't write body for \"%s\": %s\n", req.RequestURI(), err)
html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
return true
}
log.Debug().Msg("response")
if res != nil && ctx.Err() == nil {
cachedResponse.exists = true
cachedResponse.mimeType = mimeType
cachedResponse.body = cacheBodyWriter.Bytes()
2021-12-05 16:57:54 +00:00
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), cachedResponse, fileCacheTimeout)
2021-12-05 13:47:33 +00:00
}
return true
}