wip

cmd/main.go

@@ -18,6 +18,7 @@ import (
 	"codeberg.org/codeberg/pages/server/cache"
 	"codeberg.org/codeberg/pages/server/certificates"
 	"codeberg.org/codeberg/pages/server/database"
+	"codeberg.org/codeberg/pages/server/gitea"
 )
 
 // AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed.
@@ -81,9 +82,12 @@ func Serve(ctx *cli.Context) error {
 	// TODO: make this an MRU cache with a size limit
 	fileResponseCache := cache.NewKeyValueCache()
 
+	giteaClient := gitea.NewClient(giteaRoot, giteaAPIToken)
+
 	// Create handler based on settings
 	handler := server.Handler(mainDomainSuffix, []byte(rawDomain),
-		giteaRoot, rawInfoPage, giteaAPIToken,
+		giteaClient,
+		giteaRoot, rawInfoPage,
 		BlacklistedPaths, allowedCorsDomains,
 		dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache)
 

server/gitea/client.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net/url"
 	"path"
+	"strings"
 	"time"
 
 	"github.com/valyala/fasthttp"
@@ -12,11 +13,21 @@ import (
 	"codeberg.org/codeberg/pages/server/shared"
 )
 
+const giteaAPIRepos = "/api/v1/repos/"
+
 type Client struct {
 	giteaRoot     string
 	giteaAPIToken string
 }
 
+type FileResponse struct {
+	Exists   bool
+	MimeType string
+	Body     []byte
+}
+
+func (f FileResponse) IsEmpty() bool { return len(f.Body) != 0 }
+
 func NewClient(giteaRoot, giteaAPIToken string) *Client {
 	return &Client{
 		giteaRoot:     giteaRoot,
@@ -24,8 +35,6 @@ func NewClient(giteaRoot, giteaAPIToken string) *Client {
 	}
 }
 
-const giteaAPIRepos = "/api/v1/repos/"
-
 // TODOs:
 // * own client to store token & giteaRoot
 // * handle 404 -> page will show 500 atm
@@ -46,6 +55,19 @@ func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource str
 	return res.Body(), nil
 }
 
+func (client *Client) ServeRawContent(uri string) (*fasthttp.Response, error) {
+	fastClient := shared.GetFastHTTPClient(10 * time.Second)
+
+	req := fasthttp.AcquireRequest()
+	req.SetRequestURI(path.Join(client.giteaRoot, giteaAPIRepos, uri))
+	req.Header.Set(fasthttp.HeaderAuthorization, client.giteaAPIToken)
+
+	resp := fasthttp.AcquireResponse()
+	resp.SetBodyStream(&strings.Reader{}, -1)
+
+	return resp, fastClient.Do(req, resp)
+}
+
 func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchName string) (time.Time, error) {
 	fastClient := shared.GetFastHTTPClient(5 * time.Second)
 

server/gitea/fasthttp.go

@@ -0,0 +1,16 @@
+package gitea
+
+import (
+	"time"
+
+	"github.com/valyala/fasthttp"
+)
+
+func getFastHTTPClient(timeout time.Duration) *fasthttp.Client {
+	return &fasthttp.Client{
+		ReadTimeout:        timeout,
+		MaxConnDuration:    60 * time.Second,
+		MaxConnWaitTimeout: 1000 * time.Millisecond,
+		MaxConnsPerHost:    128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
+	}
+}

server/handler.go

@@ -17,7 +17,8 @@ import (
 
 // Handler handles a single HTTP request to the web server.
 func Handler(mainDomainSuffix, rawDomain []byte,
-	giteaRoot, rawInfoPage, giteaAPIToken string,
+	giteaClient *gitea.Client,
+	giteaRoot, rawInfoPage string,
 	blacklistedPaths, allowedCorsDomains [][]byte,
 	dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
 ) func(ctx *fasthttp.RequestCtx) {

server/handler_test.go

@@ -8,15 +8,16 @@ import (
 	"github.com/valyala/fasthttp"
 
 	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/gitea"
 )
 
 func TestHandlerPerformance(t *testing.T) {
+	giteaRoot := "https://codeberg.org"
+	giteaClient := gitea.NewClient(giteaRoot, "")
 	testHandler := Handler(
-		[]byte("codeberg.page"),
+		[]byte("codeberg.page"), []byte("raw.codeberg.org"),
-		[]byte("raw.codeberg.org"),
+		giteaClient,
-		"https://codeberg.org",
+		giteaRoot, "https://docs.codeberg.org/pages/raw-content/",
-		"https://docs.codeberg.org/pages/raw-content/",
-		"",
 		[][]byte{[]byte("/.well-known/acme-challenge/")},
 		[][]byte{[]byte("raw.codeberg.org"), []byte("fonts.codeberg.org"), []byte("design.codeberg.org")},
 		cache.NewKeyValueCache(),

server/upstream/gitea.go

@@ -1,69 +0,0 @@
-package upstream
-
-import (
-	"fmt"
-	"net/url"
-	"path"
-	"time"
-
-	"github.com/valyala/fasthttp"
-	"github.com/valyala/fastjson"
-
-	"codeberg.org/codeberg/pages/server/gitea"
-)
-
-const giteaAPIRepos = "/api/v1/repos/"
-
-// TODOs:
-// * own client to store token & giteaRoot
-// * handle 404 -> page will show 500 atm
-
-func giteaRawContent(client *gitea.Client, targetOwner, targetRepo, ref, giteaRoot, giteaAPIToken, resource string) ([]byte, error) {
-	req := fasthttp.AcquireRequest()
-
-	req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref)))
-	req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken)
-	res := fasthttp.AcquireResponse()
-
-	if err := getFastHTTPClient(10*time.Second).Do(req, res); err != nil {
-		return nil, err
-	}
-	if res.StatusCode() != fasthttp.StatusOK {
-		return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
-	}
-	return res.Body(), nil
-}
-
-func giteaGetRepoBranchTimestamp(giteaRoot, repoOwner, repoName, branchName, giteaAPIToken string) (time.Time, error) {
-	client := getFastHTTPClient(5 * time.Second)
-
-	req := fasthttp.AcquireRequest()
-	req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, repoOwner, repoName, "branches", branchName))
-	req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken)
-	res := fasthttp.AcquireResponse()
-
-	if err := client.Do(req, res); err != nil {
-		return time.Time{}, err
-	}
-	if res.StatusCode() != fasthttp.StatusOK {
-		return time.Time{}, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
-	}
-	return time.Parse(time.RFC3339, fastjson.GetString(res.Body(), "commit", "timestamp"))
-}
-
-func giteaGetRepoDefaultBranch(giteaRoot, repoOwner, repoName, giteaAPIToken string) (string, error) {
-	client := getFastHTTPClient(5 * time.Second)
-
-	req := fasthttp.AcquireRequest()
-	req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, repoOwner, repoName))
-	req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken)
-	res := fasthttp.AcquireResponse()
-
-	if err := client.Do(req, res); err != nil {
-		return "", err
-	}
-	if res.StatusCode() != fasthttp.StatusOK {
-		return "", fmt.Errorf("unexpected status code '%d'", res.StatusCode())
-	}
-	return fastjson.GetString(res.Body(), "default_branch"), nil
-}

server/upstream/helper.go

@@ -1,6 +1,10 @@
 package upstream
 
 import (
+	"mime"
+	"path"
+	"strconv"
+	"strings"
 	"time"
 
 	"codeberg.org/codeberg/pages/server/cache"
@@ -43,8 +47,23 @@ func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string, b
 	return result
 }
 
-type fileResponse struct {
+func (o *Options) getMimeTypeByExtension() string {
-	exists   bool
+	mimeType := mime.TypeByExtension(path.Ext(o.TargetPath))
-	mimeType string
+	mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
-	body     []byte
+	if _, ok := o.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" {
+		if o.DefaultMimeType != "" {
+			mimeType = o.DefaultMimeType
+		} else {
+			mimeType = "application/octet-stream"
+		}
+	}
+	return mimeType
+}
+
+func (o *Options) generateUri() string {
+	return path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath)
+}
+
+func (o *Options) timestamp() string {
+	return strconv.FormatInt(o.BranchTimestamp.Unix(), 10)
 }

server/upstream/upstream.go

@@ -4,9 +4,6 @@ import (
 	"bytes"
 	"fmt"
 	"io"
-	"mime"
-	"path"
-	"strconv"
 	"strings"
 	"time"
 
@@ -39,15 +36,6 @@ type Options struct {
 	redirectIfExists    string
 }
 
-func getFastHTTPClient(timeout time.Duration) *fasthttp.Client {
-	return &fasthttp.Client{
-		ReadTimeout:        timeout,
-		MaxConnDuration:    60 * time.Second,
-		MaxConnWaitTimeout: 1000 * time.Millisecond,
-		MaxConnsPerHost:    128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
-	}
-}
-
 // Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
 func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) {
 	log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger()
@@ -83,25 +71,19 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
 	log.Debug().Msg("preparations")
 
 	// Make a GET request to the upstream URL
-	uri := path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath)
+	uri := o.generateUri()
-	var req *fasthttp.Request
 	var res *fasthttp.Response
-	var cachedResponse fileResponse
+	var cachedResponse gitea.FileResponse
 	var err error
-	if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + strconv.FormatInt(o.BranchTimestamp.Unix(), 10)); ok && len(cachedValue.(fileResponse).body) > 0 {
+	if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() {
-		cachedResponse = cachedValue.(fileResponse)
+		cachedResponse = cachedValue.(gitea.FileResponse)
 	} else {
-		req = fasthttp.AcquireRequest()
+		res, err = giteaClient.ServeRawContent(uri)
-		req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, uri))
-		req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken)
-		res = fasthttp.AcquireResponse()
-		res.SetBodyStream(&strings.Reader{}, -1)
-		err = getFastHTTPClient(10*time.Second).Do(req, res)
 	}
 	log.Debug().Msg("acquisition")
 
 	// Handle errors
-	if (res == nil && !cachedResponse.exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) {
+	if (res == nil && !cachedResponse.Exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) {
 		if o.TryIndexPages {
 			// copy the o struct & try if an index page exists
 			optionsForIndexPages := *o
@@ -110,8 +92,8 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
 			for _, indexPage := range upstreamIndexPages {
 				optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage
 				if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
-					_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
+					_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
-						exists: false,
+						Exists: false,
 					}, fileCacheTimeout)
 					return true
 				}
@@ -121,8 +103,8 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
 			optionsForIndexPages.redirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html"
 			optionsForIndexPages.TargetPath = o.TargetPath + ".html"
 			if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
-				_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
+				_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
-					exists: false,
+					Exists: false,
 				}, fileCacheTimeout)
 				return true
 			}
@@ -130,14 +112,14 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
 		ctx.Response.SetStatusCode(fasthttp.StatusNotFound)
 		if res != nil {
 			// Update cache if the request is fresh
-			_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
+			_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
-				exists: false,
+				Exists: false,
 			}, fileCacheTimeout)
 		}
 		return false
 	}
 	if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) {
-		fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", req.RequestURI(), err, res.StatusCode())
+		fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", uri, err, res.StatusCode())
 		html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
 		return true
 	}
@@ -159,15 +141,7 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
 	log.Debug().Msg("error handling")
 
 	// Set the MIME type
-	mimeType := mime.TypeByExtension(path.Ext(o.TargetPath))
+	mimeType := o.getMimeTypeByExtension()
-	mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
-	if _, ok := o.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" {
-		if o.DefaultMimeType != "" {
-			mimeType = o.DefaultMimeType
-		} else {
-			mimeType = "application/octet-stream"
-		}
-	}
 	ctx.Response.Header.SetContentType(mimeType)
 
 	// Everything's okay so far
@@ -186,20 +160,20 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
 			err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter))
 		}
 	} else {
-		_, err = ctx.Write(cachedResponse.body)
+		_, err = ctx.Write(cachedResponse.Body)
 	}
 	if err != nil {
-		fmt.Printf("Couldn't write body for \"%s\": %s\n", req.RequestURI(), err)
+		fmt.Printf("Couldn't write body for \"%s\": %s\n", uri, err)
 		html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
 		return true
 	}
 	log.Debug().Msg("response")
 
 	if res != nil && ctx.Err() == nil {
-		cachedResponse.exists = true
+		cachedResponse.Exists = true
-		cachedResponse.mimeType = mimeType
+		cachedResponse.MimeType = mimeType
-		cachedResponse.body = cacheBodyWriter.Bytes()
+		cachedResponse.Body = cacheBodyWriter.Bytes()
-		_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), cachedResponse, fileCacheTimeout)
+		_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), cachedResponse, fileCacheTimeout)
 	}
 
 	return true