Move gitea api calls in own "client" package (#78)

continue #75
close #16
- fix regression (from #34) _thanks to @crystal_
- create own gitea client package
- more logging
- add mock impl of CertDB

Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: crystal <crystal@noreply.codeberg.org>
Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/78
Reviewed-by: crapStone <crapstone@noreply.codeberg.org>

.woodpecker.yml

@@ -1,3 +1,5 @@
+branches: main
+
 pipeline:
   # use vendor to cache dependencies
   vendor:

Justfile

@@ -15,6 +15,9 @@ lint: tool-golangci tool-gofumpt
     [ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }; \
     golangci-lint run --timeout 5m
 
+fmt: tool-gofumpt
+    gofumpt -w --extra .
+
 tool-golangci:
     @hash golangci-lint> /dev/null 2>&1; if [ $? -ne 0 ]; then \
     go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest; \

cmd/certs.go

@@ -61,7 +61,7 @@ func removeCert(ctx *cli.Context) error {
 
 	for _, domain := range domains {
 		fmt.Printf("Removing domain %s from the database...\n", domain)
-		if err := keyDatabase.Delete([]byte(domain)); err != nil {
+		if err := keyDatabase.Delete(domain); err != nil {
 			return err
 		}
 	}

cmd/main.go

@@ -18,6 +18,7 @@ import (
 	"codeberg.org/codeberg/pages/server/cache"
 	"codeberg.org/codeberg/pages/server/certificates"
 	"codeberg.org/codeberg/pages/server/database"
+	"codeberg.org/codeberg/pages/server/gitea"
 )
 
 // AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed.
@@ -81,9 +82,12 @@ func Serve(ctx *cli.Context) error {
 	// TODO: make this an MRU cache with a size limit
 	fileResponseCache := cache.NewKeyValueCache()
 
+	giteaClient := gitea.NewClient(giteaRoot, giteaAPIToken)
+
 	// Create handler based on settings
 	handler := server.Handler(mainDomainSuffix, []byte(rawDomain),
-		giteaRoot, rawInfoPage, giteaAPIToken,
+		giteaClient,
+		giteaRoot, rawInfoPage,
 		BlacklistedPaths, allowedCorsDomains,
 		dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache)
 
@@ -105,7 +109,8 @@ func Serve(ctx *cli.Context) error {
 	defer certDB.Close() //nolint:errcheck    // database has no close ... sync behave like it
 
 	listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix,
-		giteaRoot, giteaAPIToken, dnsProvider,
+		giteaClient,
+		dnsProvider,
 		acmeUseRateLimits,
 		keyCache, challengeCache, dnsLookupCache, canonicalDomainCache,
 		certDB))
@@ -126,6 +131,7 @@ func Serve(ctx *cli.Context) error {
 
 	if enableHTTPServer {
 		go func() {
+			log.Info().Timestamp().Msg("Start listening on :80")
 			err := httpServer.ListenAndServe("[::]:80")
 			if err != nil {
 				log.Panic().Err(err).Msg("Couldn't start HTTP fastServer")
@@ -134,6 +140,7 @@ func Serve(ctx *cli.Context) error {
 	}
 
 	// Start the web fastServer
+	log.Info().Timestamp().Msgf("Start listening on %s", listener.Addr())
 	err = fastServer.Serve(listener)
 	if err != nil {
 		log.Panic().Err(err).Msg("Couldn't start fastServer")

server/certificates/certificates.go

@@ -32,12 +32,14 @@ import (
 	"codeberg.org/codeberg/pages/server/cache"
 	"codeberg.org/codeberg/pages/server/database"
 	dnsutils "codeberg.org/codeberg/pages/server/dns"
+	"codeberg.org/codeberg/pages/server/gitea"
 	"codeberg.org/codeberg/pages/server/upstream"
 )
 
 // TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates.
 func TLSConfig(mainDomainSuffix []byte,
-	giteaRoot, giteaAPIToken, dnsProvider string,
+	giteaClient *gitea.Client,
+	dnsProvider string,
 	acmeUseRateLimits bool,
 	keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey,
 	certDB database.CertDB,
@@ -81,7 +83,7 @@ func TLSConfig(mainDomainSuffix []byte,
 					sni = string(sniBytes)
 				} else {
 					_, _ = targetRepo, targetBranch
-					_, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)
+					_, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), canonicalDomainCache)
 					if !valid {
 						sniBytes = mainDomainSuffix
 						sni = string(sniBytes)
@@ -193,7 +195,7 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
 
 func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (tls.Certificate, bool) {
 	// parse certificate from database
-	res, err := certDB.Get(sni)
+	res, err := certDB.Get(string(sni))
 	if err != nil {
 		panic(err) // TODO: no panic
 	}
@@ -406,7 +408,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
 
 func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) error {
 	// getting main cert before ACME account so that we can fail here without hitting rate limits
-	mainCertBytes, err := certDB.Get(mainDomainSuffix)
+	mainCertBytes, err := certDB.Get(string(mainDomainSuffix))
 	if err != nil {
 		return fmt.Errorf("cert database is not working")
 	}
@@ -478,7 +480,7 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
 
 				tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
 				if err != nil || !tlsCertificates[0].NotAfter.After(now) {
-					err := certDB.Delete(key)
+					err := certDB.Delete(string(key))
 					if err != nil {
 						log.Printf("[ERROR] Deleting expired certificate for %s failed: %s", string(key), err)
 					} else {
@@ -491,15 +493,15 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
 		log.Printf("[INFO] Removed %d expired certificates from the database", expiredCertCount)
 
 		// compact the database
-		result, err := certDB.Compact()
+		msg, err := certDB.Compact()
 		if err != nil {
 			log.Printf("[ERROR] Compacting key database failed: %s", err)
 		} else {
-			log.Printf("[INFO] Compacted key database (%+v)", result)
+			log.Printf("[INFO] Compacted key database (%s)", msg)
 		}
 
 		// update main cert
-		res, err := certDB.Get(mainDomainSuffix)
+		res, err := certDB.Get(string(mainDomainSuffix))
 		if err != nil {
 			log.Err(err).Msgf("could not get cert for domain '%s'", mainDomainSuffix)
 		} else if res == nil {

server/certificates/mock_test.go

@@ -0,0 +1,17 @@
+package certificates
+
+import (
+	"testing"
+
+	"codeberg.org/codeberg/pages/server/database"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMockCert(t *testing.T) {
+	db, err := database.NewTmpDB()
+	assert.NoError(t, err)
+	cert := mockCert("example.com", "some error msg", "codeberg.page", db)
+	if assert.NotEmpty(t, cert) {
+		assert.NotEmpty(t, cert.Certificate)
+	}
+}

server/database/interface.go

@@ -8,8 +8,8 @@ import (
 type CertDB interface {
 	Close() error
 	Put(name string, cert *certificate.Resource) error
-	Get(name []byte) (*certificate.Resource, error)
+	Get(name string) (*certificate.Resource, error)
-	Delete(key []byte) error
+	Delete(key string) error
-	Compact() (pogreb.CompactionResult, error)
+	Compact() (string, error)
 	Items() *pogreb.ItemIterator
 }

server/database/mock.go

@@ -0,0 +1,55 @@
+package database
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/OrlovEvgeny/go-mcache"
+	"github.com/akrylysov/pogreb"
+	"github.com/go-acme/lego/v4/certificate"
+)
+
+var _ CertDB = tmpDB{}
+
+type tmpDB struct {
+	intern *mcache.CacheDriver
+	ttl    time.Duration
+}
+
+func (p tmpDB) Close() error {
+	_ = p.intern.Close()
+	return nil
+}
+
+func (p tmpDB) Put(name string, cert *certificate.Resource) error {
+	return p.intern.Set(name, cert, p.ttl)
+}
+
+func (p tmpDB) Get(name string) (*certificate.Resource, error) {
+	cert, has := p.intern.Get(name)
+	if !has {
+		return nil, fmt.Errorf("cert for '%s' not found", name)
+	}
+	return cert.(*certificate.Resource), nil
+}
+
+func (p tmpDB) Delete(key string) error {
+	p.intern.Remove(key)
+	return nil
+}
+
+func (p tmpDB) Compact() (string, error) {
+	p.intern.Truncate()
+	return "Truncate done", nil
+}
+
+func (p tmpDB) Items() *pogreb.ItemIterator {
+	panic("ItemIterator not implemented for tmpDB")
+}
+
+func NewTmpDB() (CertDB, error) {
+	return &tmpDB{
+		intern: mcache.New(),
+		ttl:    time.Minute,
+	}, nil
+}

server/database/setup.go

@@ -35,9 +35,9 @@ func (p aDB) Put(name string, cert *certificate.Resource) error {
 	return p.intern.Put([]byte(name), resGob.Bytes())
 }
 
-func (p aDB) Get(name []byte) (*certificate.Resource, error) {
+func (p aDB) Get(name string) (*certificate.Resource, error) {
 	cert := &certificate.Resource{}
-	resBytes, err := p.intern.Get(name)
+	resBytes, err := p.intern.Get([]byte(name))
 	if err != nil {
 		return nil, err
 	}
@@ -50,12 +50,16 @@ func (p aDB) Get(name []byte) (*certificate.Resource, error) {
 	return cert, nil
 }
 
-func (p aDB) Delete(key []byte) error {
+func (p aDB) Delete(key string) error {
-	return p.intern.Delete(key)
+	return p.intern.Delete([]byte(key))
 }
 
-func (p aDB) Compact() (pogreb.CompactionResult, error) {
+func (p aDB) Compact() (string, error) {
-	return p.intern.Compact()
+	result, err := p.intern.Compact()
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("%+v", result), nil
 }
 
 func (p aDB) Items() *pogreb.ItemIterator {

server/gitea/client.go

@@ -0,0 +1,119 @@
+package gitea
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"path"
+	"time"
+
+	"github.com/valyala/fasthttp"
+	"github.com/valyala/fastjson"
+)
+
+const giteaAPIRepos = "/api/v1/repos/"
+
+var ErrorNotFound = errors.New("not found")
+
+type Client struct {
+	giteaRoot      string
+	giteaAPIToken  string
+	fastClient     *fasthttp.Client
+	infoTimeout    time.Duration
+	contentTimeout time.Duration
+}
+
+type FileResponse struct {
+	Exists   bool
+	MimeType string
+	Body     []byte
+}
+
+func joinURL(giteaRoot string, paths ...string) string { return giteaRoot + path.Join(paths...) }
+
+func (f FileResponse) IsEmpty() bool { return len(f.Body) != 0 }
+
+func NewClient(giteaRoot, giteaAPIToken string) *Client {
+	return &Client{
+		giteaRoot:      giteaRoot,
+		giteaAPIToken:  giteaAPIToken,
+		infoTimeout:    5 * time.Second,
+		contentTimeout: 10 * time.Second,
+		fastClient:     getFastHTTPClient(),
+	}
+}
+
+func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource string) ([]byte, error) {
+	url := joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref))
+	res, err := client.do(client.contentTimeout, url)
+	if err != nil {
+		return nil, err
+	}
+
+	switch res.StatusCode() {
+	case fasthttp.StatusOK:
+		return res.Body(), nil
+	case fasthttp.StatusNotFound:
+		return nil, ErrorNotFound
+	default:
+		return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
+	}
+}
+
+func (client *Client) ServeRawContent(uri string) (*fasthttp.Response, error) {
+	url := joinURL(client.giteaRoot, giteaAPIRepos, uri)
+	res, err := client.do(client.contentTimeout, url)
+	if err != nil {
+		return nil, err
+	}
+	// resp.SetBodyStream(&strings.Reader{}, -1)
+
+	if err != nil {
+		return nil, err
+	}
+
+	switch res.StatusCode() {
+	case fasthttp.StatusOK:
+		return res, nil
+	case fasthttp.StatusNotFound:
+		return nil, ErrorNotFound
+	default:
+		return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
+	}
+}
+
+func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchName string) (time.Time, error) {
+	url := joinURL(client.giteaRoot, giteaAPIRepos, repoOwner, repoName, "branches", branchName)
+	res, err := client.do(client.infoTimeout, url)
+	if err != nil {
+		return time.Time{}, err
+	}
+	if res.StatusCode() != fasthttp.StatusOK {
+		return time.Time{}, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
+	}
+	return time.Parse(time.RFC3339, fastjson.GetString(res.Body(), "commit", "timestamp"))
+}
+
+func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (string, error) {
+	url := joinURL(client.giteaRoot, giteaAPIRepos, repoOwner, repoName)
+	res, err := client.do(client.infoTimeout, url)
+	if err != nil {
+		return "", err
+	}
+	if res.StatusCode() != fasthttp.StatusOK {
+		return "", fmt.Errorf("unexpected status code '%d'", res.StatusCode())
+	}
+	return fastjson.GetString(res.Body(), "default_branch"), nil
+}
+
+func (client *Client) do(timeout time.Duration, url string) (*fasthttp.Response, error) {
+	req := fasthttp.AcquireRequest()
+
+	req.SetRequestURI(url)
+	req.Header.Set(fasthttp.HeaderAuthorization, "token "+client.giteaAPIToken)
+	res := fasthttp.AcquireResponse()
+
+	err := client.fastClient.DoTimeout(req, res, timeout)
+
+	return res, err
+}

server/gitea/client_test.go

@@ -0,0 +1,23 @@
+package gitea
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestJoinURL(t *testing.T) {
+	url := joinURL("")
+	assert.EqualValues(t, "", url)
+
+	url = joinURL("", "", "")
+	assert.EqualValues(t, "", url)
+
+	url = joinURL("http://wwow.url.com", "a", "b/c/", "d")
+	// assert.EqualValues(t, "http://wwow.url.com/a/b/c/d", url)
+	assert.EqualValues(t, "http://wwow.url.coma/b/c/d", url)
+
+	url = joinURL("h:://wrong", "acdc")
+	// assert.EqualValues(t, "h:://wrong/acdc", url)
+	assert.EqualValues(t, "h:://wrongacdc", url)
+}

server/gitea/fasthttp.go

@@ -0,0 +1,15 @@
+package gitea
+
+import (
+	"time"
+
+	"github.com/valyala/fasthttp"
+)
+
+func getFastHTTPClient() *fasthttp.Client {
+	return &fasthttp.Client{
+		MaxConnDuration:    60 * time.Second,
+		MaxConnWaitTimeout: 1000 * time.Millisecond,
+		MaxConnsPerHost:    128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
+	}
+}

server/handler.go

@@ -4,19 +4,22 @@ import (
 	"bytes"
 	"strings"
 
+	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"github.com/valyala/fasthttp"
 
 	"codeberg.org/codeberg/pages/html"
 	"codeberg.org/codeberg/pages/server/cache"
 	"codeberg.org/codeberg/pages/server/dns"
+	"codeberg.org/codeberg/pages/server/gitea"
 	"codeberg.org/codeberg/pages/server/upstream"
 	"codeberg.org/codeberg/pages/server/utils"
 )
 
 // Handler handles a single HTTP request to the web server.
 func Handler(mainDomainSuffix, rawDomain []byte,
-	giteaRoot, rawInfoPage, giteaAPIToken string,
+	giteaClient *gitea.Client,
+	giteaRoot, rawInfoPage string,
 	blacklistedPaths, allowedCorsDomains [][]byte,
 	dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
 ) func(ctx *fasthttp.RequestCtx) {
@@ -74,21 +77,21 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 		// Prepare request information to Gitea
 		var targetOwner, targetRepo, targetBranch, targetPath string
 		targetOptions := &upstream.Options{
-			ForbiddenMimeTypes: map[string]struct{}{},
+			TryIndexPages: true,
-			TryIndexPages:      true,
 		}
 
 		// tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will
 		// also disallow search indexing and add a Link header to the canonical URL.
-		tryBranch := func(repo, branch string, path []string, canonicalLink string) bool {
+		tryBranch := func(log zerolog.Logger, repo, branch string, path []string, canonicalLink string) bool {
 			if repo == "" {
+				log.Debug().Msg("tryBranch: repo == ''")
 				return false
 			}
 
 			// Check if the branch exists, otherwise treat it as a file path
-			branchTimestampResult := upstream.GetBranchTimestamp(targetOwner, repo, branch, giteaRoot, giteaAPIToken, branchTimestampCache)
+			branchTimestampResult := upstream.GetBranchTimestamp(giteaClient, targetOwner, repo, branch, branchTimestampCache)
 			if branchTimestampResult == nil {
-				// branch doesn't exist
+				log.Debug().Msg("tryBranch: branch doesn't exist")
 				return false
 			}
 
@@ -108,6 +111,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 				)
 			}
 
+			log.Debug().Msg("tryBranch: true")
 			return true
 		}
 
@@ -117,7 +121,10 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 			log.Debug().Msg("raw domain")
 
 			targetOptions.TryIndexPages = false
-			targetOptions.ForbiddenMimeTypes["text/html"] = struct{}{}
+			if targetOptions.ForbiddenMimeTypes == nil {
+				targetOptions.ForbiddenMimeTypes = make(map[string]bool)
+			}
+			targetOptions.ForbiddenMimeTypes["text/html"] = true
 			targetOptions.DefaultMimeType = "text/plain; charset=utf-8"
 
 			pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
@@ -132,13 +139,13 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 			// raw.codeberg.org/example/myrepo/@main/index.html
 			if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") {
 				log.Debug().Msg("raw domain preparations, now trying with specified branch")
-				if tryBranch(targetRepo, pathElements[2][1:], pathElements[3:],
+				if tryBranch(log,
+					targetRepo, pathElements[2][1:], pathElements[3:],
 					giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
 				) {
 					log.Debug().Msg("tryBranch, now trying upstream 1")
-					tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+					tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
 						targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-						giteaRoot, giteaAPIToken,
 						canonicalDomainCache, branchTimestampCache, fileResponseCache)
 					return
 				}
@@ -148,13 +155,13 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 			}
 
 			log.Debug().Msg("raw domain preparations, now trying with default branch")
-			tryBranch(targetRepo, "", pathElements[2:],
+			tryBranch(log,
+				targetRepo, "", pathElements[2:],
 				giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
 			)
 			log.Debug().Msg("tryBranch, now trying upstream 2")
-			tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+			tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
 				targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-				giteaRoot, giteaAPIToken,
 				canonicalDomainCache, branchTimestampCache, fileResponseCache)
 			return
 
@@ -183,13 +190,13 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 				}
 
 				log.Debug().Msg("main domain preparations, now trying with specified repo & branch")
-				if tryBranch(pathElements[0], pathElements[1][1:], pathElements[2:],
+				if tryBranch(log,
+					pathElements[0], pathElements[1][1:], pathElements[2:],
 					"/"+pathElements[0]+"/%p",
 				) {
 					log.Debug().Msg("tryBranch, now trying upstream 3")
-					tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+					tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
 						targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-						giteaRoot, giteaAPIToken,
 						canonicalDomainCache, branchTimestampCache, fileResponseCache)
 				} else {
 					html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
@@ -201,11 +208,11 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 			// example.codeberg.page/@main/index.html
 			if strings.HasPrefix(pathElements[0], "@") {
 				log.Debug().Msg("main domain preparations, now trying with specified branch")
-				if tryBranch("pages", pathElements[0][1:], pathElements[1:], "/%p") {
+				if tryBranch(log,
+					"pages", pathElements[0][1:], pathElements[1:], "/%p") {
 					log.Debug().Msg("tryBranch, now trying upstream 4")
-					tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+					tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
 						targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-						giteaRoot, giteaAPIToken,
 						canonicalDomainCache, branchTimestampCache, fileResponseCache)
 				} else {
 					html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
@@ -217,11 +224,11 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 			// example.codeberg.page/myrepo/index.html
 			// example.codeberg.page/pages/... is not allowed here.
 			log.Debug().Msg("main domain preparations, now trying with specified repo")
-			if pathElements[0] != "pages" && tryBranch(pathElements[0], "pages", pathElements[1:], "") {
+			if pathElements[0] != "pages" && tryBranch(log,
+				pathElements[0], "pages", pathElements[1:], "") {
 				log.Debug().Msg("tryBranch, now trying upstream 5")
-				tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+				tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
 					targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-					giteaRoot, giteaAPIToken,
 					canonicalDomainCache, branchTimestampCache, fileResponseCache)
 				return
 			}
@@ -229,11 +236,11 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 			// Try to use the "pages" repo on its default branch
 			// example.codeberg.page/index.html
 			log.Debug().Msg("main domain preparations, now trying with default repo/branch")
-			if tryBranch("pages", "", pathElements, "") {
+			if tryBranch(log,
+				"pages", "", pathElements, "") {
 				log.Debug().Msg("tryBranch, now trying upstream 6")
-				tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+				tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
 					targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-					giteaRoot, giteaAPIToken,
 					canonicalDomainCache, branchTimestampCache, fileResponseCache)
 				return
 			}
@@ -261,8 +268,9 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 
 			// Try to use the given repo on the given branch or the default branch
 			log.Debug().Msg("custom domain preparations, now trying with details from DNS")
-			if tryBranch(targetRepo, targetBranch, pathElements, canonicalLink) {
+			if tryBranch(log,
-				canonicalDomain, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)
+				targetRepo, targetBranch, pathElements, canonicalLink) {
+				canonicalDomain, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), canonicalDomainCache)
 				if !valid {
 					html.ReturnErrorPage(ctx, fasthttp.StatusMisdirectedRequest)
 					return
@@ -279,9 +287,8 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 				}
 
 				log.Debug().Msg("tryBranch, now trying upstream 7")
-				tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+				tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
 					targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-					giteaRoot, giteaAPIToken,
 					canonicalDomainCache, branchTimestampCache, fileResponseCache)
 				return
 			}

server/handler_test.go

@@ -8,15 +8,16 @@ import (
 	"github.com/valyala/fasthttp"
 
 	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/gitea"
 )
 
 func TestHandlerPerformance(t *testing.T) {
+	giteaRoot := "https://codeberg.org"
+	giteaClient := gitea.NewClient(giteaRoot, "")
 	testHandler := Handler(
-		[]byte("codeberg.page"),
+		[]byte("codeberg.page"), []byte("raw.codeberg.org"),
-		[]byte("raw.codeberg.org"),
+		giteaClient,
-		"https://codeberg.org",
+		giteaRoot, "https://docs.codeberg.org/pages/raw-content/",
-		"https://docs.codeberg.org/pages/raw-content/",
-		"",
 		[][]byte{[]byte("/.well-known/acme-challenge/")},
 		[][]byte{[]byte("raw.codeberg.org"), []byte("fonts.codeberg.org"), []byte("design.codeberg.org")},
 		cache.NewKeyValueCache(),

server/try.go

@@ -8,22 +8,22 @@ import (
 
 	"codeberg.org/codeberg/pages/html"
 	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/gitea"
 	"codeberg.org/codeberg/pages/server/upstream"
 )
 
 // tryUpstream forwards the target request to the Gitea API, and shows an error page on failure.
-func tryUpstream(ctx *fasthttp.RequestCtx,
+func tryUpstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
 	mainDomainSuffix, trimmedHost []byte,
 
 	targetOptions *upstream.Options,
-	targetOwner, targetRepo, targetBranch, targetPath,
+	targetOwner, targetRepo, targetBranch, targetPath string,
 
-	giteaRoot, giteaAPIToken string,
 	canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
 ) {
 	// check if a canonical domain exists on a request on MainDomain
 	if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
-		canonicalDomain, _ := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)
+		canonicalDomain, _ := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), canonicalDomainCache)
 		if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) {
 			canonicalPath := string(ctx.RequestURI())
 			if targetRepo != "pages" {
@@ -43,7 +43,7 @@ func tryUpstream(ctx *fasthttp.RequestCtx,
 	targetOptions.TargetPath = targetPath
 
 	// Try to request the file from the Gitea API
-	if !targetOptions.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
+	if !targetOptions.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
 		html.ReturnErrorPage(ctx, ctx.Response.StatusCode())
 	}
 }

server/upstream/domains.go

@@ -4,14 +4,11 @@ import (
 	"strings"
 
 	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/gitea"
 )
 
 // CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file).
-func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaAPIToken string, canonicalDomainCache cache.SetGetKey) (string, bool) {
+func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (string, bool) {
-	return checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaAPIToken, canonicalDomainCache)
-}
-
-func checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaAPIToken string, canonicalDomainCache cache.SetGetKey) (string, bool) {
 	var (
 		domains []string
 		valid   bool
@@ -25,7 +22,7 @@ func checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, m
 			}
 		}
 	} else {
-		body, err := giteaRawContent(targetOwner, targetRepo, targetBranch, giteaRoot, giteaAPIToken, canonicalDomainConfig)
+		body, err := giteaClient.GiteaRawContent(targetOwner, targetRepo, targetBranch, canonicalDomainConfig)
 		if err == nil {
 			for _, domain := range strings.Split(string(body), "\n") {
 				domain = strings.ToLower(domain)

server/upstream/gitea.go

@@ -1,67 +0,0 @@
-package upstream
-
-import (
-	"fmt"
-	"net/url"
-	"path"
-	"time"
-
-	"github.com/valyala/fasthttp"
-	"github.com/valyala/fastjson"
-)
-
-const giteaAPIRepos = "/api/v1/repos/"
-
-// TODOs:
-// * own client to store token & giteaRoot
-// * handle 404 -> page will show 500 atm
-
-func giteaRawContent(targetOwner, targetRepo, ref, giteaRoot, giteaAPIToken, resource string) ([]byte, error) {
-	req := fasthttp.AcquireRequest()
-
-	req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref)))
-	req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken)
-	res := fasthttp.AcquireResponse()
-
-	if err := getFastHTTPClient(10*time.Second).Do(req, res); err != nil {
-		return nil, err
-	}
-	if res.StatusCode() != fasthttp.StatusOK {
-		return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
-	}
-	return res.Body(), nil
-}
-
-func giteaGetRepoBranchTimestamp(giteaRoot, repoOwner, repoName, branchName, giteaAPIToken string) (time.Time, error) {
-	client := getFastHTTPClient(5 * time.Second)
-
-	req := fasthttp.AcquireRequest()
-	req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, repoOwner, repoName, "branches", branchName))
-	req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken)
-	res := fasthttp.AcquireResponse()
-
-	if err := client.Do(req, res); err != nil {
-		return time.Time{}, err
-	}
-	if res.StatusCode() != fasthttp.StatusOK {
-		return time.Time{}, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
-	}
-	return time.Parse(time.RFC3339, fastjson.GetString(res.Body(), "commit", "timestamp"))
-}
-
-func giteaGetRepoDefaultBranch(giteaRoot, repoOwner, repoName, giteaAPIToken string) (string, error) {
-	client := getFastHTTPClient(5 * time.Second)
-
-	req := fasthttp.AcquireRequest()
-	req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, repoOwner, repoName))
-	req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken)
-	res := fasthttp.AcquireResponse()
-
-	if err := client.Do(req, res); err != nil {
-		return "", err
-	}
-	if res.StatusCode() != fasthttp.StatusOK {
-		return "", fmt.Errorf("unexpected status code '%d'", res.StatusCode())
-	}
-	return fastjson.GetString(res.Body(), "default_branch"), nil
-}

server/upstream/helper.go

@@ -1,9 +1,14 @@
 package upstream
 
 import (
+	"mime"
+	"path"
+	"strconv"
+	"strings"
 	"time"
 
 	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/gitea"
 )
 
 type branchTimestamp struct {
@@ -13,7 +18,7 @@ type branchTimestamp struct {
 
 // GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch
 // (or nil if the branch doesn't exist)
-func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaAPIToken string, branchTimestampCache cache.SetGetKey) *branchTimestamp {
+func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string, branchTimestampCache cache.SetGetKey) *branchTimestamp {
 	if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok {
 		if result == nil {
 			return nil
@@ -25,7 +30,7 @@ func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaAPIToken string, br
 	}
 	if len(branch) == 0 {
 		// Get default branch
-		defaultBranch, err := giteaGetRepoDefaultBranch(giteaRoot, owner, repo, giteaAPIToken)
+		defaultBranch, err := giteaClient.GiteaGetRepoDefaultBranch(owner, repo)
 		if err != nil {
 			_ = branchTimestampCache.Set(owner+"/"+repo+"/", nil, defaultBranchCacheTimeout)
 			return nil
@@ -33,7 +38,7 @@ func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaAPIToken string, br
 		result.Branch = defaultBranch
 	}
 
-	timestamp, err := giteaGetRepoBranchTimestamp(giteaRoot, owner, repo, branch, giteaAPIToken)
+	timestamp, err := giteaClient.GiteaGetRepoBranchTimestamp(owner, repo, result.Branch)
 	if err != nil {
 		return nil
 	}
@@ -42,8 +47,26 @@ func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaAPIToken string, br
 	return result
 }
 
-type fileResponse struct {
+func (o *Options) getMimeTypeByExtension() string {
-	exists   bool
+	if o.ForbiddenMimeTypes == nil {
-	mimeType string
+		o.ForbiddenMimeTypes = make(map[string]bool)
-	body     []byte
+	}
+	mimeType := mime.TypeByExtension(path.Ext(o.TargetPath))
+	mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
+	if o.ForbiddenMimeTypes[mimeTypeSplit[0]] || mimeType == "" {
+		if o.DefaultMimeType != "" {
+			mimeType = o.DefaultMimeType
+		} else {
+			mimeType = "application/octet-stream"
+		}
+	}
+	return mimeType
+}
+
+func (o *Options) generateUri() string {
+	return path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath)
+}
+
+func (o *Options) timestamp() string {
+	return strconv.FormatInt(o.BranchTimestamp.Unix(), 10)
 }

server/upstream/upstream.go

@@ -2,11 +2,9 @@ package upstream
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"io"
-	"mime"
-	"path"
-	"strconv"
 	"strings"
 	"time"
 
@@ -15,6 +13,7 @@ import (
 
 	"codeberg.org/codeberg/pages/html"
 	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/gitea"
 )
 
 // upstreamIndexPages lists pages that may be considered as index pages for directories.
@@ -30,7 +29,7 @@ type Options struct {
 	TargetPath,
 
 	DefaultMimeType string
-	ForbiddenMimeTypes map[string]struct{}
+	ForbiddenMimeTypes map[string]bool
 	TryIndexPages      bool
 	BranchTimestamp    time.Time
 	// internal
@@ -38,26 +37,13 @@ type Options struct {
 	redirectIfExists    string
 }
 
-func getFastHTTPClient(timeout time.Duration) *fasthttp.Client {
-	return &fasthttp.Client{
-		ReadTimeout:        timeout,
-		MaxConnDuration:    60 * time.Second,
-		MaxConnWaitTimeout: 1000 * time.Millisecond,
-		MaxConnsPerHost:    128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
-	}
-}
-
 // Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
-func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken string, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) {
+func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) {
 	log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger()
 
-	if o.ForbiddenMimeTypes == nil {
-		o.ForbiddenMimeTypes = map[string]struct{}{}
-	}
-
 	// Check if the branch exists and when it was modified
 	if o.BranchTimestamp.IsZero() {
-		branch := GetBranchTimestamp(o.TargetOwner, o.TargetRepo, o.TargetBranch, giteaRoot, giteaAPIToken, branchTimestampCache)
+		branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch, branchTimestampCache)
 
 		if branch == nil {
 			html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
@@ -82,25 +68,19 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
 	log.Debug().Msg("preparations")
 
 	// Make a GET request to the upstream URL
-	uri := path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath)
+	uri := o.generateUri()
-	var req *fasthttp.Request
 	var res *fasthttp.Response
-	var cachedResponse fileResponse
+	var cachedResponse gitea.FileResponse
 	var err error
-	if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + strconv.FormatInt(o.BranchTimestamp.Unix(), 10)); ok && len(cachedValue.(fileResponse).body) > 0 {
+	if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() {
-		cachedResponse = cachedValue.(fileResponse)
+		cachedResponse = cachedValue.(gitea.FileResponse)
 	} else {
-		req = fasthttp.AcquireRequest()
+		res, err = giteaClient.ServeRawContent(uri)
-		req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, uri))
-		req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken)
-		res = fasthttp.AcquireResponse()
-		res.SetBodyStream(&strings.Reader{}, -1)
-		err = getFastHTTPClient(10*time.Second).Do(req, res)
 	}
 	log.Debug().Msg("acquisition")
 
 	// Handle errors
-	if (res == nil && !cachedResponse.exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) {
+	if (err != nil && errors.Is(err, gitea.ErrorNotFound)) || (res == nil && !cachedResponse.Exists) {
 		if o.TryIndexPages {
 			// copy the o struct & try if an index page exists
 			optionsForIndexPages := *o
@@ -108,9 +88,9 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
 			optionsForIndexPages.appendTrailingSlash = true
 			for _, indexPage := range upstreamIndexPages {
 				optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage
-				if optionsForIndexPages.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
+				if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
-					_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
+					_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
-						exists: false,
+						Exists: false,
 					}, fileCacheTimeout)
 					return true
 				}
@@ -119,9 +99,9 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
 			optionsForIndexPages.appendTrailingSlash = false
 			optionsForIndexPages.redirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html"
 			optionsForIndexPages.TargetPath = o.TargetPath + ".html"
-			if optionsForIndexPages.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
+			if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
-				_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
+				_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
-					exists: false,
+					Exists: false,
 				}, fileCacheTimeout)
 				return true
 			}
@@ -129,14 +109,14 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
 		ctx.Response.SetStatusCode(fasthttp.StatusNotFound)
 		if res != nil {
 			// Update cache if the request is fresh
-			_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
+			_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
-				exists: false,
+				Exists: false,
 			}, fileCacheTimeout)
 		}
 		return false
 	}
 	if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) {
-		fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", req.RequestURI(), err, res.StatusCode())
+		fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", uri, err, res.StatusCode())
 		html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
 		return true
 	}
@@ -158,15 +138,7 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
 	log.Debug().Msg("error handling")
 
 	// Set the MIME type
-	mimeType := mime.TypeByExtension(path.Ext(o.TargetPath))
+	mimeType := o.getMimeTypeByExtension()
-	mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
-	if _, ok := o.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" {
-		if o.DefaultMimeType != "" {
-			mimeType = o.DefaultMimeType
-		} else {
-			mimeType = "application/octet-stream"
-		}
-	}
 	ctx.Response.Header.SetContentType(mimeType)
 
 	// Everything's okay so far
@@ -185,20 +157,20 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
 			err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter))
 		}
 	} else {
-		_, err = ctx.Write(cachedResponse.body)
+		_, err = ctx.Write(cachedResponse.Body)
 	}
 	if err != nil {
-		fmt.Printf("Couldn't write body for \"%s\": %s\n", req.RequestURI(), err)
+		fmt.Printf("Couldn't write body for \"%s\": %s\n", uri, err)
 		html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
 		return true
 	}
 	log.Debug().Msg("response")
 
 	if res != nil && ctx.Err() == nil {
-		cachedResponse.exists = true
+		cachedResponse.Exists = true
-		cachedResponse.mimeType = mimeType
+		cachedResponse.MimeType = mimeType
-		cachedResponse.body = cacheBodyWriter.Bytes()
+		cachedResponse.Body = cacheBodyWriter.Bytes()
-		_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), cachedResponse, fileCacheTimeout)
+		_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), cachedResponse, fileCacheTimeout)
 	}
 
 	return true