Merge remote-tracking branch 'main'

.gitignore

@@ -7,3 +7,4 @@ build/
 vendor/
 pages
 certs.sqlite
+.bash_history

Justfile

@@ -9,6 +9,8 @@ dev:
     export PAGES_DOMAIN=localhost.mock.directory
     export RAW_DOMAIN=raw.localhost.mock.directory
     export PORT=4430
+    export HTTP_PORT=8880
+    export ENABLE_HTTP_SERVER=true
     export LOG_LEVEL=trace
     go run -tags '{{TAGS}}' .
 
@@ -50,3 +52,6 @@ integration:
 
 integration-run TEST:
     go test -race -tags 'integration {{TAGS}}' -run "^{{TEST}}$" codeberg.org/codeberg/pages/integration/...
+
+docker:
+    docker run --rm -it --user $(id -u) -v $(pwd):/work --workdir /work -e HOME=/work codeberg.org/6543/docker-images/golang_just

cmd/flags.go

@@ -8,11 +8,13 @@ var (
 	CertStorageFlags = []cli.Flag{
 		&cli.StringFlag{
 			Name:    "db-type",
+			Usage:   "Specify the database driver. Valid options are \"sqlite3\", \"mysql\" and \"postgres\". Read more at https://xorm.io",
 			Value:   "sqlite3",
 			EnvVars: []string{"DB_TYPE"},
 		},
 		&cli.StringFlag{
 			Name:    "db-conn",
+			Usage:   "Specify the database connection. For \"sqlite3\" it's the filepath. Read more at https://go.dev/doc/tutorial/database-access",
 			Value:   "certs.sqlite",
 			EnvVars: []string{"DB_CONN"},
 		},
@@ -87,15 +89,21 @@ var (
 			EnvVars: []string{"HOST"},
 			Value:   "[::]",
 		},
-		&cli.StringFlag{
+		&cli.UintFlag{
 			Name:    "port",
-			Usage:   "specifies port of listening address",
+			Usage:   "specifies the https port to listen to ssl requests",
-			EnvVars: []string{"PORT"},
+			EnvVars: []string{"PORT", "HTTPS_PORT"},
-			Value:   "443",
+			Value:   443,
+		},
+		&cli.UintFlag{
+			Name:    "http-port",
+			Usage:   "specifies the http port, you also have to enable http server via ENABLE_HTTP_SERVER=true",
+			EnvVars: []string{"HTTP_PORT"},
+			Value:   80,
 		},
 		&cli.BoolFlag{
-			Name: "enable-http-server",
+			Name:    "enable-http-server",
-			// TODO: desc
+			Usage:   "start a http server to redirect to https and respond to http acme challenges",
 			EnvVars: []string{"ENABLE_HTTP_SERVER"},
 		},
 		&cli.StringFlag{
@@ -104,6 +112,13 @@ var (
 			Usage:   "specify at which log level should be logged. Possible options: info, warn, error, fatal",
 			EnvVars: []string{"LOG_LEVEL"},
 		},
+		// Default branches to fetch assets from
+		&cli.StringSliceFlag{
+			Name:    "pages-branch",
+			Usage:   "define a branch to fetch assets from",
+			EnvVars: []string{"PAGES_BRANCHES"},
+			Value:   cli.NewStringSlice("pages"),
+		},
 
 		// ############################
 		// ### ACME Client Settings ###
@@ -125,23 +140,23 @@ var (
 			Value:   true,
 		},
 		&cli.BoolFlag{
-			Name: "acme-accept-terms",
+			Name:    "acme-accept-terms",
-			// TODO: Usage
+			Usage:   "To accept the ACME ToS",
 			EnvVars: []string{"ACME_ACCEPT_TERMS"},
 		},
 		&cli.StringFlag{
-			Name: "acme-eab-kid",
+			Name:    "acme-eab-kid",
-			// TODO: Usage
+			Usage:   "Register the current account to the ACME server with external binding.",
 			EnvVars: []string{"ACME_EAB_KID"},
 		},
 		&cli.StringFlag{
-			Name: "acme-eab-hmac",
+			Name:    "acme-eab-hmac",
-			// TODO: Usage
+			Usage:   "Register the current account to the ACME server with external binding.",
 			EnvVars: []string{"ACME_EAB_HMAC"},
 		},
 		&cli.StringFlag{
 			Name:    "dns-provider",
-			Usage:   "Use DNS-Challenge for main domain\n\nRead more at: https://go-acme.github.io/lego/dns/",
+			Usage:   "Use DNS-Challenge for main domain. Read more at: https://go-acme.github.io/lego/dns/",
 			EnvVars: []string{"DNS_PROVIDER"},
 		},
 		&cli.StringFlag{

cmd/main.go

@@ -14,7 +14,6 @@ import (
 	"github.com/rs/zerolog/log"
 	"github.com/urfave/cli/v2"
 
-	"codeberg.org/codeberg/pages/server"
 	"codeberg.org/codeberg/pages/server/cache"
 	"codeberg.org/codeberg/pages/server/certificates"
 	"codeberg.org/codeberg/pages/server/gitea"
@@ -46,9 +45,13 @@ func Serve(ctx *cli.Context) error {
 	giteaRoot := ctx.String("gitea-root")
 	giteaAPIToken := ctx.String("gitea-api-token")
 	rawDomain := ctx.String("raw-domain")
+	defaultBranches := ctx.StringSlice("pages-branch")
 	mainDomainSuffix := ctx.String("pages-domain")
 	rawInfoPage := ctx.String("raw-info-page")
-	listeningAddress := fmt.Sprintf("%s:%s", ctx.String("host"), ctx.String("port"))
+	listeningHost := ctx.String("host")
+	listeningSSLPort := ctx.Uint("port")
+	listeningSSLAddress := fmt.Sprintf("%s:%d", listeningHost, listeningSSLPort)
+	listeningHTTPAddress := fmt.Sprintf("%s:%d", listeningHost, ctx.Uint("http-port"))
 	enableHTTPServer := ctx.Bool("enable-http-server")
 
 	allowedCorsDomains := AllowedCorsDomains
@@ -61,6 +64,10 @@ func Serve(ctx *cli.Context) error {
 		mainDomainSuffix = "." + mainDomainSuffix
 	}
 
+	if len(defaultBranches) == 0 {
+		return fmt.Errorf("no default branches set (PAGES_BRANCHES)")
+	}
+
 	// Init ssl cert database
 	certDB, closeFn, err := openCertDB(ctx)
 	if err != nil {
@@ -93,25 +100,19 @@ func Serve(ctx *cli.Context) error {
 		return err
 	}
 
-	// Create handler based on settings
+	// Create listener for SSL connections
-	httpsHandler := handler.Handler(mainDomainSuffix, rawDomain,
+	log.Info().Msgf("Create TCP listener for SSL on %s", listeningSSLAddress)
-		giteaClient,
+	listener, err := net.Listen("tcp", listeningSSLAddress)
-		rawInfoPage,
-		BlacklistedPaths, allowedCorsDomains,
-		dnsLookupCache, canonicalDomainCache, redirectsCache)
 
-	httpHandler := server.SetupHTTPACMEChallengeServer(challengeCache)
-
-	// Setup listener and TLS
-	log.Info().Msgf("Listening on https://%s", listeningAddress)
-	listener, err := net.Listen("tcp", listeningAddress)
 	if err != nil {
 		return fmt.Errorf("couldn't create listener: %v", err)
 	}
 
+	// Setup listener for SSL connections
 	listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix,
 		giteaClient,
 		acmeClient,
+		defaultBranches[0],
 		keyCache, challengeCache, dnsLookupCache, canonicalDomainCache,
 		certDB))
 
@@ -121,18 +122,30 @@ func Serve(ctx *cli.Context) error {
 	go certificates.MaintainCertDB(certMaintainCtx, interval, acmeClient, mainDomainSuffix, certDB)
 
 	if enableHTTPServer {
+		// Create handler for http->https redirect and http acme challenges
+		httpHandler := certificates.SetupHTTPACMEChallengeServer(challengeCache, listeningSSLPort)
+
+		// Create listener for http and start listening
 		go func() {
-			log.Info().Msg("Start HTTP server listening on :80")
+			log.Info().Msgf("Start HTTP server listening on %s", listeningHTTPAddress)
-			err := http.ListenAndServe("[::]:80", httpHandler)
+			err := http.ListenAndServe(listeningHTTPAddress, httpHandler)
 			if err != nil {
 				log.Panic().Err(err).Msg("Couldn't start HTTP fastServer")
 			}
 		}()
 	}
 
-	// Start the web fastServer
+	// Create ssl handler based on settings
-	log.Info().Msgf("Start listening on %s", listener.Addr())
+	sslHandler := handler.Handler(mainDomainSuffix, rawDomain,
-	if err := http.Serve(listener, httpsHandler); err != nil {
+		giteaClient,
+		rawInfoPage,
+		BlacklistedPaths, allowedCorsDomains,
+		defaultBranches,
+		dnsLookupCache, canonicalDomainCache, redirectsCache)
+
+	// Start the ssl listener
+	log.Info().Msgf("Start SSL server using TCP listener on %s", listener.Addr())
+	if err := http.Serve(listener, sslHandler); err != nil {
 		log.Panic().Err(err).Msg("Couldn't start fastServer")
 	}
 

cmd/setup.go

@@ -43,6 +43,11 @@ func createAcmeClient(ctx *cli.Context, enableHTTPServer bool, challengeCache ca
 	if (!acmeAcceptTerms || dnsProvider == "") && acmeAPI != "https://acme.mock.directory" {
 		return nil, fmt.Errorf("%w: you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER, unless $ACME_API is set to https://acme.mock.directory", ErrAcmeMissConfig)
 	}
+	if acmeEabHmac != "" && acmeEabKID == "" {
+		return nil, fmt.Errorf("%w: ACME_EAB_HMAC also needs ACME_EAB_KID to be set", ErrAcmeMissConfig)
+	} else if acmeEabHmac == "" && acmeEabKID != "" {
+		return nil, fmt.Errorf("%w: ACME_EAB_KID also needs ACME_EAB_HMAC to be set", ErrAcmeMissConfig)
+	}
 
 	return certificates.NewAcmeClient(
 		acmeAccountConf,

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/rs/zerolog v1.27.0
 	github.com/stretchr/testify v1.7.0
 	github.com/urfave/cli/v2 v2.3.0
+	golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb
 	xorm.io/xorm v1.3.2
 )
 
@@ -117,7 +118,7 @@ require (
 	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect
 	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
-	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
+	golang.org/x/sys v0.1.0 // indirect
 	golang.org/x/text v0.3.6 // indirect
 	golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 // indirect
 	google.golang.org/api v0.20.0 // indirect

go.sum

@@ -768,6 +768,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb h1:PaBZQdo+iSDyHT053FjUCgZQ/9uqVwPOcl7KSWhKn6w=
+golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -899,6 +901,8 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg=
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

integration/get_test.go

@@ -109,6 +109,34 @@ func TestCustomDomainRedirects(t *testing.T) {
 	assert.EqualValues(t, "https://mock-pages.codeberg-test.org/README.md", resp.Header.Get("Location"))
 }
 
+func TestRawCustomDomain(t *testing.T) {
+	log.Println("=== TestRawCustomDomain ===")
+	// test raw domain response for custom domain branch
+	resp, err := getTestHTTPSClient().Get("https://raw.localhost.mock.directory:4430/cb_pages_tests/raw-test/example") // need cb_pages_tests fork
+	assert.NoError(t, err)
+	if !assert.NotNil(t, resp) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, http.StatusOK, resp.StatusCode)
+	assert.EqualValues(t, "text/plain; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "76", resp.Header.Get("Content-Length"))
+	assert.EqualValues(t, 76, getSize(resp.Body))
+}
+
+func TestRawIndex(t *testing.T) {
+	log.Println("=== TestRawCustomDomain ===")
+	// test raw domain response for index.html
+	resp, err := getTestHTTPSClient().Get("https://raw.localhost.mock.directory:4430/cb_pages_tests/raw-test/@branch-test/index.html") // need cb_pages_tests fork
+	assert.NoError(t, err)
+	if !assert.NotNil(t, resp) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, http.StatusOK, resp.StatusCode)
+	assert.EqualValues(t, "text/plain; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "597", resp.Header.Get("Content-Length"))
+	assert.EqualValues(t, 597, getSize(resp.Body))
+}
+
 func TestGetNotFound(t *testing.T) {
 	log.Println("=== TestGetNotFound ===")
 	// test custom not found pages
@@ -165,6 +193,18 @@ func TestGetOptions(t *testing.T) {
 	assert.EqualValues(t, "GET, HEAD, OPTIONS", resp.Header.Get("Allow"))
 }
 
+func TestHttpRedirect(t *testing.T) {
+	log.Println("=== TestHttpRedirect ===")
+	resp, err := getTestHTTPSClient().Get("http://mock-pages.codeberg-test.org:8880/README.md")
+	assert.NoError(t, err)
+	if !assert.NotNil(t, resp) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, http.StatusMovedPermanently, resp.StatusCode)
+	assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "https://mock-pages.codeberg-test.org:4430/README.md", resp.Header.Get("Location"))
+}
+
 func getTestHTTPSClient() *http.Client {
 	cookieJar, _ := cookiejar.New(nil)
 	return &http.Client{

integration/main_test.go

@@ -39,7 +39,10 @@ func startServer(ctx context.Context) error {
 	setEnvIfNotSet("ACME_API", "https://acme.mock.directory")
 	setEnvIfNotSet("PAGES_DOMAIN", "localhost.mock.directory")
 	setEnvIfNotSet("RAW_DOMAIN", "raw.localhost.mock.directory")
+	setEnvIfNotSet("PAGES_BRANCHES", "pages,main,master")
 	setEnvIfNotSet("PORT", "4430")
+	setEnvIfNotSet("HTTP_PORT", "8880")
+	setEnvIfNotSet("ENABLE_HTTP_SERVER", "true")
 	setEnvIfNotSet("DB_TYPE", "sqlite3")
 
 	app := cli.NewApp()

server/certificates/acme_config.go

@@ -14,6 +14,8 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
+const challengePath = "/.well-known/acme-challenge/"
+
 func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcceptTerms bool) (*lego.Config, error) {
 	var myAcmeAccount AcmeAccount
 	var myAcmeConfig *lego.Config

server/certificates/cached_challengers.go

@@ -1,11 +1,17 @@
 package certificates
 
 import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
 	"time"
 
 	"github.com/go-acme/lego/v4/challenge"
+	"github.com/rs/zerolog/log"
 
 	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/context"
 )
 
 type AcmeTLSChallengeProvider struct {
@@ -39,3 +45,39 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
 	a.challengeCache.Remove(domain + "/" + token)
 	return nil
 }
+
+func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey, sslPort uint) http.HandlerFunc {
+	// handle custom-ssl-ports to be added on https redirects
+	portPart := ""
+	if sslPort != 443 {
+		portPart = fmt.Sprintf(":%d", sslPort)
+	}
+
+	return func(w http.ResponseWriter, req *http.Request) {
+		ctx := context.New(w, req)
+		domain := ctx.TrimHostPort()
+
+		// it's an acme request
+		if strings.HasPrefix(ctx.Path(), challengePath) {
+			challenge, ok := challengeCache.Get(domain + "/" + strings.TrimPrefix(ctx.Path(), challengePath))
+			if !ok || challenge == nil {
+				log.Info().Msgf("HTTP-ACME challenge for '%s' failed: token not found", domain)
+				ctx.String("no challenge for this token", http.StatusNotFound)
+			}
+			log.Info().Msgf("HTTP-ACME challenge for '%s' succeeded", domain)
+			ctx.String(challenge.(string))
+			return
+		}
+
+		// it's a normal http request that needs to be redirected
+		u, err := url.Parse(fmt.Sprintf("https://%s%s%s", domain, portPart, ctx.Path()))
+		if err != nil {
+			log.Error().Err(err).Msg("could not craft http to https redirect")
+			ctx.String("", http.StatusInternalServerError)
+		}
+
+		newURL := u.String()
+		log.Debug().Msgf("redirect http to https: %s", newURL)
+		ctx.Redirect(newURL, http.StatusMovedPermanently)
+	}
+}

server/certificates/certificates.go

@@ -30,28 +30,31 @@ var ErrUserRateLimitExceeded = errors.New("rate limit exceeded: 10 certificates
 func TLSConfig(mainDomainSuffix string,
 	giteaClient *gitea.Client,
 	acmeClient *AcmeClient,
+	firstDefaultBranch string,
 	keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey,
 	certDB database.CertDB,
 ) *tls.Config {
 	return &tls.Config{
 		// check DNS name & get certificate from Let's Encrypt
 		GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
-			sni := strings.ToLower(strings.TrimSpace(info.ServerName))
+			domain := strings.ToLower(strings.TrimSpace(info.ServerName))
-			if len(sni) < 1 {
+			if len(domain) < 1 {
-				return nil, errors.New("missing sni")
+				return nil, errors.New("missing domain info via SNI (RFC 4366, Section 3.1)")
 			}
 
+			// https request init is actually a acme challenge
 			if info.SupportedProtos != nil {
 				for _, proto := range info.SupportedProtos {
 					if proto != tlsalpn01.ACMETLS1Protocol {
 						continue
 					}
+					log.Info().Msgf("Detect ACME-TLS1 challenge for '%s'", domain)
 
-					challenge, ok := challengeCache.Get(sni)
+					challenge, ok := challengeCache.Get(domain)
 					if !ok {
 						return nil, errors.New("no challenge for this domain")
 					}
-					cert, err := tlsalpn01.ChallengeCert(sni, challenge.(string))
+					cert, err := tlsalpn01.ChallengeCert(domain, challenge.(string))
 					if err != nil {
 						return nil, err
 					}
@@ -61,22 +64,22 @@ func TLSConfig(mainDomainSuffix string,
 
 			targetOwner := ""
 			mayObtainCert := true
-			if strings.HasSuffix(sni, mainDomainSuffix) || strings.EqualFold(sni, mainDomainSuffix[1:]) {
+			if strings.HasSuffix(domain, mainDomainSuffix) || strings.EqualFold(domain, mainDomainSuffix[1:]) {
 				// deliver default certificate for the main domain (*.codeberg.page)
-				sni = mainDomainSuffix
+				domain = mainDomainSuffix
 			} else {
 				var targetRepo, targetBranch string
-				targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(sni, mainDomainSuffix, dnsLookupCache)
+				targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch, dnsLookupCache)
 				if targetOwner == "" {
 					// DNS not set up, return main certificate to redirect to the docs
-					sni = mainDomainSuffix
+					domain = mainDomainSuffix
 				} else {
 					targetOpt := &upstream.Options{
 						TargetOwner:  targetOwner,
 						TargetRepo:   targetRepo,
 						TargetBranch: targetBranch,
 					}
-					_, valid := targetOpt.CheckCanonicalDomain(giteaClient, sni, mainDomainSuffix, canonicalDomainCache)
+					_, valid := targetOpt.CheckCanonicalDomain(giteaClient, domain, mainDomainSuffix, canonicalDomainCache)
 					if !valid {
 						// We shouldn't obtain a certificate when we cannot check if the
 						// repository has specified this domain in the `.domains` file.
@@ -85,30 +88,34 @@ func TLSConfig(mainDomainSuffix string,
 				}
 			}
 
-			if tlsCertificate, ok := keyCache.Get(sni); ok {
+			if tlsCertificate, ok := keyCache.Get(domain); ok {
 				// we can use an existing certificate object
 				return tlsCertificate.(*tls.Certificate), nil
 			}
 
 			var tlsCertificate *tls.Certificate
 			var err error
-			if tlsCertificate, err = acmeClient.retrieveCertFromDB(sni, mainDomainSuffix, false, certDB); err != nil {
+			if tlsCertificate, err = acmeClient.retrieveCertFromDB(domain, mainDomainSuffix, false, certDB); err != nil {
-				// request a new certificate
+				if !errors.Is(err, database.ErrNotFound) {
-				if strings.EqualFold(sni, mainDomainSuffix) {
+					return nil, err
+				}
+				// we could not find a cert in db, request a new certificate
+
+				// first check if we are allowed to obtain a cert for this domain
+				if strings.EqualFold(domain, mainDomainSuffix) {
 					return nil, errors.New("won't request certificate for main domain, something really bad has happened")
 				}
-
 				if !mayObtainCert {
-					return nil, fmt.Errorf("won't request certificate for %q", sni)
+					return nil, fmt.Errorf("won't request certificate for %q", domain)
 				}
 
-				tlsCertificate, err = acmeClient.obtainCert(acmeClient.legoClient, []string{sni}, nil, targetOwner, false, mainDomainSuffix, certDB)
+				tlsCertificate, err = acmeClient.obtainCert(acmeClient.legoClient, []string{domain}, nil, targetOwner, false, mainDomainSuffix, certDB)
 				if err != nil {
 					return nil, err
 				}
 			}
 
-			if err := keyCache.Set(sni, tlsCertificate, 15*time.Minute); err != nil {
+			if err := keyCache.Set(domain, tlsCertificate, 15*time.Minute); err != nil {
 				return nil, err
 			}
 			return tlsCertificate, nil
@@ -164,7 +171,7 @@ func (c *AcmeClient) retrieveCertFromDB(sni, mainDomainSuffix string, useDnsProv
 	if !strings.EqualFold(sni, mainDomainSuffix) {
 		tlsCertificate.Leaf, err = x509.ParseCertificate(tlsCertificate.Certificate[0])
 		if err != nil {
-			return nil, fmt.Errorf("error parsin leaf tlsCert: %w", err)
+			return nil, fmt.Errorf("error parsing leaf tlsCert: %w", err)
 		}
 
 		// renew certificates 7 days before they expire

server/certificates/mock_test.go

@@ -4,13 +4,15 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 
 	"codeberg.org/codeberg/pages/server/database"
 )
 
 func TestMockCert(t *testing.T) {
-	db, err := database.NewTmpDB()
+	db := database.NewMockCertDB(t)
-	assert.NoError(t, err)
+	db.Mock.On("Put", mock.Anything, mock.Anything).Return(nil)
+
 	cert, err := mockCert("example.com", "some error msg", "codeberg.page", db)
 	assert.NoError(t, err)
 	if assert.NotEmpty(t, cert) {

server/database/interface.go

@@ -8,6 +8,9 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
+//go:generate go install github.com/vektra/mockery/v2@latest
+//go:generate mockery --name CertDB --output . --filename mock.go --inpackage --case underscore
+
 type CertDB interface {
 	Close() error
 	Put(name string, cert *certificate.Resource) error

server/database/mock.go

@@ -1,49 +1,122 @@
+// Code generated by mockery v2.20.0. DO NOT EDIT.
+
 package database
 
 import (
-	"fmt"
+	certificate "github.com/go-acme/lego/v4/certificate"
-	"time"
+	mock "github.com/stretchr/testify/mock"
-
-	"github.com/OrlovEvgeny/go-mcache"
-	"github.com/go-acme/lego/v4/certificate"
 )
 
-var _ CertDB = tmpDB{}
+// MockCertDB is an autogenerated mock type for the CertDB type
-
+type MockCertDB struct {
-type tmpDB struct {
+	mock.Mock
-	intern *mcache.CacheDriver
-	ttl    time.Duration
 }
 
-func (p tmpDB) Close() error {
+// Close provides a mock function with given fields:
-	_ = p.intern.Close()
+func (_m *MockCertDB) Close() error {
-	return nil
+	ret := _m.Called()
-}
 
-func (p tmpDB) Put(name string, cert *certificate.Resource) error {
+	var r0 error
-	return p.intern.Set(name, cert, p.ttl)
+	if rf, ok := ret.Get(0).(func() error); ok {
-}
+		r0 = rf()
-
+	} else {
-func (p tmpDB) Get(name string) (*certificate.Resource, error) {
+		r0 = ret.Error(0)
-	cert, has := p.intern.Get(name)
-	if !has {
-		return nil, fmt.Errorf("cert for %q not found", name)
 	}
-	return cert.(*certificate.Resource), nil
+
+	return r0
 }
 
-func (p tmpDB) Delete(key string) error {
+// Delete provides a mock function with given fields: key
-	p.intern.Remove(key)
+func (_m *MockCertDB) Delete(key string) error {
-	return nil
+	ret := _m.Called(key)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string) error); ok {
+		r0 = rf(key)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
 }
 
-func (p tmpDB) Items(page, pageSize int) ([]*Cert, error) {
+// Get provides a mock function with given fields: name
-	return nil, fmt.Errorf("items not implemented for tmpDB")
+func (_m *MockCertDB) Get(name string) (*certificate.Resource, error) {
+	ret := _m.Called(name)
+
+	var r0 *certificate.Resource
+	var r1 error
+	if rf, ok := ret.Get(0).(func(string) (*certificate.Resource, error)); ok {
+		return rf(name)
+	}
+	if rf, ok := ret.Get(0).(func(string) *certificate.Resource); ok {
+		r0 = rf(name)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*certificate.Resource)
+		}
+	}
+
+	if rf, ok := ret.Get(1).(func(string) error); ok {
+		r1 = rf(name)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
 }
 
-func NewTmpDB() (CertDB, error) {
+// Items provides a mock function with given fields: page, pageSize
-	return &tmpDB{
+func (_m *MockCertDB) Items(page int, pageSize int) ([]*Cert, error) {
-		intern: mcache.New(),
+	ret := _m.Called(page, pageSize)
-		ttl:    time.Minute,
+
-	}, nil
+	var r0 []*Cert
+	var r1 error
+	if rf, ok := ret.Get(0).(func(int, int) ([]*Cert, error)); ok {
+		return rf(page, pageSize)
+	}
+	if rf, ok := ret.Get(0).(func(int, int) []*Cert); ok {
+		r0 = rf(page, pageSize)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]*Cert)
+		}
+	}
+
+	if rf, ok := ret.Get(1).(func(int, int) error); ok {
+		r1 = rf(page, pageSize)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Put provides a mock function with given fields: name, cert
+func (_m *MockCertDB) Put(name string, cert *certificate.Resource) error {
+	ret := _m.Called(name, cert)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string, *certificate.Resource) error); ok {
+		r0 = rf(name, cert)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+type mockConstructorTestingTNewMockCertDB interface {
+	mock.TestingT
+	Cleanup(func())
+}
+
+// NewMockCertDB creates a new instance of MockCertDB. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
+func NewMockCertDB(t mockConstructorTestingTNewMockCertDB) *MockCertDB {
+	mock := &MockCertDB{}
+	mock.Mock.Test(t)
+
+	t.Cleanup(func() { mock.AssertExpectations(t) })
+
+	return mock
 }

server/dns/dns.go

@@ -11,9 +11,11 @@ import (
 // lookupCacheTimeout specifies the timeout for the DNS lookup cache.
 var lookupCacheTimeout = 15 * time.Minute
 
+var defaultPagesRepo = "pages"
+
 // GetTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix.
 // If everything is fine, it returns the target data.
-func GetTargetFromDNS(domain, mainDomainSuffix string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) {
+func GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) {
 	// Get CNAME or TXT
 	var cname string
 	var err error
@@ -50,10 +52,10 @@ func GetTargetFromDNS(domain, mainDomainSuffix string, dnsLookupCache cache.SetG
 		targetBranch = cnameParts[len(cnameParts)-3]
 	}
 	if targetRepo == "" {
-		targetRepo = "pages"
+		targetRepo = defaultPagesRepo
 	}
-	if targetBranch == "" && targetRepo != "pages" {
+	if targetBranch == "" && targetRepo != defaultPagesRepo {
-		targetBranch = "pages"
+		targetBranch = firstDefaultBranch
 	}
 	// if targetBranch is still empty, the caller must find the default branch
 	return

server/handler/handler.go

@@ -17,7 +17,6 @@ const (
 	headerAccessControlAllowOrigin  = "Access-Control-Allow-Origin"
 	headerAccessControlAllowMethods = "Access-Control-Allow-Methods"
 	defaultPagesRepo                = "pages"
-	defaultPagesBranch              = "pages"
 )
 
 // Handler handles a single HTTP request to the web server.
@@ -25,6 +24,7 @@ func Handler(mainDomainSuffix, rawDomain string,
 	giteaClient *gitea.Client,
 	rawInfoPage string,
 	blacklistedPaths, allowedCorsDomains []string,
+	defaultPagesBranches []string,
 	dnsLookupCache, canonicalDomainCache, redirectsCache cache.SetGetKey,
 ) http.HandlerFunc {
 	return func(w http.ResponseWriter, req *http.Request) {
@@ -98,6 +98,7 @@ func Handler(mainDomainSuffix, rawDomain string,
 			log.Debug().Msg("subdomain request detecded")
 			handleSubDomain(log, ctx, giteaClient,
 				mainDomainSuffix,
+				defaultPagesBranches,
 				trimmedHost,
 				pathElements,
 				canonicalDomainCache, redirectsCache)
@@ -107,6 +108,7 @@ func Handler(mainDomainSuffix, rawDomain string,
 				mainDomainSuffix,
 				trimmedHost,
 				pathElements,
+				defaultPagesBranches[0],
 				dnsLookupCache, canonicalDomainCache, redirectsCache)
 		}
 	}

server/handler/handler_custom_domain.go

@@ -18,10 +18,11 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g
 	mainDomainSuffix string,
 	trimmedHost string,
 	pathElements []string,
+	firstDefaultBranch string,
 	dnsLookupCache, canonicalDomainCache, redirectsCache cache.SetGetKey,
 ) {
 	// Serve pages from custom domains
-	targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, mainDomainSuffix, dnsLookupCache)
+	targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, mainDomainSuffix, firstDefaultBranch, dnsLookupCache)
 	if targetOwner == "" {
 		html.ReturnErrorPage(ctx,
 			"could not obtain repo owner from custom domain",
@@ -52,7 +53,7 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g
 			return
 		} else if canonicalDomain != trimmedHost {
 			// only redirect if the target is also a codeberg page!
-			targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix, dnsLookupCache)
+			targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix, firstDefaultBranch, dnsLookupCache)
 			if targetOwner != "" {
 				ctx.Redirect("https://"+canonicalDomain+"/"+targetOpt.TargetPath, http.StatusTemporaryRedirect)
 				return

server/handler/handler_sub_domain.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"github.com/rs/zerolog"
+	"golang.org/x/exp/slices"
 
 	"codeberg.org/codeberg/pages/html"
 	"codeberg.org/codeberg/pages/server/cache"
@@ -17,6 +18,7 @@ import (
 
 func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client,
 	mainDomainSuffix string,
+	defaultPagesBranches []string,
 	trimmedHost string,
 	pathElements []string,
 	canonicalDomainCache, redirectsCache cache.SetGetKey,
@@ -63,12 +65,21 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite
 	// Check if the first directory is a branch for the defaultPagesRepo
 	// example.codeberg.page/@main/index.html
 	if strings.HasPrefix(pathElements[0], "@") {
+		targetBranch := pathElements[0][1:]
+
+		// if the default pages branch can be determined exactly, it does not need to be set
+		if len(defaultPagesBranches) == 1 && slices.Contains(defaultPagesBranches, targetBranch) {
+			// example.codeberg.org/@pages/... redirects to example.codeberg.org/...
+			ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), http.StatusTemporaryRedirect)
+			return
+		}
+
 		log.Debug().Msg("main domain preparations, now trying with specified branch")
 		if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
 			TryIndexPages: true,
 			TargetOwner:   targetOwner,
 			TargetRepo:    defaultPagesRepo,
-			TargetBranch:  pathElements[0][1:],
+			TargetBranch:  targetBranch,
 			TargetPath:    path.Join(pathElements[1:]...),
 		}, true); works {
 			log.Trace().Msg("tryUpstream: serve default pages repo with specified branch")
@@ -81,19 +92,36 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite
 		return
 	}
 
-	// Check if the first directory is a repo with a defaultPagesRepo branch
+	for _, defaultPagesBranch := range defaultPagesBranches {
-	// example.codeberg.page/myrepo/index.html
+		// Check if the first directory is a repo with a default pages branch
-	// example.codeberg.page/pages/... is not allowed here.
+		// example.codeberg.page/myrepo/index.html
-	log.Debug().Msg("main domain preparations, now trying with specified repo")
+		// example.codeberg.page/{PAGES_BRANCHE}/... is not allowed here.
-	if pathElements[0] != defaultPagesRepo {
+		log.Debug().Msg("main domain preparations, now trying with specified repo")
+		if pathElements[0] != defaultPagesBranch {
+			if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
+				TryIndexPages: true,
+				TargetOwner:   targetOwner,
+				TargetRepo:    pathElements[0],
+				TargetBranch:  defaultPagesBranch,
+				TargetPath:    path.Join(pathElements[1:]...),
+			}, false); works {
+				log.Debug().Msg("tryBranch, now trying upstream 5")
+				tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
+				return
+			}
+		}
+
+		// Try to use the defaultPagesRepo on an default pages branch
+		// example.codeberg.page/index.html
+		log.Debug().Msg("main domain preparations, now trying with default repo")
 		if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
 			TryIndexPages: true,
 			TargetOwner:   targetOwner,
-			TargetRepo:    pathElements[0],
+			TargetRepo:    defaultPagesRepo,
 			TargetBranch:  defaultPagesBranch,
-			TargetPath:    path.Join(pathElements[1:]...),
+			TargetPath:    path.Join(pathElements...),
 		}, false); works {
-			log.Debug().Msg("tryBranch, now trying upstream 5")
+			log.Debug().Msg("tryBranch, now trying upstream 6")
 			tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
 			return
 		}

server/handler/handler_test.go

@@ -18,6 +18,7 @@ func TestHandlerPerformance(t *testing.T) {
 		"https://docs.codeberg.org/pages/raw-content/",
 		[]string{"/.well-known/acme-challenge/"},
 		[]string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"},
+		[]string{"pages"},
 		cache.NewKeyValueCache(),
 		cache.NewKeyValueCache(),
 		cache.NewKeyValueCache(),

server/handler/try.go

@@ -21,7 +21,7 @@ func tryUpstream(ctx *context.Context, giteaClient *gitea.Client,
 	redirectsCache cache.SetGetKey,
 ) {
 	// check if a canonical domain exists on a request on MainDomain
-	if strings.HasSuffix(trimmedHost, mainDomainSuffix) {
+	if strings.HasSuffix(trimmedHost, mainDomainSuffix) && !options.ServeRaw {
 		canonicalDomain, _ := options.CheckCanonicalDomain(giteaClient, "", mainDomainSuffix, canonicalDomainCache)
 		if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix) {
 			canonicalPath := ctx.Req.RequestURI

server/setup.go

@@ -1,27 +0,0 @@
-package server
-
-import (
-	"net/http"
-	"strings"
-
-	"codeberg.org/codeberg/pages/server/cache"
-	"codeberg.org/codeberg/pages/server/context"
-	"codeberg.org/codeberg/pages/server/utils"
-)
-
-func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey) http.HandlerFunc {
-	challengePath := "/.well-known/acme-challenge/"
-
-	return func(w http.ResponseWriter, req *http.Request) {
-		ctx := context.New(w, req)
-		if strings.HasPrefix(ctx.Path(), challengePath) {
-			challenge, ok := challengeCache.Get(utils.TrimHostPort(ctx.Host()) + "/" + strings.TrimPrefix(ctx.Path(), challengePath))
-			if !ok || challenge == nil {
-				ctx.String("no challenge for this token", http.StatusNotFound)
-			}
-			ctx.String(challenge.(string))
-		} else {
-			ctx.Redirect("https://"+ctx.Host()+ctx.Path(), http.StatusMovedPermanently)
-		}
-	}
-}

server/upstream/upstream.go

@@ -168,7 +168,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin
 		ctx.Redirect(ctx.Path()+"/", http.StatusTemporaryRedirect)
 		return true
 	}
-	if strings.HasSuffix(ctx.Path(), "/index.html") {
+	if strings.HasSuffix(ctx.Path(), "/index.html") && !o.ServeRaw {
 		ctx.Redirect(strings.TrimSuffix(ctx.Path(), "index.html"), http.StatusTemporaryRedirect)
 		return true
 	}