+
## Quickstart
@@ -61,18 +60,18 @@ but if you want to run it on a shared IP address (and not a standalone),
you'll need to configure your reverse proxy not to terminate the TLS connections,
but forward the requests on the IP level to the Pages Server.
-You can check out a proof of concept in the `haproxy-sni` folder,
-and especially have a look at [this section of the haproxy.cfg](https://codeberg.org/Codeberg/pages-server/src/branch/main/haproxy-sni/haproxy.cfg#L38).
+You can check out a proof of concept in the `examples/haproxy-sni` folder,
+and especially have a look at [this section of the haproxy.cfg](https://codeberg.org/Codeberg/pages-server/src/branch/main/examples/haproxy-sni/haproxy.cfg#L38).
-### Environment
+### Environment Variables
- `HOST` & `PORT` (default: `[::]` & `443`): listen address.
- `PAGES_DOMAIN` (default: `codeberg.page`): main domain for pages.
-- `RAW_DOMAIN` (default: `raw.codeberg.page`): domain for raw resources.
+- `RAW_DOMAIN` (default: `raw.codeberg.page`): domain for raw resources (must be subdomain of `PAGES_DOMAIN`).
- `GITEA_ROOT` (default: `https://codeberg.org`): root of the upstream Gitea instance.
- `GITEA_API_TOKEN` (default: empty): API token for the Gitea instance to access non-public (e.g. limited) repos.
-- `RAW_INFO_PAGE` (default: https://docs.codeberg.org/pages/raw-content/): info page for raw resources, shown if no resource is provided.
-- `ACME_API` (default: https://acme-v02.api.letsencrypt.org/directory): set this to https://acme.mock.director to use invalid certificates without any verification (great for debugging).
+- `RAW_INFO_PAGE` (default: 
- Static pages made easy - Codeberg Pages
-
-
-
diff --git a/html/error.go b/html/error.go
deleted file mode 100644
index 206b123..0000000
--- a/html/error.go
+++ /dev/null
@@ -1,50 +0,0 @@
-package html
-
-import (
- "html/template"
- "net/http"
- "strconv"
- "strings"
-
- "codeberg.org/codeberg/pages/server/context"
-)
-
-// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body,
-// with "%status%" and %message% replaced with the provided statusCode and msg
-func ReturnErrorPage(ctx *context.Context, msg string, statusCode int) {
- ctx.RespWriter.Header().Set("Content-Type", "text/html; charset=utf-8")
- ctx.RespWriter.WriteHeader(statusCode)
-
- msg = generateResponse(msg, statusCode)
-
- _, _ = ctx.RespWriter.Write([]byte(msg))
-}
-
-// TODO: use template engine
-func generateResponse(msg string, statusCode int) string {
- if msg == "" {
- msg = strings.ReplaceAll(NotFoundPage,
- "%status%",
- strconv.Itoa(statusCode)+" "+errorMessage(statusCode))
- } else {
- msg = strings.ReplaceAll(
- strings.ReplaceAll(ErrorPage, "%message%", template.HTMLEscapeString(msg)),
- "%status%",
- http.StatusText(statusCode))
- }
-
- return msg
-}
-
-func errorMessage(statusCode int) string {
- message := http.StatusText(statusCode)
-
- switch statusCode {
- case http.StatusMisdirectedRequest:
- message += " - domain not specified in .domains file"
- case http.StatusFailedDependency:
- message += " - target repo/branch doesn't exist or is private"
- }
-
- return message
-}
diff --git a/html/error.html b/html/error.html
deleted file mode 100644
index f1975f7..0000000
--- a/html/error.html
+++ /dev/null
@@ -1,38 +0,0 @@
-
-
-
-
-
-
- Static pages made easy - Codeberg Pages
-
-
-
diff --git a/html/error_test.go b/html/error_test.go
deleted file mode 100644
index f5da08c..0000000
--- a/html/error_test.go
+++ /dev/null
@@ -1,38 +0,0 @@
-package html
-
-import (
- "net/http"
- "strings"
- "testing"
-)
-
-func TestValidMessage(t *testing.T) {
- testString := "requested blacklisted path"
- statusCode := http.StatusForbidden
-
- expected := strings.ReplaceAll(
- strings.ReplaceAll(ErrorPage, "%message%", testString),
- "%status%",
- http.StatusText(statusCode))
- actual := generateResponse(testString, statusCode)
-
- if expected != actual {
- t.Errorf("generated response did not match: expected: '%s', got: '%s'", expected, actual)
- }
-}
-
-func TestMessageWithHtml(t *testing.T) {
- testString := `abchtml tag"
+
+ assert.Equal(t, str, sanitizer.Sanitize(str))
+}
+
+func TestSanitizerStringWithCodeTagWithAttribute(t *testing.T) {
+ str := "simple text message with html tag"
+ expected := "simple text message with html tag"
+
+ assert.Equal(t, expected, sanitizer.Sanitize(str))
+}
+
+func TestSanitizerStringWithATag(t *testing.T) {
+ str := "simple text message with a link to another page"
+ expected := "simple text message with a link to another page"
+
+ assert.Equal(t, expected, sanitizer.Sanitize(str))
+}
+
+func TestSanitizerStringWithATagAndHref(t *testing.T) {
+ str := "simple text message with a link to another page"
+ expected := "simple text message with a link to another page"
+
+ assert.Equal(t, expected, sanitizer.Sanitize(str))
+}
+
+func TestSanitizerStringWithImgTag(t *testing.T) {
+ str := "simple text message with a Sorry, but this page couldn't be served.
+"{{.Message}}"
++ We hope this isn't a problem on our end ;) - Make sure to check the + troubleshooting section in the Docs! +
+ + +
+ Static pages made easy -
+ Codeberg Pages
+
+
+
diff --git a/integration/main_test.go b/integration/main_test.go
index 6566f78..86fd9d3 100644
--- a/integration/main_test.go
+++ b/integration/main_test.go
@@ -10,9 +10,10 @@ import (
"testing"
"time"
- "codeberg.org/codeberg/pages/cmd"
-
"github.com/urfave/cli/v2"
+
+ cmd "codeberg.org/codeberg/pages/cli"
+ "codeberg.org/codeberg/pages/server"
)
func TestMain(m *testing.M) {
@@ -32,10 +33,7 @@ func TestMain(m *testing.M) {
}
func startServer(ctx context.Context) error {
- args := []string{
- "--verbose",
- "--acme-accept-terms", "true",
- }
+ args := []string{"integration"}
setEnvIfNotSet("ACME_API", "https://acme.mock.directory")
setEnvIfNotSet("PAGES_DOMAIN", "localhost.mock.directory")
setEnvIfNotSet("RAW_DOMAIN", "raw.localhost.mock.directory")
@@ -44,10 +42,15 @@ func startServer(ctx context.Context) error {
setEnvIfNotSet("HTTP_PORT", "8880")
setEnvIfNotSet("ENABLE_HTTP_SERVER", "true")
setEnvIfNotSet("DB_TYPE", "sqlite3")
+ setEnvIfNotSet("GITEA_ROOT", "https://codeberg.org")
+ setEnvIfNotSet("LOG_LEVEL", "trace")
+ setEnvIfNotSet("ENABLE_LFS_SUPPORT", "true")
+ setEnvIfNotSet("ENABLE_SYMLINK_SUPPORT", "true")
+ setEnvIfNotSet("ACME_ACCOUNT_CONFIG", "integration/acme-account.json")
app := cli.NewApp()
app.Name = "pages-server"
- app.Action = cmd.Serve
+ app.Action = server.Serve
app.Flags = cmd.ServerFlags
go func() {
diff --git a/main.go b/main.go
index 6c1d0cc..87e21f3 100644
--- a/main.go
+++ b/main.go
@@ -1,29 +1,21 @@
package main
import (
- "fmt"
"os"
_ "github.com/joho/godotenv/autoload"
- "github.com/urfave/cli/v2"
+ "github.com/rs/zerolog/log"
- "codeberg.org/codeberg/pages/cmd"
- "codeberg.org/codeberg/pages/server/version"
+ "codeberg.org/codeberg/pages/cli"
+ "codeberg.org/codeberg/pages/server"
)
func main() {
- app := cli.NewApp()
- app.Name = "pages-server"
- app.Version = version.Version
- app.Usage = "pages server"
- app.Action = cmd.Serve
- app.Flags = cmd.ServerFlags
- app.Commands = []*cli.Command{
- cmd.Certs,
- }
+ app := cli.CreatePagesApp()
+ app.Action = server.Serve
if err := app.Run(os.Args); err != nil {
- _, _ = fmt.Fprintln(os.Stderr, err)
+ log.Error().Err(err).Msg("A fatal error occurred")
os.Exit(1)
}
}
diff --git a/renovate.json b/renovate.json
new file mode 100644
index 0000000..9dd1cd7
--- /dev/null
+++ b/renovate.json
@@ -0,0 +1,27 @@
+{
+ "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+ "extends": [
+ "config:recommended",
+ ":maintainLockFilesWeekly",
+ ":enablePreCommit",
+ "schedule:automergeDaily",
+ "schedule:weekends"
+ ],
+ "automergeType": "branch",
+ "automergeMajor": false,
+ "automerge": true,
+ "prConcurrentLimit": 5,
+ "labels": ["dependencies"],
+ "packageRules": [
+ {
+ "matchManagers": ["gomod", "dockerfile"]
+ },
+ {
+ "groupName": "golang deps non-major",
+ "matchManagers": ["gomod"],
+ "matchUpdateTypes": ["minor", "patch"],
+ "extends": ["schedule:daily"]
+ }
+ ],
+ "postUpdateOptions": ["gomodTidy", "gomodUpdateImportPaths"]
+}
diff --git a/server/acme/client.go b/server/acme/client.go
new file mode 100644
index 0000000..d5c83d0
--- /dev/null
+++ b/server/acme/client.go
@@ -0,0 +1,26 @@
+package acme
+
+import (
+ "errors"
+ "fmt"
+
+ "codeberg.org/codeberg/pages/config"
+ "codeberg.org/codeberg/pages/server/cache"
+ "codeberg.org/codeberg/pages/server/certificates"
+)
+
+var ErrAcmeMissConfig = errors.New("ACME client has wrong config")
+
+func CreateAcmeClient(cfg config.ACMEConfig, enableHTTPServer bool, challengeCache cache.ICache) (*certificates.AcmeClient, error) {
+ // check config
+ if (!cfg.AcceptTerms || (cfg.DNSProvider == "" && !cfg.NoDNS01)) && cfg.APIEndpoint != "https://acme.mock.directory" {
+ return nil, fmt.Errorf("%w: you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER or $NO_DNS_01, unless $ACME_API is set to https://acme.mock.directory", ErrAcmeMissConfig)
+ }
+ if cfg.EAB_HMAC != "" && cfg.EAB_KID == "" {
+ return nil, fmt.Errorf("%w: ACME_EAB_HMAC also needs ACME_EAB_KID to be set", ErrAcmeMissConfig)
+ } else if cfg.EAB_HMAC == "" && cfg.EAB_KID != "" {
+ return nil, fmt.Errorf("%w: ACME_EAB_KID also needs ACME_EAB_HMAC to be set", ErrAcmeMissConfig)
+ }
+
+ return certificates.NewAcmeClient(cfg, enableHTTPServer, challengeCache)
+}
diff --git a/server/cache/interface.go b/server/cache/interface.go
index 2952b29..b3412cc 100644
--- a/server/cache/interface.go
+++ b/server/cache/interface.go
@@ -2,7 +2,8 @@ package cache
import "time"
-type SetGetKey interface {
+// ICache is an interface that defines how the pages server interacts with the cache.
+type ICache interface {
Set(key string, value interface{}, ttl time.Duration) error
Get(key string) (interface{}, bool)
Remove(key string)
diff --git a/server/cache/setup.go b/server/cache/memory.go
similarity index 69%
rename from server/cache/setup.go
rename to server/cache/memory.go
index a5928b0..093696f 100644
--- a/server/cache/setup.go
+++ b/server/cache/memory.go
@@ -2,6 +2,6 @@ package cache
import "github.com/OrlovEvgeny/go-mcache"
-func NewKeyValueCache() SetGetKey {
+func NewInMemoryCache() ICache {
return mcache.New()
}
diff --git a/server/certificates/acme_client.go b/server/certificates/acme_client.go
index ba83e50..f42fd8f 100644
--- a/server/certificates/acme_client.go
+++ b/server/certificates/acme_client.go
@@ -10,6 +10,7 @@ import (
"github.com/reugn/equalizer"
"github.com/rs/zerolog/log"
+ "codeberg.org/codeberg/pages/config"
"codeberg.org/codeberg/pages/server/cache"
)
@@ -28,8 +29,8 @@ type AcmeClient struct {
acmeClientCertificateLimitPerUser map[string]*equalizer.TokenBucket
}
-func NewAcmeClient(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider string, acmeAcceptTerms, enableHTTPServer, acmeUseRateLimits bool, challengeCache cache.SetGetKey) (*AcmeClient, error) {
- acmeConfig, err := setupAcmeConfig(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, acmeAcceptTerms)
+func NewAcmeClient(cfg config.ACMEConfig, enableHTTPServer bool, challengeCache cache.ICache) (*AcmeClient, error) {
+ acmeConfig, err := setupAcmeConfig(cfg)
if err != nil {
return nil, err
}
@@ -54,15 +55,12 @@ func NewAcmeClient(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID,
if err != nil {
log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
} else {
- if dnsProvider == "" {
- // using mock server, don't use wildcard certs
- err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
- if err != nil {
- log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider")
- }
+ if cfg.DNSProvider == "" {
+ // using mock wildcard certs
+ mainDomainAcmeClient = nil
} else {
// use DNS-Challenge https://go-acme.github.io/lego/dns/
- provider, err := dns.NewDNSChallengeProviderByName(dnsProvider)
+ provider, err := dns.NewDNSChallengeProviderByName(cfg.DNSProvider)
if err != nil {
return nil, fmt.Errorf("can not create DNS Challenge provider: %w", err)
}
@@ -76,7 +74,7 @@ func NewAcmeClient(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID,
legoClient: acmeClient,
dnsChallengerLegoClient: mainDomainAcmeClient,
- acmeUseRateLimits: acmeUseRateLimits,
+ acmeUseRateLimits: cfg.UseRateLimits,
obtainLocks: sync.Map{},
diff --git a/server/certificates/acme_config.go b/server/certificates/acme_config.go
index 12ad7c6..2b5151d 100644
--- a/server/certificates/acme_config.go
+++ b/server/certificates/acme_config.go
@@ -8,6 +8,7 @@ import (
"fmt"
"os"
+ "codeberg.org/codeberg/pages/config"
"github.com/go-acme/lego/v4/certcrypto"
"github.com/go-acme/lego/v4/lego"
"github.com/go-acme/lego/v4/registration"
@@ -16,21 +17,27 @@ import (
const challengePath = "/.well-known/acme-challenge/"
-func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcceptTerms bool) (*lego.Config, error) {
+func setupAcmeConfig(cfg config.ACMEConfig) (*lego.Config, error) {
var myAcmeAccount AcmeAccount
var myAcmeConfig *lego.Config
- if account, err := os.ReadFile(configFile); err == nil {
- log.Info().Msgf("found existing acme account config file '%s'", configFile)
+ if cfg.AccountConfigFile == "" {
+ return nil, fmt.Errorf("invalid acme config file: '%s'", cfg.AccountConfigFile)
+ }
+
+ if account, err := os.ReadFile(cfg.AccountConfigFile); err == nil {
+ log.Info().Msgf("found existing acme account config file '%s'", cfg.AccountConfigFile)
if err := json.Unmarshal(account, &myAcmeAccount); err != nil {
return nil, err
}
+
myAcmeAccount.Key, err = certcrypto.ParsePEMPrivateKey([]byte(myAcmeAccount.KeyPEM))
if err != nil {
return nil, err
}
+
myAcmeConfig = lego.NewConfig(&myAcmeAccount)
- myAcmeConfig.CADirURL = acmeAPI
+ myAcmeConfig.CADirURL = cfg.APIEndpoint
myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
// Validate Config
@@ -39,6 +46,7 @@ func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID stri
log.Info().Err(err).Msg("config validation failed, you might just delete the config file and let it recreate")
return nil, fmt.Errorf("acme config validation failed: %w", err)
}
+
return myAcmeConfig, nil
} else if !os.IsNotExist(err) {
return nil, err
@@ -51,20 +59,20 @@ func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID stri
return nil, err
}
myAcmeAccount = AcmeAccount{
- Email: acmeMail,
+ Email: cfg.Email,
Key: privateKey,
KeyPEM: string(certcrypto.PEMEncode(privateKey)),
}
myAcmeConfig = lego.NewConfig(&myAcmeAccount)
- myAcmeConfig.CADirURL = acmeAPI
+ myAcmeConfig.CADirURL = cfg.APIEndpoint
myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
tempClient, err := lego.NewClient(myAcmeConfig)
if err != nil {
log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
} else {
// accept terms & log in to EAB
- if acmeEabKID == "" || acmeEabHmac == "" {
- reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: acmeAcceptTerms})
+ if cfg.EAB_KID == "" || cfg.EAB_HMAC == "" {
+ reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: cfg.AcceptTerms})
if err != nil {
log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
} else {
@@ -72,9 +80,9 @@ func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID stri
}
} else {
reg, err := tempClient.Registration.RegisterWithExternalAccountBinding(registration.RegisterEABOptions{
- TermsOfServiceAgreed: acmeAcceptTerms,
- Kid: acmeEabKID,
- HmacEncoded: acmeEabHmac,
+ TermsOfServiceAgreed: cfg.AcceptTerms,
+ Kid: cfg.EAB_KID,
+ HmacEncoded: cfg.EAB_HMAC,
})
if err != nil {
log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
@@ -89,8 +97,8 @@ func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID stri
log.Error().Err(err).Msg("json.Marshalfailed, waiting for manual restart to avoid rate limits")
select {}
}
- log.Info().Msgf("new acme account created. write to config file '%s'", configFile)
- err = os.WriteFile(configFile, acmeAccountJSON, 0o600)
+ log.Info().Msgf("new acme account created. write to config file '%s'", cfg.AccountConfigFile)
+ err = os.WriteFile(cfg.AccountConfigFile, acmeAccountJSON, 0o600)
if err != nil {
log.Error().Err(err).Msg("os.WriteFile failed, waiting for manual restart to avoid rate limits")
select {}
diff --git a/server/certificates/cached_challengers.go b/server/certificates/cached_challengers.go
index bc9ea67..39439fb 100644
--- a/server/certificates/cached_challengers.go
+++ b/server/certificates/cached_challengers.go
@@ -15,7 +15,7 @@ import (
)
type AcmeTLSChallengeProvider struct {
- challengeCache cache.SetGetKey
+ challengeCache cache.ICache
}
// make sure AcmeTLSChallengeProvider match Provider interface
@@ -31,7 +31,7 @@ func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error {
}
type AcmeHTTPChallengeProvider struct {
- challengeCache cache.SetGetKey
+ challengeCache cache.ICache
}
// make sure AcmeHTTPChallengeProvider match Provider interface
@@ -46,7 +46,7 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
return nil
}
-func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey, sslPort uint) http.HandlerFunc {
+func SetupHTTPACMEChallengeServer(challengeCache cache.ICache, sslPort uint) http.HandlerFunc {
// handle custom-ssl-ports to be added on https redirects
portPart := ""
if sslPort != 443 {
diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go
index 3ae891a..67219dd 100644
--- a/server/certificates/certificates.go
+++ b/server/certificates/certificates.go
@@ -31,8 +31,10 @@ func TLSConfig(mainDomainSuffix string,
giteaClient *gitea.Client,
acmeClient *AcmeClient,
firstDefaultBranch string,
- keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey,
+ keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.ICache,
certDB database.CertDB,
+ noDNS01 bool,
+ rawDomain string,
) *tls.Config {
return &tls.Config{
// check DNS name & get certificate from Let's Encrypt
@@ -64,9 +66,24 @@ func TLSConfig(mainDomainSuffix string,
targetOwner := ""
mayObtainCert := true
+
if strings.HasSuffix(domain, mainDomainSuffix) || strings.EqualFold(domain, mainDomainSuffix[1:]) {
- // deliver default certificate for the main domain (*.codeberg.page)
- domain = mainDomainSuffix
+ if noDNS01 {
+ // Limit the domains allowed to request a certificate to pages-server domains
+ // and domains for an existing user of org
+ if !strings.EqualFold(domain, mainDomainSuffix[1:]) && !strings.EqualFold(domain, rawDomain) {
+ targetOwner := strings.TrimSuffix(domain, mainDomainSuffix)
+ owner_exist, err := giteaClient.GiteaCheckIfOwnerExists(targetOwner)
+ mayObtainCert = owner_exist
+ if err != nil {
+ log.Error().Err(err).Msgf("Failed to check '%s' existence on the forge: %s", targetOwner, err)
+ mayObtainCert = false
+ }
+ }
+ } else {
+ // deliver default certificate for the main domain (*.codeberg.page)
+ domain = mainDomainSuffix
+ }
} else {
var targetRepo, targetBranch string
targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch, dnsLookupCache)
@@ -199,9 +216,6 @@ func (c *AcmeClient) retrieveCertFromDB(sni, mainDomainSuffix string, useDnsProv
func (c *AcmeClient) obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user string, useDnsProvider bool, mainDomainSuffix string, keyDatabase database.CertDB) (*tls.Certificate, error) {
name := strings.TrimPrefix(domains[0], "*")
- if useDnsProvider && len(domains[0]) > 0 && domains[0][0] == '*' {
- domains = domains[1:]
- }
// lock to avoid simultaneous requests
_, working := c.obtainLocks.LoadOrStore(name, struct{}{})
@@ -219,7 +233,11 @@ func (c *AcmeClient) obtainCert(acmeClient *lego.Client, domains []string, renew
defer c.obtainLocks.Delete(name)
if acmeClient == nil {
- return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!", mainDomainSuffix, keyDatabase)
+ if useDnsProvider {
+ return mockCert(domains[0], "DNS ACME client is not defined", mainDomainSuffix, keyDatabase)
+ } else {
+ return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!", mainDomainSuffix, keyDatabase)
+ }
}
// request actual cert
diff --git a/server/database/xorm.go b/server/database/xorm.go
index 217b6d1..63fa39e 100644
--- a/server/database/xorm.go
+++ b/server/database/xorm.go
@@ -52,7 +52,6 @@ func (x xDB) Close() error {
func (x xDB) Put(domain string, cert *certificate.Resource) error {
log.Trace().Str("domain", cert.Domain).Msg("inserting cert to db")
- domain = integrationTestReplacements(domain)
c, err := toCert(domain, cert)
if err != nil {
return err
@@ -82,7 +81,6 @@ func (x xDB) Get(domain string) (*certificate.Resource, error) {
if domain[:1] == "." {
domain = "*" + domain
}
- domain = integrationTestReplacements(domain)
cert := new(Cert)
log.Trace().Str("domain", domain).Msg("get cert from db")
@@ -99,7 +97,6 @@ func (x xDB) Delete(domain string) error {
if domain[:1] == "." {
domain = "*" + domain
}
- domain = integrationTestReplacements(domain)
log.Trace().Str("domain", domain).Msg("delete cert from db")
_, err := x.engine.ID(domain).Delete(new(Cert))
@@ -139,13 +136,3 @@ func supportedDriver(driver string) bool {
return false
}
}
-
-// integrationTestReplacements is needed because integration tests use a single domain cert,
-// while production use a wildcard cert
-// TODO: find a better way to handle this
-func integrationTestReplacements(domainKey string) string {
- if domainKey == "*.localhost.mock.directory" {
- return "localhost.mock.directory"
- }
- return domainKey
-}
diff --git a/server/dns/dns.go b/server/dns/dns.go
index c11b278..970f0c0 100644
--- a/server/dns/dns.go
+++ b/server/dns/dns.go
@@ -15,7 +15,7 @@ var defaultPagesRepo = "pages"
// GetTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix.
// If everything is fine, it returns the target data.
-func GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) {
+func GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch string, dnsLookupCache cache.ICache) (targetOwner, targetRepo, targetBranch string) {
// Get CNAME or TXT
var cname string
var err error
diff --git a/server/gitea/cache.go b/server/gitea/cache.go
index af61edf..97024a1 100644
--- a/server/gitea/cache.go
+++ b/server/gitea/cache.go
@@ -26,6 +26,9 @@ const (
// TODO: move as option into cache interface
fileCacheTimeout = 5 * time.Minute
+ // ownerExistenceCacheTimeout specifies the timeout for the existence of a repo/org
+ ownerExistenceCacheTimeout = 5 * time.Minute
+
// fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default.
fileCacheSizeLimit = int64(1000 * 1000)
)
@@ -39,7 +42,7 @@ type FileResponse struct {
}
func (f FileResponse) IsEmpty() bool {
- return len(f.Body) != 0
+ return len(f.Body) == 0
}
func (f FileResponse) createHttpResponse(cacheKey string) (header http.Header, statusCode int) {
@@ -72,13 +75,14 @@ type BranchTimestamp struct {
type writeCacheReader struct {
originalReader io.ReadCloser
buffer *bytes.Buffer
- rileResponse *FileResponse
+ fileResponse *FileResponse
cacheKey string
- cache cache.SetGetKey
+ cache cache.ICache
hasError bool
}
func (t *writeCacheReader) Read(p []byte) (n int, err error) {
+ log.Trace().Msgf("[cache] read %q", t.cacheKey)
n, err = t.originalReader.Read(p)
if err != nil && err != io.EOF {
log.Trace().Err(err).Msgf("[cache] original reader for %q has returned an error", t.cacheKey)
@@ -90,16 +94,24 @@ func (t *writeCacheReader) Read(p []byte) (n int, err error) {
}
func (t *writeCacheReader) Close() error {
- if !t.hasError {
- fc := *t.rileResponse
- fc.Body = t.buffer.Bytes()
- _ = t.cache.Set(t.cacheKey, fc, fileCacheTimeout)
+ doWrite := !t.hasError
+ fc := *t.fileResponse
+ fc.Body = t.buffer.Bytes()
+ if fc.IsEmpty() {
+ log.Trace().Msg("[cache] file response is empty")
+ doWrite = false
}
- log.Trace().Msgf("cacheReader for %q saved=%t closed", t.cacheKey, !t.hasError)
+ if doWrite {
+ err := t.cache.Set(t.cacheKey, fc, fileCacheTimeout)
+ if err != nil {
+ log.Trace().Err(err).Msgf("[cache] writer for %q has returned an error", t.cacheKey)
+ }
+ }
+ log.Trace().Msgf("cacheReader for %q saved=%t closed", t.cacheKey, doWrite)
return t.originalReader.Close()
}
-func (f FileResponse) CreateCacheReader(r io.ReadCloser, cache cache.SetGetKey, cacheKey string) io.ReadCloser {
+func (f FileResponse) CreateCacheReader(r io.ReadCloser, cache cache.ICache, cacheKey string) io.ReadCloser {
if r == nil || cache == nil || cacheKey == "" {
log.Error().Msg("could not create CacheReader")
return nil
@@ -108,7 +120,7 @@ func (f FileResponse) CreateCacheReader(r io.ReadCloser, cache cache.SetGetKey,
return &writeCacheReader{
originalReader: r,
buffer: bytes.NewBuffer(make([]byte, 0)),
- rileResponse: &f,
+ fileResponse: &f,
cache: cache,
cacheKey: cacheKey,
}
diff --git a/server/gitea/client.go b/server/gitea/client.go
index 7a2bf63..5955bfb 100644
--- a/server/gitea/client.go
+++ b/server/gitea/client.go
@@ -16,16 +16,19 @@ import (
"code.gitea.io/sdk/gitea"
"github.com/rs/zerolog/log"
+ "codeberg.org/codeberg/pages/config"
"codeberg.org/codeberg/pages/server/cache"
+ "codeberg.org/codeberg/pages/server/version"
)
var ErrorNotFound = errors.New("not found")
const (
- // cache key prefixe
+ // cache key prefixes
branchTimestampCacheKeyPrefix = "branchTime"
defaultBranchCacheKeyPrefix = "defaultBranch"
rawContentCacheKeyPrefix = "rawContent"
+ ownerExistenceKeyPrefix = "ownerExist"
// pages server
PagesCacheIndicatorHeader = "X-Pages-Cache"
@@ -43,7 +46,7 @@ const (
type Client struct {
sdkClient *gitea.Client
- responseCache cache.SetGetKey
+ responseCache cache.ICache
giteaRoot string
@@ -54,37 +57,40 @@ type Client struct {
defaultMimeType string
}
-func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, followSymlinks, supportLFS bool) (*Client, error) {
- rootURL, err := url.Parse(giteaRoot)
+func NewClient(cfg config.GiteaConfig, respCache cache.ICache) (*Client, error) {
+ rootURL, err := url.Parse(cfg.Root)
if err != nil {
return nil, err
}
- giteaRoot = strings.Trim(rootURL.String(), "/")
+ giteaRoot := strings.Trim(rootURL.String(), "/")
stdClient := http.Client{Timeout: 10 * time.Second}
- // TODO: pass down
- var (
- forbiddenMimeTypes map[string]bool
- defaultMimeType string
- )
-
- if forbiddenMimeTypes == nil {
- forbiddenMimeTypes = make(map[string]bool)
+ forbiddenMimeTypes := make(map[string]bool, len(cfg.ForbiddenMimeTypes))
+ for _, mimeType := range cfg.ForbiddenMimeTypes {
+ forbiddenMimeTypes[mimeType] = true
}
+
+ defaultMimeType := cfg.DefaultMimeType
if defaultMimeType == "" {
defaultMimeType = "application/octet-stream"
}
- sdk, err := gitea.NewClient(giteaRoot, gitea.SetHTTPClient(&stdClient), gitea.SetToken(giteaAPIToken))
+ sdk, err := gitea.NewClient(
+ giteaRoot,
+ gitea.SetHTTPClient(&stdClient),
+ gitea.SetToken(cfg.Token),
+ gitea.SetUserAgent("pages-server/"+version.Version),
+ )
+
return &Client{
sdkClient: sdk,
responseCache: respCache,
giteaRoot: giteaRoot,
- followSymlinks: followSymlinks,
- supportLFS: supportLFS,
+ followSymlinks: cfg.FollowSymlinks,
+ supportLFS: cfg.LFSEnabled,
forbiddenMimeTypes: forbiddenMimeTypes,
defaultMimeType: defaultMimeType,
@@ -107,26 +113,27 @@ func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource str
func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource string) (io.ReadCloser, http.Header, int, error) {
cacheKey := fmt.Sprintf("%s/%s/%s|%s|%s", rawContentCacheKeyPrefix, targetOwner, targetRepo, ref, resource)
log := log.With().Str("cache_key", cacheKey).Logger()
-
+ log.Trace().Msg("try file in cache")
// handle if cache entry exist
if cache, ok := client.responseCache.Get(cacheKey); ok {
cache := cache.(FileResponse)
cachedHeader, cachedStatusCode := cache.createHttpResponse(cacheKey)
// TODO: check against some timestamp mismatch?!?
if cache.Exists {
+ log.Debug().Msg("[cache] exists")
if cache.IsSymlink {
linkDest := string(cache.Body)
log.Debug().Msgf("[cache] follow symlink from %q to %q", resource, linkDest)
return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest)
- } else {
- log.Debug().Msg("[cache] return bytes")
+ } else if !cache.IsEmpty() {
+ log.Debug().Msgf("[cache] return %d bytes", len(cache.Body))
return io.NopCloser(bytes.NewReader(cache.Body)), cachedHeader, cachedStatusCode, nil
+ } else if cache.IsEmpty() {
+ log.Debug().Msg("[cache] is empty")
}
- } else {
- return nil, cachedHeader, cachedStatusCode, ErrorNotFound
}
}
-
+ log.Trace().Msg("file not in cache")
// not in cache, open reader via gitea api
reader, resp, err := client.sdkClient.GetFileReader(targetOwner, targetRepo, ref, resource, client.supportLFS)
if resp != nil {
@@ -150,12 +157,14 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
linkDest = path.Join(path.Dir(resource), linkDest)
// we store symlink not content to reduce duplicates in cache
- if err := client.responseCache.Set(cacheKey, FileResponse{
+ fileResponse := FileResponse{
Exists: true,
IsSymlink: true,
Body: []byte(linkDest),
ETag: resp.Header.Get(ETagHeader),
- }, fileCacheTimeout); err != nil {
+ }
+ log.Trace().Msgf("file response has %d bytes", len(fileResponse.Body))
+ if err := client.responseCache.Set(cacheKey, fileResponse, fileCacheTimeout); err != nil {
log.Error().Err(err).Msg("[cache] error on cache write")
}
@@ -172,7 +181,7 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
return reader, resp.Response.Header, resp.StatusCode, err
}
- // now we write to cache and respond at the sime time
+ // now we write to cache and respond at the same time
fileResp := FileResponse{
Exists: true,
ETag: resp.Header.Get(ETagHeader),
@@ -258,6 +267,38 @@ func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (str
return branch, nil
}
+func (client *Client) GiteaCheckIfOwnerExists(owner string) (bool, error) {
+ cacheKey := fmt.Sprintf("%s/%s", ownerExistenceKeyPrefix, owner)
+
+ if exist, ok := client.responseCache.Get(cacheKey); ok && exist != nil {
+ return exist.(bool), nil
+ }
+
+ _, resp, err := client.sdkClient.GetUserInfo(owner)
+ if resp.StatusCode == http.StatusOK && err == nil {
+ if err := client.responseCache.Set(cacheKey, true, ownerExistenceCacheTimeout); err != nil {
+ log.Error().Err(err).Msg("[cache] error on cache write")
+ }
+ return true, nil
+ } else if resp.StatusCode != http.StatusNotFound {
+ return false, err
+ }
+
+ _, resp, err = client.sdkClient.GetOrg(owner)
+ if resp.StatusCode == http.StatusOK && err == nil {
+ if err := client.responseCache.Set(cacheKey, true, ownerExistenceCacheTimeout); err != nil {
+ log.Error().Err(err).Msg("[cache] error on cache write")
+ }
+ return true, nil
+ } else if resp.StatusCode != http.StatusNotFound {
+ return false, err
+ }
+ if err := client.responseCache.Set(cacheKey, false, ownerExistenceCacheTimeout); err != nil {
+ log.Error().Err(err).Msg("[cache] error on cache write")
+ }
+ return false, nil
+}
+
func (client *Client) getMimeTypeByExtension(resource string) string {
mimeType := mime.TypeByExtension(path.Ext(resource))
mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
@@ -278,11 +319,11 @@ func shouldRespBeSavedToCache(resp *http.Response) bool {
return false
}
- contentLeng, err := strconv.ParseInt(contentLengthRaw, 10, 64)
+ contentLength, err := strconv.ParseInt(contentLengthRaw, 10, 64)
if err != nil {
log.Error().Err(err).Msg("could not parse content length")
}
// if content to big or could not be determined we not cache it
- return contentLeng > 0 && contentLeng < fileCacheSizeLimit
+ return contentLength > 0 && contentLength < fileCacheSizeLimit
}
diff --git a/server/handler/handler.go b/server/handler/handler.go
index 7edcf95..ffc3400 100644
--- a/server/handler/handler.go
+++ b/server/handler/handler.go
@@ -6,11 +6,11 @@ import (
"github.com/rs/zerolog/log"
+ "codeberg.org/codeberg/pages/config"
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/context"
"codeberg.org/codeberg/pages/server/gitea"
- "codeberg.org/codeberg/pages/server/version"
)
const (
@@ -20,18 +20,17 @@ const (
)
// Handler handles a single HTTP request to the web server.
-func Handler(mainDomainSuffix, rawDomain string,
+func Handler(
+ cfg config.ServerConfig,
giteaClient *gitea.Client,
- rawInfoPage string,
- blacklistedPaths, allowedCorsDomains []string,
- defaultPagesBranches []string,
- dnsLookupCache, canonicalDomainCache, redirectsCache cache.SetGetKey,
+ dnsLookupCache, canonicalDomainCache, redirectsCache cache.ICache,
) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
+ log.Debug().Msg("\n----------------------------------------------------------")
log := log.With().Strs("Handler", []string{req.Host, req.RequestURI}).Logger()
ctx := context.New(w, req)
- ctx.RespWriter.Header().Set("Server", "CodebergPages/"+version.Version)
+ ctx.RespWriter.Header().Set("Server", "pages-server")
// Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin
ctx.RespWriter.Header().Set("Referrer-Policy", "strict-origin-when-cross-origin")
@@ -41,8 +40,8 @@ func Handler(mainDomainSuffix, rawDomain string,
trimmedHost := ctx.TrimHostPort()
- // Add HSTS for RawDomain and MainDomainSuffix
- if hsts := getHSTSHeader(trimmedHost, mainDomainSuffix, rawDomain); hsts != "" {
+ // Add HSTS for RawDomain and MainDomain
+ if hsts := getHSTSHeader(trimmedHost, cfg.MainDomain, cfg.RawDomain); hsts != "" {
ctx.RespWriter.Header().Set("Strict-Transport-Security", hsts)
}
@@ -64,16 +63,16 @@ func Handler(mainDomainSuffix, rawDomain string,
}
// Block blacklisted paths (like ACME challenges)
- for _, blacklistedPath := range blacklistedPaths {
+ for _, blacklistedPath := range cfg.BlacklistedPaths {
if strings.HasPrefix(ctx.Path(), blacklistedPath) {
- html.ReturnErrorPage(ctx, "requested blacklisted path", http.StatusForbidden)
+ html.ReturnErrorPage(ctx, "requested path is blacklisted", http.StatusForbidden)
return
}
}
// Allow CORS for specified domains
allowCors := false
- for _, allowedCorsDomain := range allowedCorsDomains {
+ for _, allowedCorsDomain := range cfg.AllowedCorsDomains {
if strings.EqualFold(trimmedHost, allowedCorsDomain) {
allowCors = true
break
@@ -87,28 +86,28 @@ func Handler(mainDomainSuffix, rawDomain string,
// Prepare request information to Gitea
pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/")
- if rawDomain != "" && strings.EqualFold(trimmedHost, rawDomain) {
- log.Debug().Msg("raw domain request detecded")
+ if cfg.RawDomain != "" && strings.EqualFold(trimmedHost, cfg.RawDomain) {
+ log.Debug().Msg("raw domain request detected")
handleRaw(log, ctx, giteaClient,
- mainDomainSuffix, rawInfoPage,
+ cfg.MainDomain,
trimmedHost,
pathElements,
canonicalDomainCache, redirectsCache)
- } else if strings.HasSuffix(trimmedHost, mainDomainSuffix) {
- log.Debug().Msg("subdomain request detecded")
+ } else if strings.HasSuffix(trimmedHost, cfg.MainDomain) {
+ log.Debug().Msg("subdomain request detected")
handleSubDomain(log, ctx, giteaClient,
- mainDomainSuffix,
- defaultPagesBranches,
+ cfg.MainDomain,
+ cfg.PagesBranches,
trimmedHost,
pathElements,
canonicalDomainCache, redirectsCache)
} else {
- log.Debug().Msg("custom domain request detecded")
+ log.Debug().Msg("custom domain request detected")
handleCustomDomain(log, ctx, giteaClient,
- mainDomainSuffix,
+ cfg.MainDomain,
trimmedHost,
pathElements,
- defaultPagesBranches[0],
+ cfg.PagesBranches[0],
dnsLookupCache, canonicalDomainCache, redirectsCache)
}
}
diff --git a/server/handler/handler_custom_domain.go b/server/handler/handler_custom_domain.go
index 299d7cf..82953f9 100644
--- a/server/handler/handler_custom_domain.go
+++ b/server/handler/handler_custom_domain.go
@@ -19,7 +19,7 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g
trimmedHost string,
pathElements []string,
firstDefaultBranch string,
- dnsLookupCache, canonicalDomainCache, redirectsCache cache.SetGetKey,
+ dnsLookupCache, canonicalDomainCache, redirectsCache cache.ICache,
) {
// Serve pages from custom domains
targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, mainDomainSuffix, firstDefaultBranch, dnsLookupCache)
@@ -49,7 +49,7 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g
}, canonicalLink); works {
canonicalDomain, valid := targetOpt.CheckCanonicalDomain(giteaClient, trimmedHost, mainDomainSuffix, canonicalDomainCache)
if !valid {
- html.ReturnErrorPage(ctx, "", http.StatusMisdirectedRequest)
+ html.ReturnErrorPage(ctx, "domain not specified in .domains file", http.StatusMisdirectedRequest)
return
} else if canonicalDomain != trimmedHost {
// only redirect if the target is also a codeberg page!
diff --git a/server/handler/handler_raw_domain.go b/server/handler/handler_raw_domain.go
index aa41c52..f48e8e4 100644
--- a/server/handler/handler_raw_domain.go
+++ b/server/handler/handler_raw_domain.go
@@ -16,17 +16,21 @@ import (
)
func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client,
- mainDomainSuffix, rawInfoPage string,
+ mainDomainSuffix string,
trimmedHost string,
pathElements []string,
- canonicalDomainCache, redirectsCache cache.SetGetKey,
+ canonicalDomainCache, redirectsCache cache.ICache,
) {
// Serve raw content from RawDomain
log.Debug().Msg("raw domain")
if len(pathElements) < 2 {
- // https://{RawDomain}/{owner}/{repo}[/@{branch}]/{path} is required
- ctx.Redirect(rawInfoPage, http.StatusTemporaryRedirect)
+ html.ReturnErrorPage(
+ ctx,
+ "a url in the form of https://{domain}/{owner}/{repo}[/@{branch}]/{path} is required",
+ http.StatusBadRequest,
+ )
+
return
}
@@ -61,7 +65,7 @@ func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Clie
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
} else {
html.ReturnErrorPage(ctx,
- fmt.Sprintf("raw domain could not find repo '%s/%s' or repo is empty", targetOpt.TargetOwner, targetOpt.TargetRepo),
+ fmt.Sprintf("raw domain could not find repo %s/%s or repo is empty", targetOpt.TargetOwner, targetOpt.TargetRepo),
http.StatusNotFound)
}
}
diff --git a/server/handler/handler_sub_domain.go b/server/handler/handler_sub_domain.go
index 8731bec..806fe7f 100644
--- a/server/handler/handler_sub_domain.go
+++ b/server/handler/handler_sub_domain.go
@@ -21,7 +21,7 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite
defaultPagesBranches []string,
trimmedHost string,
pathElements []string,
- canonicalDomainCache, redirectsCache cache.SetGetKey,
+ canonicalDomainCache, redirectsCache cache.ICache,
) {
// Serve pages from subdomains of MainDomainSuffix
log.Debug().Msg("main domain suffix")
@@ -55,9 +55,11 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite
log.Trace().Msg("tryUpstream: serve with specified repo and branch")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
} else {
- html.ReturnErrorPage(ctx,
- fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo),
- http.StatusFailedDependency)
+ html.ReturnErrorPage(
+ ctx,
+ formatSetBranchNotFoundMessage(pathElements[1][1:], targetOwner, pathElements[0]),
+ http.StatusFailedDependency,
+ )
}
return
}
@@ -85,9 +87,11 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite
log.Trace().Msg("tryUpstream: serve default pages repo with specified branch")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
} else {
- html.ReturnErrorPage(ctx,
- fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo),
- http.StatusFailedDependency)
+ html.ReturnErrorPage(
+ ctx,
+ formatSetBranchNotFoundMessage(targetBranch, targetOwner, defaultPagesRepo),
+ http.StatusFailedDependency,
+ )
}
return
}
@@ -143,6 +147,10 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite
// Couldn't find a valid repo/branch
html.ReturnErrorPage(ctx,
- fmt.Sprintf("could not find a valid repository[%s]", targetRepo),
+ fmt.Sprintf("could not find a valid repository or branch for repository: %s", targetRepo),
http.StatusNotFound)
}
+
+func formatSetBranchNotFoundMessage(branch, owner, repo string) string {
+ return fmt.Sprintf("explicitly set branch %q does not exist at %s/%s", branch, owner, repo)
+}
diff --git a/server/handler/handler_test.go b/server/handler/handler_test.go
index 6521633..4cb859a 100644
--- a/server/handler/handler_test.go
+++ b/server/handler/handler_test.go
@@ -6,24 +6,30 @@ import (
"testing"
"time"
+ "codeberg.org/codeberg/pages/config"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/gitea"
"github.com/rs/zerolog/log"
)
func TestHandlerPerformance(t *testing.T) {
- giteaClient, _ := gitea.NewClient("https://codeberg.org", "", cache.NewKeyValueCache(), false, false)
- testHandler := Handler(
- "codeberg.page", "raw.codeberg.org",
- giteaClient,
- "https://docs.codeberg.org/pages/raw-content/",
- []string{"/.well-known/acme-challenge/"},
- []string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"},
- []string{"pages"},
- cache.NewKeyValueCache(),
- cache.NewKeyValueCache(),
- cache.NewKeyValueCache(),
- )
+ cfg := config.GiteaConfig{
+ Root: "https://codeberg.org",
+ Token: "",
+ LFSEnabled: false,
+ FollowSymlinks: false,
+ }
+ giteaClient, _ := gitea.NewClient(cfg, cache.NewInMemoryCache())
+ serverCfg := config.ServerConfig{
+ MainDomain: "codeberg.page",
+ RawDomain: "raw.codeberg.page",
+ BlacklistedPaths: []string{
+ "/.well-known/acme-challenge/",
+ },
+ AllowedCorsDomains: []string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"},
+ PagesBranches: []string{"pages"},
+ }
+ testHandler := Handler(serverCfg, giteaClient, cache.NewInMemoryCache(), cache.NewInMemoryCache(), cache.NewInMemoryCache())
testCase := func(uri string, status int) {
t.Run(uri, func(t *testing.T) {
diff --git a/server/handler/try.go b/server/handler/try.go
index 6cfe08e..e76891d 100644
--- a/server/handler/try.go
+++ b/server/handler/try.go
@@ -1,6 +1,7 @@
package handler
import (
+ "fmt"
"net/http"
"strings"
@@ -17,8 +18,8 @@ import (
func tryUpstream(ctx *context.Context, giteaClient *gitea.Client,
mainDomainSuffix, trimmedHost string,
options *upstream.Options,
- canonicalDomainCache cache.SetGetKey,
- redirectsCache cache.SetGetKey,
+ canonicalDomainCache cache.ICache,
+ redirectsCache cache.ICache,
) {
// check if a canonical domain exists on a request on MainDomain
if strings.HasSuffix(trimmedHost, mainDomainSuffix) && !options.ServeRaw {
@@ -41,7 +42,7 @@ func tryUpstream(ctx *context.Context, giteaClient *gitea.Client,
// Try to request the file from the Gitea API
if !options.Upstream(ctx, giteaClient, redirectsCache) {
- html.ReturnErrorPage(ctx, "", ctx.StatusCode)
+ html.ReturnErrorPage(ctx, fmt.Sprintf("Forge returned %d %s", ctx.StatusCode, http.StatusText(ctx.StatusCode)), ctx.StatusCode)
}
}
diff --git a/server/startup.go b/server/startup.go
new file mode 100644
index 0000000..149a07d
--- /dev/null
+++ b/server/startup.go
@@ -0,0 +1,143 @@
+package server
+
+import (
+ "context"
+ "crypto/tls"
+ "encoding/json"
+ "fmt"
+ "net"
+ "net/http"
+ "os"
+ "strings"
+ "time"
+
+ "github.com/rs/zerolog"
+ "github.com/rs/zerolog/log"
+ "github.com/urfave/cli/v2"
+
+ cmd "codeberg.org/codeberg/pages/cli"
+ "codeberg.org/codeberg/pages/config"
+ "codeberg.org/codeberg/pages/server/acme"
+ "codeberg.org/codeberg/pages/server/cache"
+ "codeberg.org/codeberg/pages/server/certificates"
+ "codeberg.org/codeberg/pages/server/gitea"
+ "codeberg.org/codeberg/pages/server/handler"
+)
+
+// Serve sets up and starts the web server.
+func Serve(ctx *cli.Context) error {
+ // initialize logger with Trace, overridden later with actual level
+ log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger().Level(zerolog.TraceLevel)
+
+ cfg, err := config.ReadConfig(ctx)
+ if err != nil {
+ log.Error().Err(err).Msg("could not read config")
+ }
+
+ config.MergeConfig(ctx, cfg)
+
+ // Initialize the logger.
+ logLevel, err := zerolog.ParseLevel(cfg.LogLevel)
+ if err != nil {
+ return err
+ }
+ log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger().Level(logLevel)
+
+ foo, _ := json.Marshal(cfg)
+ log.Trace().RawJSON("config", foo).Msg("starting server with config")
+
+ listeningSSLAddress := fmt.Sprintf("%s:%d", cfg.Server.Host, cfg.Server.Port)
+ listeningHTTPAddress := fmt.Sprintf("%s:%d", cfg.Server.Host, cfg.Server.HttpPort)
+
+ if cfg.Server.RawDomain != "" {
+ cfg.Server.AllowedCorsDomains = append(cfg.Server.AllowedCorsDomains, cfg.Server.RawDomain)
+ }
+
+ // Make sure MainDomain has a leading dot
+ if !strings.HasPrefix(cfg.Server.MainDomain, ".") {
+ // TODO make this better
+ cfg.Server.MainDomain = "." + cfg.Server.MainDomain
+ }
+
+ if len(cfg.Server.PagesBranches) == 0 {
+ return fmt.Errorf("no default branches set (PAGES_BRANCHES)")
+ }
+
+ // Init ssl cert database
+ certDB, closeFn, err := cmd.OpenCertDB(ctx)
+ if err != nil {
+ return err
+ }
+ defer closeFn()
+
+ keyCache := cache.NewInMemoryCache()
+ challengeCache := cache.NewInMemoryCache()
+ // canonicalDomainCache stores canonical domains
+ canonicalDomainCache := cache.NewInMemoryCache()
+ // dnsLookupCache stores DNS lookups for custom domains
+ dnsLookupCache := cache.NewInMemoryCache()
+ // redirectsCache stores redirects in _redirects files
+ redirectsCache := cache.NewInMemoryCache()
+ // clientResponseCache stores responses from the Gitea server
+ clientResponseCache := cache.NewInMemoryCache()
+
+ giteaClient, err := gitea.NewClient(cfg.Gitea, clientResponseCache)
+ if err != nil {
+ return fmt.Errorf("could not create new gitea client: %v", err)
+ }
+
+ acmeClient, err := acme.CreateAcmeClient(cfg.ACME, cfg.Server.HttpServerEnabled, challengeCache)
+ if err != nil {
+ return err
+ }
+
+ if err := certificates.SetupMainDomainCertificates(cfg.Server.MainDomain, acmeClient, certDB); err != nil {
+ return err
+ }
+
+ // Create listener for SSL connections
+ log.Info().Msgf("Create TCP listener for SSL on %s", listeningSSLAddress)
+ listener, err := net.Listen("tcp", listeningSSLAddress)
+ if err != nil {
+ return fmt.Errorf("couldn't create listener: %v", err)
+ }
+
+ // Setup listener for SSL connections
+ listener = tls.NewListener(listener, certificates.TLSConfig(
+ cfg.Server.MainDomain,
+ giteaClient,
+ acmeClient,
+ cfg.Server.PagesBranches[0],
+ keyCache, challengeCache, dnsLookupCache, canonicalDomainCache,
+ certDB,
+ cfg.ACME.NoDNS01,
+ cfg.Server.RawDomain,
+ ))
+
+ interval := 12 * time.Hour
+ certMaintainCtx, cancelCertMaintain := context.WithCancel(context.Background())
+ defer cancelCertMaintain()
+ go certificates.MaintainCertDB(certMaintainCtx, interval, acmeClient, cfg.Server.MainDomain, certDB)
+
+ if cfg.Server.HttpServerEnabled {
+ // Create handler for http->https redirect and http acme challenges
+ httpHandler := certificates.SetupHTTPACMEChallengeServer(challengeCache, uint(cfg.Server.Port))
+
+ // Create listener for http and start listening
+ go func() {
+ log.Info().Msgf("Start HTTP server listening on %s", listeningHTTPAddress)
+ err := http.ListenAndServe(listeningHTTPAddress, httpHandler)
+ if err != nil {
+ log.Error().Err(err).Msg("Couldn't start HTTP server")
+ }
+ }()
+ }
+
+ // Create ssl handler based on settings
+ sslHandler := handler.Handler(cfg.Server, giteaClient, dnsLookupCache, canonicalDomainCache, redirectsCache)
+
+ // Start the ssl listener
+ log.Info().Msgf("Start SSL server using TCP listener on %s", listener.Addr())
+
+ return http.Serve(listener, sslHandler)
+}
diff --git a/server/upstream/domains.go b/server/upstream/domains.go
index eb30394..d53d586 100644
--- a/server/upstream/domains.go
+++ b/server/upstream/domains.go
@@ -17,7 +17,7 @@ var canonicalDomainCacheTimeout = 15 * time.Minute
const canonicalDomainConfig = ".domains"
// CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file).
-func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (domain string, valid bool) {
+func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.ICache) (domain string, valid bool) {
// Check if this request is cached.
if cachedValue, ok := canonicalDomainCache.Get(o.TargetOwner + "/" + o.TargetRepo + "/" + o.TargetBranch); ok {
domains := cachedValue.([]string)
@@ -41,7 +41,7 @@ func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain,
domain = strings.TrimSpace(domain)
domain = strings.TrimPrefix(domain, "http://")
domain = strings.TrimPrefix(domain, "https://")
- if len(domain) > 0 && !strings.HasPrefix(domain, "#") && !strings.ContainsAny(domain, "\t /") && strings.ContainsRune(domain, '.') {
+ if domain != "" && !strings.HasPrefix(domain, "#") && !strings.ContainsAny(domain, "\t /") && strings.ContainsRune(domain, '.') {
domains = append(domains, domain)
}
if domain == actualDomain {
diff --git a/server/upstream/redirects.go b/server/upstream/redirects.go
index ab6c971..ddde375 100644
--- a/server/upstream/redirects.go
+++ b/server/upstream/redirects.go
@@ -17,13 +17,34 @@ type Redirect struct {
StatusCode int
}
+// rewriteURL returns the destination URL and true if r matches reqURL.
+func (r *Redirect) rewriteURL(reqURL string) (dstURL string, ok bool) {
+ // check if from url matches request url
+ if strings.TrimSuffix(r.From, "/") == strings.TrimSuffix(reqURL, "/") {
+ return r.To, true
+ }
+ // handle wildcard redirects
+ if strings.HasSuffix(r.From, "/*") {
+ trimmedFromURL := strings.TrimSuffix(r.From, "/*")
+ if reqURL == trimmedFromURL || strings.HasPrefix(reqURL, trimmedFromURL+"/") {
+ if strings.Contains(r.To, ":splat") {
+ matched := strings.TrimPrefix(reqURL, trimmedFromURL)
+ matched = strings.TrimPrefix(matched, "/")
+ return strings.ReplaceAll(r.To, ":splat", matched), true
+ }
+ return r.To, true
+ }
+ }
+ return "", false
+}
+
// redirectsCacheTimeout specifies the timeout for the redirects cache.
var redirectsCacheTimeout = 10 * time.Minute
const redirectsConfig = "_redirects"
// getRedirects returns redirects specified in the _redirects file.
-func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.SetGetKey) []Redirect {
+func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.ICache) []Redirect {
var redirects []Redirect
cacheKey := o.TargetOwner + "/" + o.TargetRepo + "/" + o.TargetBranch
@@ -63,53 +84,22 @@ func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.S
return redirects
}
-func (o *Options) matchRedirects(ctx *context.Context, giteaClient *gitea.Client, redirects []Redirect, redirectsCache cache.SetGetKey) (final bool) {
- if len(redirects) > 0 {
- for _, redirect := range redirects {
- reqUrl := ctx.Req.RequestURI
- // remove repo and branch from request url
- reqUrl = strings.TrimPrefix(reqUrl, "/"+o.TargetRepo)
- reqUrl = strings.TrimPrefix(reqUrl, "/@"+o.TargetBranch)
+func (o *Options) matchRedirects(ctx *context.Context, giteaClient *gitea.Client, redirects []Redirect, redirectsCache cache.ICache) (final bool) {
+ reqURL := ctx.Req.RequestURI
+ // remove repo and branch from request url
+ reqURL = strings.TrimPrefix(reqURL, "/"+o.TargetRepo)
+ reqURL = strings.TrimPrefix(reqURL, "/@"+o.TargetBranch)
- // check if from url matches request url
- if strings.TrimSuffix(redirect.From, "/") == strings.TrimSuffix(reqUrl, "/") {
- // do rewrite if status code is 200
- if redirect.StatusCode == 200 {
- o.TargetPath = redirect.To
- o.Upstream(ctx, giteaClient, redirectsCache)
- return true
- } else {
- ctx.Redirect(redirect.To, redirect.StatusCode)
- return true
- }
- }
-
- // handle wildcard redirects
- trimmedFromUrl := strings.TrimSuffix(redirect.From, "/*")
- if strings.HasSuffix(redirect.From, "/*") && strings.HasPrefix(reqUrl, trimmedFromUrl) {
- if strings.Contains(redirect.To, ":splat") {
- splatUrl := strings.ReplaceAll(redirect.To, ":splat", strings.TrimPrefix(reqUrl, trimmedFromUrl))
- // do rewrite if status code is 200
- if redirect.StatusCode == 200 {
- o.TargetPath = splatUrl
- o.Upstream(ctx, giteaClient, redirectsCache)
- return true
- } else {
- ctx.Redirect(splatUrl, redirect.StatusCode)
- return true
- }
- } else {
- // do rewrite if status code is 200
- if redirect.StatusCode == 200 {
- o.TargetPath = redirect.To
- o.Upstream(ctx, giteaClient, redirectsCache)
- return true
- } else {
- ctx.Redirect(redirect.To, redirect.StatusCode)
- return true
- }
- }
+ for _, redirect := range redirects {
+ if dstURL, ok := redirect.rewriteURL(reqURL); ok {
+ // do rewrite if status code is 200
+ if redirect.StatusCode == 200 {
+ o.TargetPath = dstURL
+ o.Upstream(ctx, giteaClient, redirectsCache)
+ } else {
+ ctx.Redirect(dstURL, redirect.StatusCode)
}
+ return true
}
}
diff --git a/server/upstream/redirects_test.go b/server/upstream/redirects_test.go
new file mode 100644
index 0000000..6118a70
--- /dev/null
+++ b/server/upstream/redirects_test.go
@@ -0,0 +1,36 @@
+package upstream
+
+import (
+ "testing"
+)
+
+func TestRedirect_rewriteURL(t *testing.T) {
+ for _, tc := range []struct {
+ redirect Redirect
+ reqURL string
+ wantDstURL string
+ wantOk bool
+ }{
+ {Redirect{"/", "/dst", 200}, "/", "/dst", true},
+ {Redirect{"/", "/dst", 200}, "/foo", "", false},
+ {Redirect{"/src", "/dst", 200}, "/src", "/dst", true},
+ {Redirect{"/src", "/dst", 200}, "/foo", "", false},
+ {Redirect{"/src", "/dst", 200}, "/src/foo", "", false},
+ {Redirect{"/*", "/dst", 200}, "/", "/dst", true},
+ {Redirect{"/*", "/dst", 200}, "/src", "/dst", true},
+ {Redirect{"/src/*", "/dst/:splat", 200}, "/src", "/dst/", true},
+ {Redirect{"/src/*", "/dst/:splat", 200}, "/src/", "/dst/", true},
+ {Redirect{"/src/*", "/dst/:splat", 200}, "/src/foo", "/dst/foo", true},
+ {Redirect{"/src/*", "/dst/:splat", 200}, "/src/foo/bar", "/dst/foo/bar", true},
+ {Redirect{"/src/*", "/dst/:splatsuffix", 200}, "/src/foo", "/dst/foosuffix", true},
+ {Redirect{"/src/*", "/dst:splat", 200}, "/src/foo", "/dstfoo", true},
+ {Redirect{"/src/*", "/dst", 200}, "/srcfoo", "", false},
+ // This is the example from FEATURES.md:
+ {Redirect{"/articles/*", "/posts/:splat", 302}, "/articles/2022/10/12/post-1/", "/posts/2022/10/12/post-1/", true},
+ } {
+ if dstURL, ok := tc.redirect.rewriteURL(tc.reqURL); dstURL != tc.wantDstURL || ok != tc.wantOk {
+ t.Errorf("%#v.rewriteURL(%q) = %q, %v; want %q, %v",
+ tc.redirect, tc.reqURL, dstURL, ok, tc.wantDstURL, tc.wantOk)
+ }
+ }
+}
diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go
index f97c6ae..d9c131e 100644
--- a/server/upstream/upstream.go
+++ b/server/upstream/upstream.go
@@ -53,11 +53,13 @@ type Options struct {
}
// Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
-func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redirectsCache cache.SetGetKey) (final bool) {
+func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redirectsCache cache.ICache) bool {
log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger()
+ log.Debug().Msg("Start")
+
if o.TargetOwner == "" || o.TargetRepo == "" {
- html.ReturnErrorPage(ctx, "either repo owner or name info is missing", http.StatusBadRequest)
+ html.ReturnErrorPage(ctx, "forge client: either repo owner or name info is missing", http.StatusBadRequest)
return true
}
@@ -67,7 +69,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi
// handle 404
if err != nil && errors.Is(err, gitea.ErrorNotFound) || !branchExist {
html.ReturnErrorPage(ctx,
- fmt.Sprintf("branch %q for '%s/%s' not found", o.TargetBranch, o.TargetOwner, o.TargetRepo),
+ fmt.Sprintf("branch %q for %s/%s not found", o.TargetBranch, o.TargetOwner, o.TargetRepo),
http.StatusNotFound)
return true
}
@@ -75,7 +77,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi
// handle unexpected errors
if err != nil {
html.ReturnErrorPage(ctx,
- fmt.Sprintf("could not get timestamp of branch %q: %v", o.TargetBranch, err),
+ fmt.Sprintf("could not get timestamp of branch %q: '%v'", o.TargetBranch, err),
http.StatusFailedDependency)
return true
}
@@ -104,13 +106,16 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi
// Handle not found error
if err != nil && errors.Is(err, gitea.ErrorNotFound) {
+ log.Debug().Msg("Handling not found error")
// Get and match redirects
redirects := o.getRedirects(giteaClient, redirectsCache)
if o.matchRedirects(ctx, giteaClient, redirects, redirectsCache) {
+ log.Trace().Msg("redirect")
return true
}
if o.TryIndexPages {
+ log.Trace().Msg("try index page")
// copy the o struct & try if an index page exists
optionsForIndexPages := *o
optionsForIndexPages.TryIndexPages = false
@@ -121,6 +126,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi
return true
}
}
+ log.Trace().Msg("try html file with path name")
// compatibility fix for GitHub Pages (/example → /example.html)
optionsForIndexPages.appendTrailingSlash = false
optionsForIndexPages.redirectIfExists = strings.TrimSuffix(ctx.Path(), "/") + ".html"
@@ -130,8 +136,11 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi
}
}
+ log.Trace().Msg("not found")
+
ctx.StatusCode = http.StatusNotFound
if o.TryIndexPages {
+ log.Trace().Msg("try not found page")
// copy the o struct & try if a not found page exists
optionsForNotFoundPages := *o
optionsForNotFoundPages.TryIndexPages = false
@@ -142,6 +151,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi
return true
}
}
+ log.Trace().Msg("not found page missing")
}
return false
@@ -153,16 +163,16 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi
var msg string
if err != nil {
- msg = "gitea client returned unexpected error"
+ msg = "forge client: returned unexpected error"
log.Error().Err(err).Msg(msg)
- msg = fmt.Sprintf("%s: %v", msg, err)
+ msg = fmt.Sprintf("%s: '%v'", msg, err)
}
if reader == nil {
- msg = "gitea client returned no reader"
+ msg = "forge client: returned no reader"
log.Error().Msg(msg)
}
if statusCode != http.StatusOK {
- msg = fmt.Sprintf("Couldn't fetch contents (status code %d)", statusCode)
+ msg = fmt.Sprintf("forge client: couldn't fetch contents: %d - %s", statusCode, http.StatusText(statusCode))
log.Error().Msg(msg)
}
@@ -173,10 +183,12 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi
// Append trailing slash if missing (for index files), and redirect to fix filenames in general
// o.appendTrailingSlash is only true when looking for index pages
if o.appendTrailingSlash && !strings.HasSuffix(ctx.Path(), "/") {
+ log.Trace().Msg("append trailing slash and redirect")
ctx.Redirect(ctx.Path()+"/", http.StatusTemporaryRedirect)
return true
}
if strings.HasSuffix(ctx.Path(), "/index.html") && !o.ServeRaw {
+ log.Trace().Msg("remove index.html from path and redirect")
ctx.Redirect(strings.TrimSuffix(ctx.Path(), "index.html"), http.StatusTemporaryRedirect)
return true
}