From a8272f0ce9fab991229aad53f512e6aab9c771d2 Mon Sep 17 00:00:00 2001 From: crapStone Date: Wed, 15 Nov 2023 01:49:29 +0000 Subject: [PATCH 01/19] Don't send server version to client (#254) closes #247 Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/254 Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: crapStone Co-committed-by: crapStone --- server/handler/handler.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/server/handler/handler.go b/server/handler/handler.go index 7edcf95..fbdafd7 100644 --- a/server/handler/handler.go +++ b/server/handler/handler.go @@ -10,7 +10,6 @@ import ( "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/context" "codeberg.org/codeberg/pages/server/gitea" - "codeberg.org/codeberg/pages/server/version" ) const ( @@ -31,7 +30,7 @@ func Handler(mainDomainSuffix, rawDomain string, log := log.With().Strs("Handler", []string{req.Host, req.RequestURI}).Logger() ctx := context.New(w, req) - ctx.RespWriter.Header().Set("Server", "CodebergPages/"+version.Version) + ctx.RespWriter.Header().Set("Server", "pages-server") // Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin ctx.RespWriter.Header().Set("Referrer-Policy", "strict-origin-when-cross-origin") @@ -88,14 +87,14 @@ func Handler(mainDomainSuffix, rawDomain string, pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/") if rawDomain != "" && strings.EqualFold(trimmedHost, rawDomain) { - log.Debug().Msg("raw domain request detecded") + log.Debug().Msg("raw domain request detected") handleRaw(log, ctx, giteaClient, mainDomainSuffix, rawInfoPage, trimmedHost, pathElements, canonicalDomainCache, redirectsCache) } else if strings.HasSuffix(trimmedHost, mainDomainSuffix) { - log.Debug().Msg("subdomain request detecded") + log.Debug().Msg("subdomain request detected") handleSubDomain(log, ctx, giteaClient, mainDomainSuffix, defaultPagesBranches, @@ -103,7 +102,7 @@ func Handler(mainDomainSuffix, rawDomain string, pathElements, canonicalDomainCache, redirectsCache) } else { - log.Debug().Msg("custom domain request detecded") + log.Debug().Msg("custom domain request detected") handleCustomDomain(log, ctx, giteaClient, mainDomainSuffix, trimmedHost, From be92f30e6434131fda8a78f31ef6e10ea7e9dfdd Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Wed, 15 Nov 2023 10:27:27 +0000 Subject: [PATCH 02/19] Update gitea sdk to e23e8aa3004f (#257) Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/257 Reviewed-by: crapStone Co-authored-by: 6543 <6543@obermui.de> Co-committed-by: 6543 <6543@obermui.de> --- go.mod | 2 +- go.sum | 13 +++++-------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index bfde7f7..72eb289 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module codeberg.org/codeberg/pages go 1.20 require ( - code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa + code.gitea.io/sdk/gitea v0.16.1-0.20231115014337-e23e8aa3004f github.com/OrlovEvgeny/go-mcache v0.0.0-20200121124330-1a8195b34f3a github.com/go-acme/lego/v4 v4.5.3 github.com/go-sql-driver/mysql v1.6.0 diff --git a/go.sum b/go.sum index b10305c..f78ae18 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,8 @@ cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIA cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa h1:OVwgYrY6vr6gWZvgnmevFhtL0GVA4HKaFOhD+joPoNk= -code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa/go.mod h1:aRmrQC3CAHdJAU1LQt0C9zqzqI8tUB/5oQtNE746aYE= +code.gitea.io/sdk/gitea v0.16.1-0.20231115014337-e23e8aa3004f h1:nMmwDgUIAWj9XQjzHz5unC3ZMfhhwHRk6rnuwLzdu1o= +code.gitea.io/sdk/gitea v0.16.1-0.20231115014337-e23e8aa3004f/go.mod h1:ndkDk99BnfiUCCYEUhpNzi0lpmApXlwRFqClBlOlEBg= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:lSA0F4e9A2NcQSqGqTOXqu2aRi/XEQxDCBwM8yJtE6s= gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:EXuID2Zs0pAQhH8yz+DNjUbjppKQzKFAn28TMYPB6IU= @@ -249,8 +249,8 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= @@ -789,8 +789,8 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -899,8 +899,6 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210902050250-f475640dd07b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -964,14 +962,13 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e h1:4nW4NLDYnU28ojHaHO8OVxFHk/aQ33U01a9cjED+pzE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= From 1e1c67be93f59af3fad68cdc8faad72aac8a742b Mon Sep 17 00:00:00 2001 From: crapStone Date: Wed, 15 Nov 2023 15:25:14 +0000 Subject: [PATCH 03/19] let gitea client send user-agent with version (#258) closes #255 Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/258 Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: crapStone Co-committed-by: crapStone --- server/gitea/client.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/server/gitea/client.go b/server/gitea/client.go index 7a2bf63..f3bda54 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -17,12 +17,13 @@ import ( "github.com/rs/zerolog/log" "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/version" ) var ErrorNotFound = errors.New("not found") const ( - // cache key prefixe + // cache key prefixes branchTimestampCacheKeyPrefix = "branchTime" defaultBranchCacheKeyPrefix = "defaultBranch" rawContentCacheKeyPrefix = "rawContent" @@ -76,7 +77,13 @@ func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, follo defaultMimeType = "application/octet-stream" } - sdk, err := gitea.NewClient(giteaRoot, gitea.SetHTTPClient(&stdClient), gitea.SetToken(giteaAPIToken)) + sdk, err := gitea.NewClient( + giteaRoot, + gitea.SetHTTPClient(&stdClient), + gitea.SetToken(giteaAPIToken), + gitea.SetUserAgent("pages-server/"+version.Version), + ) + return &Client{ sdkClient: sdk, responseCache: respCache, @@ -172,7 +179,7 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str return reader, resp.Response.Header, resp.StatusCode, err } - // now we write to cache and respond at the sime time + // now we write to cache and respond at the same time fileResp := FileResponse{ Exists: true, ETag: resp.Header.Get(ETagHeader), @@ -278,11 +285,11 @@ func shouldRespBeSavedToCache(resp *http.Response) bool { return false } - contentLeng, err := strconv.ParseInt(contentLengthRaw, 10, 64) + contentLength, err := strconv.ParseInt(contentLengthRaw, 10, 64) if err != nil { log.Error().Err(err).Msg("could not parse content length") } // if content to big or could not be determined we not cache it - return contentLeng > 0 && contentLeng < fileCacheSizeLimit + return contentLength > 0 && contentLength < fileCacheSizeLimit } From ea68a82cd22a8a8c1f265260af22b9406f13e3a9 Mon Sep 17 00:00:00 2001 From: crapStone Date: Wed, 15 Nov 2023 15:25:56 +0000 Subject: [PATCH 04/19] new maintainer (#256) Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/256 Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: crapStone Co-committed-by: crapStone --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1cb6967..9ec3aab 100644 --- a/README.md +++ b/README.md @@ -91,9 +91,13 @@ Since we are working nicely in a team, it might be hard at times to get started (still check out the issues, we always aim to have some things to get you started). If you have any questions, want to work on a feature or could imagine collaborating with us for some time, -feel free to ping us in an issue or in a general Matrix chatgroup. +feel free to ping us in an issue or in a general [Matrix chat room](#chat-for-admins--devs). -You can also contact the maintainers of this project: +You can also contact the maintainer(s) of this project: + +- [crapStone](https://codeberg.org/crapStone) [(Matrix)](https://matrix.to/#/@crapstone:obermui.de) + +Previous maintainers: - [momar](https://codeberg.org/momar) [(Matrix)](https://matrix.to/#/@moritz:wuks.space) - [6543](https://codeberg.org/6543) [(Matrix)](https://matrix.to/#/@marddl:obermui.de) From 7f0a4e5ca999189527bf0739467787fe74a51f8a Mon Sep 17 00:00:00 2001 From: crapStone Date: Wed, 15 Nov 2023 17:59:04 +0000 Subject: [PATCH 05/19] small cleanup (#259) Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/259 Co-authored-by: crapStone Co-committed-by: crapStone --- FEATURES.md | 14 +++++++--- README.md | 27 +++++++++---------- .../haproxy-sni}/.gitignore | 0 .../haproxy-sni}/README.md | 0 .../haproxy-sni}/dhparam.pem | 0 .../haproxy-sni}/docker-compose.yml | 0 .../haproxy-sni}/gitea-www/index.html | 0 .../haproxy-sni}/gitea.Caddyfile | 0 .../haproxy-certificates/codeberg.org.pem | 0 .../haproxy-certificates/codeberg.org.pem.key | 0 .../haproxy-sni}/haproxy.cfg | 0 .../haproxy-sni}/pages-www/index.html | 0 .../haproxy-sni}/pages.Caddyfile | 0 {haproxy-sni => examples/haproxy-sni}/test.sh | 0 14 files changed, 23 insertions(+), 18 deletions(-) rename {haproxy-sni => examples/haproxy-sni}/.gitignore (100%) rename {haproxy-sni => examples/haproxy-sni}/README.md (100%) rename {haproxy-sni => examples/haproxy-sni}/dhparam.pem (100%) rename {haproxy-sni => examples/haproxy-sni}/docker-compose.yml (100%) rename {haproxy-sni => examples/haproxy-sni}/gitea-www/index.html (100%) rename {haproxy-sni => examples/haproxy-sni}/gitea.Caddyfile (100%) rename {haproxy-sni => examples/haproxy-sni}/haproxy-certificates/codeberg.org.pem (100%) rename {haproxy-sni => examples/haproxy-sni}/haproxy-certificates/codeberg.org.pem.key (100%) rename {haproxy-sni => examples/haproxy-sni}/haproxy.cfg (100%) rename {haproxy-sni => examples/haproxy-sni}/pages-www/index.html (100%) rename {haproxy-sni => examples/haproxy-sni}/pages.Caddyfile (100%) rename {haproxy-sni => examples/haproxy-sni}/test.sh (100%) diff --git a/FEATURES.md b/FEATURES.md index 7560a1d..3d2f394 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -2,13 +2,19 @@ ## Custom domains -... +Custom domains can be used by creating a `.domains` file with the domain name, e.g.: + +```text +codeberg.page +``` + +You also have to set some DNS records, see the [Codeberg Documentation](https://docs.codeberg.org/codeberg-pages/using-custom-domain/). ## Redirects Redirects can be created with a `_redirects` file with the following format: -``` +```text # Comment from to [status] ``` @@ -30,7 +36,7 @@ from to [status] Redirects all paths to `/index.html` for single-page apps. -``` +```text /* /index.html 200 ``` @@ -38,7 +44,7 @@ Redirects all paths to `/index.html` for single-page apps. Redirects every path under `/articles` to `/posts` while keeping the path. -``` +```text /articles/* /posts/:splat 302 ``` diff --git a/README.md b/README.md index 9ec3aab..39caa24 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,7 @@ It is suitable to be deployed by other Gitea instances, too, to offer static pag **End user documentation** can mainly be found at the [Wiki](https://codeberg.org/Codeberg/pages-server/wiki/Overview) and the [Codeberg Documentation](https://docs.codeberg.org/codeberg-pages/). - - Get It On Codeberg + Get It On Codeberg ## Quickstart @@ -61,18 +60,18 @@ but if you want to run it on a shared IP address (and not a standalone), you'll need to configure your reverse proxy not to terminate the TLS connections, but forward the requests on the IP level to the Pages Server. -You can check out a proof of concept in the `haproxy-sni` folder, -and especially have a look at [this section of the haproxy.cfg](https://codeberg.org/Codeberg/pages-server/src/branch/main/haproxy-sni/haproxy.cfg#L38). +You can check out a proof of concept in the `examples/haproxy-sni` folder, +and especially have a look at [this section of the haproxy.cfg](https://codeberg.org/Codeberg/pages-server/src/branch/main/examples/haproxy-sni/haproxy.cfg#L38). -### Environment +### Environment Variables - `HOST` & `PORT` (default: `[::]` & `443`): listen address. - `PAGES_DOMAIN` (default: `codeberg.page`): main domain for pages. -- `RAW_DOMAIN` (default: `raw.codeberg.page`): domain for raw resources. +- `RAW_DOMAIN` (default: `raw.codeberg.page`): domain for raw resources (must be subdomain of `PAGES_DOMAIN`). - `GITEA_ROOT` (default: `https://codeberg.org`): root of the upstream Gitea instance. - `GITEA_API_TOKEN` (default: empty): API token for the Gitea instance to access non-public (e.g. limited) repos. -- `RAW_INFO_PAGE` (default: https://docs.codeberg.org/pages/raw-content/): info page for raw resources, shown if no resource is provided. -- `ACME_API` (default: https://acme-v02.api.letsencrypt.org/directory): set this to https://acme.mock.director to use invalid certificates without any verification (great for debugging). +- `RAW_INFO_PAGE` (default: ): info page for raw resources, shown if no resource is provided. +- `ACME_API` (default: ): set this to to use invalid certificates without any verification (great for debugging). ZeroSSL might be better in the future as it doesn't have rate limits and doesn't clash with the official Codeberg certificates (which are using Let's Encrypt), but I couldn't get it to work yet. - `ACME_EMAIL` (default: `noreply@example.email`): Set the email sent to the ACME API server to receive, for example, renewal reminders. - `ACME_EAB_KID` & `ACME_EAB_HMAC` (default: don't use EAB): EAB credentials, for example for ZeroSSL. @@ -80,10 +79,9 @@ and especially have a look at [this section of the haproxy.cfg](https://codeberg - `ACME_USE_RATE_LIMITS` (default: true): Set this to false to disable rate limits, e.g. with ZeroSSL. - `ENABLE_HTTP_SERVER` (default: false): Set this to true to enable the HTTP-01 challenge and redirect all other HTTP requests to HTTPS. Currently only works with port 80. - `DNS_PROVIDER` (default: use self-signed certificate): Code of the ACME DNS provider for the main domain wildcard. - See https://go-acme.github.io/lego/dns/ for available values & additional environment variables. + See for available values & additional environment variables. - `LOG_LEVEL` (default: warn): Set this to specify the level of logging. - ## Contributing to the development The Codeberg team is very open to your contribution. @@ -119,7 +117,8 @@ Make sure you have [golang](https://go.dev) v1.20 or newer and [just](https://ju run `just dev` now this pages should work: - - https://cb_pages_tests.localhost.mock.directory:4430/images/827679288a.jpg - - https://momar.localhost.mock.directory:4430/ci-testing/ - - https://momar.localhost.mock.directory:4430/pag/@master/ - - https://mock-pages.codeberg-test.org:4430/README.md + +- +- +- +- diff --git a/haproxy-sni/.gitignore b/examples/haproxy-sni/.gitignore similarity index 100% rename from haproxy-sni/.gitignore rename to examples/haproxy-sni/.gitignore diff --git a/haproxy-sni/README.md b/examples/haproxy-sni/README.md similarity index 100% rename from haproxy-sni/README.md rename to examples/haproxy-sni/README.md diff --git a/haproxy-sni/dhparam.pem b/examples/haproxy-sni/dhparam.pem similarity index 100% rename from haproxy-sni/dhparam.pem rename to examples/haproxy-sni/dhparam.pem diff --git a/haproxy-sni/docker-compose.yml b/examples/haproxy-sni/docker-compose.yml similarity index 100% rename from haproxy-sni/docker-compose.yml rename to examples/haproxy-sni/docker-compose.yml diff --git a/haproxy-sni/gitea-www/index.html b/examples/haproxy-sni/gitea-www/index.html similarity index 100% rename from haproxy-sni/gitea-www/index.html rename to examples/haproxy-sni/gitea-www/index.html diff --git a/haproxy-sni/gitea.Caddyfile b/examples/haproxy-sni/gitea.Caddyfile similarity index 100% rename from haproxy-sni/gitea.Caddyfile rename to examples/haproxy-sni/gitea.Caddyfile diff --git a/haproxy-sni/haproxy-certificates/codeberg.org.pem b/examples/haproxy-sni/haproxy-certificates/codeberg.org.pem similarity index 100% rename from haproxy-sni/haproxy-certificates/codeberg.org.pem rename to examples/haproxy-sni/haproxy-certificates/codeberg.org.pem diff --git a/haproxy-sni/haproxy-certificates/codeberg.org.pem.key b/examples/haproxy-sni/haproxy-certificates/codeberg.org.pem.key similarity index 100% rename from haproxy-sni/haproxy-certificates/codeberg.org.pem.key rename to examples/haproxy-sni/haproxy-certificates/codeberg.org.pem.key diff --git a/haproxy-sni/haproxy.cfg b/examples/haproxy-sni/haproxy.cfg similarity index 100% rename from haproxy-sni/haproxy.cfg rename to examples/haproxy-sni/haproxy.cfg diff --git a/haproxy-sni/pages-www/index.html b/examples/haproxy-sni/pages-www/index.html similarity index 100% rename from haproxy-sni/pages-www/index.html rename to examples/haproxy-sni/pages-www/index.html diff --git a/haproxy-sni/pages.Caddyfile b/examples/haproxy-sni/pages.Caddyfile similarity index 100% rename from haproxy-sni/pages.Caddyfile rename to examples/haproxy-sni/pages.Caddyfile diff --git a/haproxy-sni/test.sh b/examples/haproxy-sni/test.sh similarity index 100% rename from haproxy-sni/test.sh rename to examples/haproxy-sni/test.sh From cbb2ce6d0732bcf2372cbc201fc1c1f2733aadba Mon Sep 17 00:00:00 2001 From: crapStone Date: Thu, 16 Nov 2023 17:11:35 +0000 Subject: [PATCH 06/19] add go templating engine for error page and make errors more clear (#260) ping #199 closes #213 Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/260 Co-authored-by: crapStone Co-committed-by: crapStone --- .woodpecker.yml | 2 +- README.md | 2 +- go.mod | 15 ++++-- go.sum | 30 +++++++---- html/404.html | 37 ------------- html/error.go | 50 ------------------ html/error.html | 38 -------------- html/error_test.go | 38 -------------- html/html.go | 54 +++++++++++++++++-- html/html_test.go | 54 +++++++++++++++++++ html/templates/error.html | 69 +++++++++++++++++++++++++ server/handler/handler.go | 2 +- server/handler/handler_custom_domain.go | 2 +- server/handler/handler_raw_domain.go | 2 +- server/handler/handler_sub_domain.go | 22 +++++--- server/handler/try.go | 2 +- server/upstream/upstream.go | 16 +++--- 17 files changed, 232 insertions(+), 203 deletions(-) delete mode 100644 html/404.html delete mode 100644 html/error.go delete mode 100644 html/error.html delete mode 100644 html/error_test.go create mode 100644 html/html_test.go create mode 100644 html/templates/error.html diff --git a/.woodpecker.yml b/.woodpecker.yml index 7de8ac8..759e2f1 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,7 +1,7 @@ steps: # use vendor to cache dependencies vendor: - image: golang:1.20 + image: golang:1.21 commands: - go mod vendor diff --git a/README.md b/README.md index 39caa24..fb2a4b9 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ Thank you very much. ### Test Server -Make sure you have [golang](https://go.dev) v1.20 or newer and [just](https://just.systems/man/en/) installed. +Make sure you have [golang](https://go.dev) v1.21 or newer and [just](https://just.systems/man/en/) installed. run `just dev` now this pages should work: diff --git a/go.mod b/go.mod index 72eb289..eba292e 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module codeberg.org/codeberg/pages -go 1.20 +go 1.21 + +toolchain go1.21.4 require ( code.gitea.io/sdk/gitea v0.16.1-0.20231115014337-e23e8aa3004f @@ -10,6 +12,7 @@ require ( github.com/joho/godotenv v1.4.0 github.com/lib/pq v1.10.7 github.com/mattn/go-sqlite3 v1.14.16 + github.com/microcosm-cc/bluemonday v1.0.26 github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad github.com/rs/zerolog v1.27.0 github.com/stretchr/testify v1.7.0 @@ -35,6 +38,7 @@ require ( github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1 // indirect github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183 // indirect github.com/aws/aws-sdk-go v1.39.0 // indirect + github.com/aymerick/douceur v0.2.0 // indirect github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect github.com/cenkalti/backoff/v4 v4.1.1 // indirect github.com/cloudflare/cloudflare-go v0.20.0 // indirect @@ -61,6 +65,7 @@ require ( github.com/googleapis/gax-go/v2 v2.0.5 // indirect github.com/gophercloud/gophercloud v0.16.0 // indirect github.com/gophercloud/utils v0.0.0-20210216074907-f6de111f2eae // indirect + github.com/gorilla/css v1.0.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.1 // indirect github.com/hashicorp/go-retryablehttp v0.7.0 // indirect github.com/hashicorp/go-version v1.6.0 // indirect @@ -115,11 +120,11 @@ require ( github.com/vultr/govultr/v2 v2.7.1 // indirect go.opencensus.io v0.22.3 // indirect go.uber.org/ratelimit v0.0.0-20180316092928-c15da0234277 // indirect - golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect - golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect - golang.org/x/sys v0.1.0 // indirect - golang.org/x/text v0.3.6 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 // indirect google.golang.org/api v0.20.0 // indirect google.golang.org/appengine v1.6.5 // indirect diff --git a/go.sum b/go.sum index f78ae18..7ea8b78 100644 --- a/go.sum +++ b/go.sum @@ -88,6 +88,8 @@ github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN github.com/aws/aws-sdk-go v1.39.0 h1:74BBwkEmiqBbi2CGflEh34l0YNtIibTjZsibGarkNjo= github.com/aws/aws-sdk-go v1.39.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -251,6 +253,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= @@ -278,6 +281,8 @@ github.com/gophercloud/utils v0.0.0-20210216074907-f6de111f2eae/go.mod h1:wx8HMD github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= +github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= @@ -477,6 +482,8 @@ github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S github.com/mattn/go-tty v0.0.0-20180219170247-931426f7535a/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3rQ0k/Khz58= +github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= @@ -756,8 +763,8 @@ golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -790,7 +797,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -826,8 +834,9 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -899,11 +908,12 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210902050250-f475640dd07b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -911,8 +921,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -963,7 +974,8 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/html/404.html b/html/404.html deleted file mode 100644 index 7c721b5..0000000 --- a/html/404.html +++ /dev/null @@ -1,37 +0,0 @@ - - - - - - %status% - - - - - - - - - -

- Page not found! -

-
- Sorry, but this page couldn't be found or is inaccessible (%status%).
- We hope this isn't a problem on our end ;) - Make sure to check the troubleshooting section in the Docs! -
- - - Static pages made easy - Codeberg Pages - - - diff --git a/html/error.go b/html/error.go deleted file mode 100644 index 206b123..0000000 --- a/html/error.go +++ /dev/null @@ -1,50 +0,0 @@ -package html - -import ( - "html/template" - "net/http" - "strconv" - "strings" - - "codeberg.org/codeberg/pages/server/context" -) - -// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body, -// with "%status%" and %message% replaced with the provided statusCode and msg -func ReturnErrorPage(ctx *context.Context, msg string, statusCode int) { - ctx.RespWriter.Header().Set("Content-Type", "text/html; charset=utf-8") - ctx.RespWriter.WriteHeader(statusCode) - - msg = generateResponse(msg, statusCode) - - _, _ = ctx.RespWriter.Write([]byte(msg)) -} - -// TODO: use template engine -func generateResponse(msg string, statusCode int) string { - if msg == "" { - msg = strings.ReplaceAll(NotFoundPage, - "%status%", - strconv.Itoa(statusCode)+" "+errorMessage(statusCode)) - } else { - msg = strings.ReplaceAll( - strings.ReplaceAll(ErrorPage, "%message%", template.HTMLEscapeString(msg)), - "%status%", - http.StatusText(statusCode)) - } - - return msg -} - -func errorMessage(statusCode int) string { - message := http.StatusText(statusCode) - - switch statusCode { - case http.StatusMisdirectedRequest: - message += " - domain not specified in .domains file" - case http.StatusFailedDependency: - message += " - target repo/branch doesn't exist or is private" - } - - return message -} diff --git a/html/error.html b/html/error.html deleted file mode 100644 index f1975f7..0000000 --- a/html/error.html +++ /dev/null @@ -1,38 +0,0 @@ - - - - - - %status% - - - - - - - - - -

- %status%! -

-
- Sorry, but this page couldn't be served.
- We got an "%message%"
- We hope this isn't a problem on our end ;) - Make sure to check the troubleshooting section in the Docs! -
- - - Static pages made easy - Codeberg Pages - - - diff --git a/html/error_test.go b/html/error_test.go deleted file mode 100644 index f5da08c..0000000 --- a/html/error_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package html - -import ( - "net/http" - "strings" - "testing" -) - -func TestValidMessage(t *testing.T) { - testString := "requested blacklisted path" - statusCode := http.StatusForbidden - - expected := strings.ReplaceAll( - strings.ReplaceAll(ErrorPage, "%message%", testString), - "%status%", - http.StatusText(statusCode)) - actual := generateResponse(testString, statusCode) - - if expected != actual { - t.Errorf("generated response did not match: expected: '%s', got: '%s'", expected, actual) - } -} - -func TestMessageWithHtml(t *testing.T) { - testString := `abc +func ReturnErrorPage(ctx *context.Context, msg string, statusCode int) { + ctx.RespWriter.Header().Set("Content-Type", "text/html; charset=utf-8") + ctx.RespWriter.WriteHeader(statusCode) + + templateContext := TemplateContext{ + StatusCode: statusCode, + StatusText: http.StatusText(statusCode), + Message: sanitizer.Sanitize(msg), + } + + err := errorTemplate.Execute(ctx.RespWriter, templateContext) + if err != nil { + log.Err(err).Str("message", msg).Int("status", statusCode).Msg("could not write response") + } +} + +func createBlueMondayPolicy() *bluemonday.Policy { + p := bluemonday.NewPolicy() + + p.AllowElements("code") + + return p +} diff --git a/html/html_test.go b/html/html_test.go new file mode 100644 index 0000000..b395bb2 --- /dev/null +++ b/html/html_test.go @@ -0,0 +1,54 @@ +package html + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSanitizerSimpleString(t *testing.T) { + str := "simple text message without any html elements" + + assert.Equal(t, str, sanitizer.Sanitize(str)) +} + +func TestSanitizerStringWithCodeTag(t *testing.T) { + str := "simple text message with html tag" + + assert.Equal(t, str, sanitizer.Sanitize(str)) +} + +func TestSanitizerStringWithCodeTagWithAttribute(t *testing.T) { + str := "simple text message with html tag" + expected := "simple text message with html tag" + + assert.Equal(t, expected, sanitizer.Sanitize(str)) +} + +func TestSanitizerStringWithATag(t *testing.T) { + str := "simple text message with a link to another page" + expected := "simple text message with a link to another page" + + assert.Equal(t, expected, sanitizer.Sanitize(str)) +} + +func TestSanitizerStringWithATagAndHref(t *testing.T) { + str := "simple text message with a link to another page" + expected := "simple text message with a link to another page" + + assert.Equal(t, expected, sanitizer.Sanitize(str)) +} + +func TestSanitizerStringWithImgTag(t *testing.T) { + str := "simple text message with a \"not" + expected := "simple text message with a " + + assert.Equal(t, expected, sanitizer.Sanitize(str)) +} + +func TestSanitizerStringWithImgTagAndOnerrorAttribute(t *testing.T) { + str := "simple text message with a \"not" + expected := "simple text message with a " + + assert.Equal(t, expected, sanitizer.Sanitize(str)) +} diff --git a/html/templates/error.html b/html/templates/error.html new file mode 100644 index 0000000..6094a26 --- /dev/null +++ b/html/templates/error.html @@ -0,0 +1,69 @@ + + + + + + {{.StatusText}} + + + + + + + + + + +

{{.StatusText}} ({{.StatusCode}})!

+
+

Sorry, but this page couldn't be served.

+

"{{.Message}}"

+

+ We hope this isn't a problem on our end ;) - Make sure to check the + troubleshooting section in the Docs! +

+
+ + + Static pages made easy - + Codeberg Pages + + + diff --git a/server/handler/handler.go b/server/handler/handler.go index fbdafd7..a3011f3 100644 --- a/server/handler/handler.go +++ b/server/handler/handler.go @@ -65,7 +65,7 @@ func Handler(mainDomainSuffix, rawDomain string, // Block blacklisted paths (like ACME challenges) for _, blacklistedPath := range blacklistedPaths { if strings.HasPrefix(ctx.Path(), blacklistedPath) { - html.ReturnErrorPage(ctx, "requested blacklisted path", http.StatusForbidden) + html.ReturnErrorPage(ctx, "requested path is blacklisted", http.StatusForbidden) return } } diff --git a/server/handler/handler_custom_domain.go b/server/handler/handler_custom_domain.go index 299d7cf..8742be4 100644 --- a/server/handler/handler_custom_domain.go +++ b/server/handler/handler_custom_domain.go @@ -49,7 +49,7 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g }, canonicalLink); works { canonicalDomain, valid := targetOpt.CheckCanonicalDomain(giteaClient, trimmedHost, mainDomainSuffix, canonicalDomainCache) if !valid { - html.ReturnErrorPage(ctx, "", http.StatusMisdirectedRequest) + html.ReturnErrorPage(ctx, "domain not specified in .domains file", http.StatusMisdirectedRequest) return } else if canonicalDomain != trimmedHost { // only redirect if the target is also a codeberg page! diff --git a/server/handler/handler_raw_domain.go b/server/handler/handler_raw_domain.go index aa41c52..b87991e 100644 --- a/server/handler/handler_raw_domain.go +++ b/server/handler/handler_raw_domain.go @@ -61,7 +61,7 @@ func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Clie tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache) } else { html.ReturnErrorPage(ctx, - fmt.Sprintf("raw domain could not find repo '%s/%s' or repo is empty", targetOpt.TargetOwner, targetOpt.TargetRepo), + fmt.Sprintf("raw domain could not find repo %s/%s or repo is empty", targetOpt.TargetOwner, targetOpt.TargetRepo), http.StatusNotFound) } } diff --git a/server/handler/handler_sub_domain.go b/server/handler/handler_sub_domain.go index 8731bec..e7cb3a6 100644 --- a/server/handler/handler_sub_domain.go +++ b/server/handler/handler_sub_domain.go @@ -55,9 +55,11 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite log.Trace().Msg("tryUpstream: serve with specified repo and branch") tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache) } else { - html.ReturnErrorPage(ctx, - fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo), - http.StatusFailedDependency) + html.ReturnErrorPage( + ctx, + formatSetBranchNotFoundMessage(targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo), + http.StatusFailedDependency, + ) } return } @@ -85,9 +87,11 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite log.Trace().Msg("tryUpstream: serve default pages repo with specified branch") tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache) } else { - html.ReturnErrorPage(ctx, - fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo), - http.StatusFailedDependency) + html.ReturnErrorPage( + ctx, + formatSetBranchNotFoundMessage(targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo), + http.StatusFailedDependency, + ) } return } @@ -143,6 +147,10 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite // Couldn't find a valid repo/branch html.ReturnErrorPage(ctx, - fmt.Sprintf("could not find a valid repository[%s]", targetRepo), + fmt.Sprintf("could not find a valid repository or branch for repository: %s", targetRepo), http.StatusNotFound) } + +func formatSetBranchNotFoundMessage(branch, owner, repo string) string { + return fmt.Sprintf("explicitly set branch %q does not exist at %s/%s", branch, owner, repo) +} diff --git a/server/handler/try.go b/server/handler/try.go index 6cfe08e..838ae27 100644 --- a/server/handler/try.go +++ b/server/handler/try.go @@ -41,7 +41,7 @@ func tryUpstream(ctx *context.Context, giteaClient *gitea.Client, // Try to request the file from the Gitea API if !options.Upstream(ctx, giteaClient, redirectsCache) { - html.ReturnErrorPage(ctx, "", ctx.StatusCode) + html.ReturnErrorPage(ctx, "gitea client failed", ctx.StatusCode) } } diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index f97c6ae..1a444e4 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -53,11 +53,11 @@ type Options struct { } // Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context. -func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redirectsCache cache.SetGetKey) (final bool) { +func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redirectsCache cache.SetGetKey) bool { log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger() if o.TargetOwner == "" || o.TargetRepo == "" { - html.ReturnErrorPage(ctx, "either repo owner or name info is missing", http.StatusBadRequest) + html.ReturnErrorPage(ctx, "gitea client: either repo owner or name info is missing", http.StatusBadRequest) return true } @@ -67,7 +67,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi // handle 404 if err != nil && errors.Is(err, gitea.ErrorNotFound) || !branchExist { html.ReturnErrorPage(ctx, - fmt.Sprintf("branch %q for '%s/%s' not found", o.TargetBranch, o.TargetOwner, o.TargetRepo), + fmt.Sprintf("branch %q for %s/%s not found", o.TargetBranch, o.TargetOwner, o.TargetRepo), http.StatusNotFound) return true } @@ -75,7 +75,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi // handle unexpected errors if err != nil { html.ReturnErrorPage(ctx, - fmt.Sprintf("could not get timestamp of branch %q: %v", o.TargetBranch, err), + fmt.Sprintf("could not get timestamp of branch %q: '%v'", o.TargetBranch, err), http.StatusFailedDependency) return true } @@ -153,16 +153,16 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi var msg string if err != nil { - msg = "gitea client returned unexpected error" + msg = "gitea client: returned unexpected error" log.Error().Err(err).Msg(msg) - msg = fmt.Sprintf("%s: %v", msg, err) + msg = fmt.Sprintf("%s: '%v'", msg, err) } if reader == nil { - msg = "gitea client returned no reader" + msg = "gitea client: returned no reader" log.Error().Msg(msg) } if statusCode != http.StatusOK { - msg = fmt.Sprintf("Couldn't fetch contents (status code %d)", statusCode) + msg = fmt.Sprintf("gitea client: couldn't fetch contents: %d - %s", statusCode, http.StatusText(statusCode)) log.Error().Msg(msg) } From fffb8ffcb68d44572830bf03f90b4fa5b9db262e Mon Sep 17 00:00:00 2001 From: crapStone Date: Thu, 16 Nov 2023 17:33:39 +0000 Subject: [PATCH 07/19] remove use of rawInfoPage redirect (#261) closes #244 Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/261 Co-authored-by: crapStone Co-committed-by: crapStone --- cmd/flags.go | 7 ------- cmd/main.go | 2 -- server/handler/handler.go | 3 +-- server/handler/handler_raw_domain.go | 10 +++++++--- server/handler/handler_test.go | 1 - 5 files changed, 8 insertions(+), 15 deletions(-) diff --git a/cmd/flags.go b/cmd/flags.go index a71dd35..7ac94e6 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -72,13 +72,6 @@ var ( EnvVars: []string{"RAW_DOMAIN"}, Value: "raw.codeberg.page", }, - // RawInfoPage will be shown (with a redirect) when trying to access RawDomain directly (or without owner/repo/path). - &cli.StringFlag{ - Name: "raw-info-page", - Usage: "will be shown (with a redirect) when trying to access $RAW_DOMAIN directly (or without owner/repo/path)", - EnvVars: []string{"RAW_INFO_PAGE"}, - Value: "https://docs.codeberg.org/codeberg-pages/raw-content/", - }, // ######################### // ### Page Server Setup ### diff --git a/cmd/main.go b/cmd/main.go index 84915c9..683e859 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -47,7 +47,6 @@ func Serve(ctx *cli.Context) error { rawDomain := ctx.String("raw-domain") defaultBranches := ctx.StringSlice("pages-branch") mainDomainSuffix := ctx.String("pages-domain") - rawInfoPage := ctx.String("raw-info-page") listeningHost := ctx.String("host") listeningSSLPort := ctx.Uint("port") listeningSSLAddress := fmt.Sprintf("%s:%d", listeningHost, listeningSSLPort) @@ -137,7 +136,6 @@ func Serve(ctx *cli.Context) error { // Create ssl handler based on settings sslHandler := handler.Handler(mainDomainSuffix, rawDomain, giteaClient, - rawInfoPage, BlacklistedPaths, allowedCorsDomains, defaultBranches, dnsLookupCache, canonicalDomainCache, redirectsCache) diff --git a/server/handler/handler.go b/server/handler/handler.go index a3011f3..7da5d39 100644 --- a/server/handler/handler.go +++ b/server/handler/handler.go @@ -21,7 +21,6 @@ const ( // Handler handles a single HTTP request to the web server. func Handler(mainDomainSuffix, rawDomain string, giteaClient *gitea.Client, - rawInfoPage string, blacklistedPaths, allowedCorsDomains []string, defaultPagesBranches []string, dnsLookupCache, canonicalDomainCache, redirectsCache cache.SetGetKey, @@ -89,7 +88,7 @@ func Handler(mainDomainSuffix, rawDomain string, if rawDomain != "" && strings.EqualFold(trimmedHost, rawDomain) { log.Debug().Msg("raw domain request detected") handleRaw(log, ctx, giteaClient, - mainDomainSuffix, rawInfoPage, + mainDomainSuffix, trimmedHost, pathElements, canonicalDomainCache, redirectsCache) diff --git a/server/handler/handler_raw_domain.go b/server/handler/handler_raw_domain.go index b87991e..caa8209 100644 --- a/server/handler/handler_raw_domain.go +++ b/server/handler/handler_raw_domain.go @@ -16,7 +16,7 @@ import ( ) func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client, - mainDomainSuffix, rawInfoPage string, + mainDomainSuffix string, trimmedHost string, pathElements []string, canonicalDomainCache, redirectsCache cache.SetGetKey, @@ -25,8 +25,12 @@ func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Clie log.Debug().Msg("raw domain") if len(pathElements) < 2 { - // https://{RawDomain}/{owner}/{repo}[/@{branch}]/{path} is required - ctx.Redirect(rawInfoPage, http.StatusTemporaryRedirect) + html.ReturnErrorPage( + ctx, + "a url in the form of https://{domain}/{owner}/{repo}[/@{branch}]/{path} is required", + http.StatusBadRequest, + ) + return } diff --git a/server/handler/handler_test.go b/server/handler/handler_test.go index 6521633..d04ebda 100644 --- a/server/handler/handler_test.go +++ b/server/handler/handler_test.go @@ -16,7 +16,6 @@ func TestHandlerPerformance(t *testing.T) { testHandler := Handler( "codeberg.page", "raw.codeberg.org", giteaClient, - "https://docs.codeberg.org/pages/raw-content/", []string{"/.well-known/acme-challenge/"}, []string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"}, []string{"pages"}, From dd5124912ecde8246bdcaad223ecde7737928429 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Fri, 17 Nov 2023 21:46:52 +0000 Subject: [PATCH 08/19] CI: run on pull only once (#264) Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/264 Co-authored-by: 6543 <6543@obermui.de> Co-committed-by: 6543 <6543@obermui.de> --- .woodpecker.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.woodpecker.yml b/.woodpecker.yml index 759e2f1..de8341c 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,3 +1,6 @@ +when: + branch: main + steps: # use vendor to cache dependencies vendor: From 15916444e18c71c066ffe8a7cd37cf272ec115eb Mon Sep 17 00:00:00 2001 From: Gusted Date: Thu, 18 Jan 2024 14:31:46 +0000 Subject: [PATCH 09/19] Fix panic in formatting not found message (#276) Fix panic that was found in the logs, `targetOpt` is `nil`: http2: panic serving 10.0.3.1:[...]: runtime error: invalid memory address or nil pointer dereference net/http.(*http2serverConn).runHandler.func1() /usr/local/go/src/net/http/h2_bundle.go:6104 +0x145 panic({0x19c6820, 0x2d66db0}) /usr/local/go/src/runtime/panic.go:884 +0x213 codeberg.org/codeberg/pages/server/handler.handleSubDomain({{0x2008c68, 0xc00047df90}, 0x2, {0x0, 0x0}, {0xc0fe3ef800, 0x55, 0x1f4}, {0xc00047dfa0, 0x1, ...}, ...}, ...) /woodpecker/src/codeberg.org/Codeberg/pages-server/server/handler/handler_sub_domain.go:59 +0x5e0 Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/276 Reviewed-by: crapStone Co-authored-by: Gusted Co-committed-by: Gusted --- server/handler/handler_sub_domain.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/handler/handler_sub_domain.go b/server/handler/handler_sub_domain.go index e7cb3a6..6c14393 100644 --- a/server/handler/handler_sub_domain.go +++ b/server/handler/handler_sub_domain.go @@ -57,7 +57,7 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite } else { html.ReturnErrorPage( ctx, - formatSetBranchNotFoundMessage(targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo), + formatSetBranchNotFoundMessage(pathElements[1][1:], targetOwner, pathElements[0]), http.StatusFailedDependency, ) } @@ -89,7 +89,7 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite } else { html.ReturnErrorPage( ctx, - formatSetBranchNotFoundMessage(targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo), + formatSetBranchNotFoundMessage(targetBranch, targetOwner, defaultPagesRepo), http.StatusFailedDependency, ) } From a09bee68adf43c2ade9eb3264708f29b68d599ea Mon Sep 17 00:00:00 2001 From: "Panagiotis \"Ivory\" Vasilopoulos" Date: Thu, 18 Jan 2024 20:35:32 +0000 Subject: [PATCH 10/19] Meta: Redirect user support to Codeberg/Community (#277) Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/277 Reviewed-by: crapStone Co-authored-by: Panagiotis "Ivory" Vasilopoulos Co-committed-by: Panagiotis "Ivory" Vasilopoulos --- .gitea/ISSUE_TEMPLATE/config.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .gitea/ISSUE_TEMPLATE/config.yml diff --git a/.gitea/ISSUE_TEMPLATE/config.yml b/.gitea/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..5a9cce6 --- /dev/null +++ b/.gitea/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: true +contact_links: + - name: Codeberg Pages Usage Support + url: https://codeberg.org/Codeberg/Community/issues/ + about: If you need help with configuring Codeberg Pages on codeberg.org, please go here. From c1fbe861fe793eb335cab63a524e5ae2bb3fb7f1 Mon Sep 17 00:00:00 2001 From: crapStone Date: Sun, 11 Feb 2024 12:43:25 +0000 Subject: [PATCH 11/19] rename gitea to forge in html error messages (#287) closes #286 Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/287 Reviewed-by: Andreas Shimokawa Co-authored-by: crapStone Co-committed-by: crapStone --- server/certificates/certificates.go | 2 +- server/handler/try.go | 2 +- server/upstream/domains.go | 2 +- server/upstream/upstream.go | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index 3ae891a..4fb99fa 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -199,7 +199,7 @@ func (c *AcmeClient) retrieveCertFromDB(sni, mainDomainSuffix string, useDnsProv func (c *AcmeClient) obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user string, useDnsProvider bool, mainDomainSuffix string, keyDatabase database.CertDB) (*tls.Certificate, error) { name := strings.TrimPrefix(domains[0], "*") - if useDnsProvider && len(domains[0]) > 0 && domains[0][0] == '*' { + if useDnsProvider && domains[0] != "" && domains[0][0] == '*' { domains = domains[1:] } diff --git a/server/handler/try.go b/server/handler/try.go index 838ae27..bb79dce 100644 --- a/server/handler/try.go +++ b/server/handler/try.go @@ -41,7 +41,7 @@ func tryUpstream(ctx *context.Context, giteaClient *gitea.Client, // Try to request the file from the Gitea API if !options.Upstream(ctx, giteaClient, redirectsCache) { - html.ReturnErrorPage(ctx, "gitea client failed", ctx.StatusCode) + html.ReturnErrorPage(ctx, "forge client failed", ctx.StatusCode) } } diff --git a/server/upstream/domains.go b/server/upstream/domains.go index eb30394..230f268 100644 --- a/server/upstream/domains.go +++ b/server/upstream/domains.go @@ -41,7 +41,7 @@ func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, domain = strings.TrimSpace(domain) domain = strings.TrimPrefix(domain, "http://") domain = strings.TrimPrefix(domain, "https://") - if len(domain) > 0 && !strings.HasPrefix(domain, "#") && !strings.ContainsAny(domain, "\t /") && strings.ContainsRune(domain, '.') { + if domain != "" && !strings.HasPrefix(domain, "#") && !strings.ContainsAny(domain, "\t /") && strings.ContainsRune(domain, '.') { domains = append(domains, domain) } if domain == actualDomain { diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index 1a444e4..4a85503 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -57,7 +57,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger() if o.TargetOwner == "" || o.TargetRepo == "" { - html.ReturnErrorPage(ctx, "gitea client: either repo owner or name info is missing", http.StatusBadRequest) + html.ReturnErrorPage(ctx, "forge client: either repo owner or name info is missing", http.StatusBadRequest) return true } @@ -153,16 +153,16 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi var msg string if err != nil { - msg = "gitea client: returned unexpected error" + msg = "forge client: returned unexpected error" log.Error().Err(err).Msg(msg) msg = fmt.Sprintf("%s: '%v'", msg, err) } if reader == nil { - msg = "gitea client: returned no reader" + msg = "forge client: returned no reader" log.Error().Msg(msg) } if statusCode != http.StatusOK { - msg = fmt.Sprintf("gitea client: couldn't fetch contents: %d - %s", statusCode, http.StatusText(statusCode)) + msg = fmt.Sprintf("forge client: couldn't fetch contents: %d - %s", statusCode, http.StatusText(statusCode)) log.Error().Msg(msg) } From 7e80ade24b8aac072804122b343a2a1a70667983 Mon Sep 17 00:00:00 2001 From: crapStone Date: Thu, 15 Feb 2024 16:08:29 +0000 Subject: [PATCH 12/19] Add config file and rework cli parsing and passing of config values (#263) Co-authored-by: 6543 <6543@obermui.de> Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/263 Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: crapStone Co-committed-by: crapStone --- .env-dev | 11 + .vscode/launch.json | 26 + Justfile | 15 +- {cmd => cli}/certs.go | 6 +- {cmd => cli}/flags.go | 52 +- cli/setup.go | 39 ++ cmd/main.go | 150 ------ cmd/setup.go | 64 --- config/assets/test_config.toml | 33 ++ config/config.go | 46 ++ config/setup.go | 147 ++++++ config/setup_test.go | 596 ++++++++++++++++++++++ example_config.toml | 32 ++ go.mod | 8 +- go.sum | 16 +- integration/main_test.go | 17 +- main.go | 20 +- server/acme/client.go | 26 + server/cache/interface.go | 3 +- server/cache/{setup.go => memory.go} | 2 +- server/certificates/acme_client.go | 11 +- server/certificates/acme_config.go | 34 +- server/certificates/cached_challengers.go | 6 +- server/certificates/certificates.go | 2 +- server/dns/dns.go | 2 +- server/gitea/cache.go | 4 +- server/gitea/client.go | 28 +- server/handler/handler.go | 30 +- server/handler/handler_custom_domain.go | 2 +- server/handler/handler_raw_domain.go | 2 +- server/handler/handler_sub_domain.go | 2 +- server/handler/handler_test.go | 29 +- server/handler/try.go | 4 +- server/startup.go | 141 +++++ server/upstream/domains.go | 2 +- server/upstream/redirects.go | 4 +- server/upstream/upstream.go | 2 +- 37 files changed, 1270 insertions(+), 344 deletions(-) create mode 100644 .env-dev create mode 100644 .vscode/launch.json rename {cmd => cli}/certs.go (92%) rename {cmd => cli}/flags.go (80%) create mode 100644 cli/setup.go delete mode 100644 cmd/main.go delete mode 100644 cmd/setup.go create mode 100644 config/assets/test_config.toml create mode 100644 config/config.go create mode 100644 config/setup.go create mode 100644 config/setup_test.go create mode 100644 example_config.toml create mode 100644 server/acme/client.go rename server/cache/{setup.go => memory.go} (69%) create mode 100644 server/startup.go diff --git a/.env-dev b/.env-dev new file mode 100644 index 0000000..2286005 --- /dev/null +++ b/.env-dev @@ -0,0 +1,11 @@ +ACME_API=https://acme.mock.directory +ACME_ACCEPT_TERMS=true +PAGES_DOMAIN=localhost.mock.directory +RAW_DOMAIN=raw.localhost.mock.directory +PAGES_BRANCHES=pages,master,main +GITEA_ROOT=https://codeberg.org +PORT=4430 +HTTP_PORT=8880 +ENABLE_HTTP_SERVER=true +LOG_LEVEL=trace +ACME_ACCOUNT_CONFIG=integration/acme-account.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..d2cc8d1 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,26 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Launch PagesServer", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/main.go", + "args": ["sqlite", "sqlite_unlock_notify", "netgo"], + "envFile": "${workspaceFolder}/.env-dev" + }, + { + "name": "Launch PagesServer integration test", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/integration/main_test.go", + "args": ["codeberg.org/codeberg/pages/integration/..."], + "buildFlags": ["-tags", "'integration sqlite sqlite_unlock_notify netgo'"] + } + ] +} diff --git a/Justfile b/Justfile index 0b8f814..0bf38a3 100644 --- a/Justfile +++ b/Justfile @@ -4,14 +4,9 @@ TAGS := 'sqlite sqlite_unlock_notify netgo' dev: #!/usr/bin/env bash set -euxo pipefail - export ACME_API=https://acme.mock.directory - export ACME_ACCEPT_TERMS=true - export PAGES_DOMAIN=localhost.mock.directory - export RAW_DOMAIN=raw.localhost.mock.directory - export PORT=4430 - export HTTP_PORT=8880 - export ENABLE_HTTP_SERVER=true - export LOG_LEVEL=trace + set -a # automatically export all variables + source .env-dev + set +a go run -tags '{{TAGS}}' . build: @@ -42,10 +37,10 @@ tool-gofumpt: fi test: - go test -race -cover -tags '{{TAGS}}' codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/ + go test -race -cover -tags '{{TAGS}}' codeberg.org/codeberg/pages/config/ codeberg.org/codeberg/pages/html/ codeberg.org/codeberg/pages/server/... test-run TEST: - go test -race -tags '{{TAGS}}' -run "^{{TEST}}$" codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/ + go test -race -tags '{{TAGS}}' -run "^{{TEST}}$" codeberg.org/codeberg/pages/config/ codeberg.org/codeberg/pages/html/ codeberg.org/codeberg/pages/server/... integration: go test -race -tags 'integration {{TAGS}}' codeberg.org/codeberg/pages/integration/... diff --git a/cmd/certs.go b/cli/certs.go similarity index 92% rename from cmd/certs.go rename to cli/certs.go index 6012b6e..76df7f0 100644 --- a/cmd/certs.go +++ b/cli/certs.go @@ -1,4 +1,4 @@ -package cmd +package cli import ( "fmt" @@ -26,7 +26,7 @@ var Certs = &cli.Command{ } func listCerts(ctx *cli.Context) error { - certDB, closeFn, err := openCertDB(ctx) + certDB, closeFn, err := OpenCertDB(ctx) if err != nil { return err } @@ -53,7 +53,7 @@ func removeCert(ctx *cli.Context) error { domains := ctx.Args().Slice() - certDB, closeFn, err := openCertDB(ctx) + certDB, closeFn, err := OpenCertDB(ctx) if err != nil { return err } diff --git a/cmd/flags.go b/cli/flags.go similarity index 80% rename from cmd/flags.go rename to cli/flags.go index 7ac94e6..097cf4f 100644 --- a/cmd/flags.go +++ b/cli/flags.go @@ -1,4 +1,4 @@ -package cmd +package cli import ( "github.com/urfave/cli/v2" @@ -29,26 +29,35 @@ var ( Name: "gitea-root", Usage: "specifies the root URL of the Gitea instance, without a trailing slash.", EnvVars: []string{"GITEA_ROOT"}, - Value: "https://codeberg.org", }, // GiteaApiToken specifies an api token for the Gitea instance &cli.StringFlag{ Name: "gitea-api-token", Usage: "specifies an api token for the Gitea instance", EnvVars: []string{"GITEA_API_TOKEN"}, - Value: "", }, &cli.BoolFlag{ Name: "enable-lfs-support", Usage: "enable lfs support, require gitea >= v1.17.0 as backend", EnvVars: []string{"ENABLE_LFS_SUPPORT"}, - Value: true, + Value: false, }, &cli.BoolFlag{ Name: "enable-symlink-support", Usage: "follow symlinks if enabled, require gitea >= v1.18.0 as backend", EnvVars: []string{"ENABLE_SYMLINK_SUPPORT"}, - Value: true, + Value: false, + }, + &cli.StringFlag{ + Name: "default-mime-type", + Usage: "specifies the default mime type for files that don't have a specific mime type.", + EnvVars: []string{"DEFAULT_MIME_TYPE"}, + Value: "application/octet-stream", + }, + &cli.StringSliceFlag{ + Name: "forbidden-mime-types", + Usage: "specifies the forbidden mime types. Use this flag multiple times for multiple mime types.", + EnvVars: []string{"FORBIDDEN_MIME_TYPES"}, }, // ########################### @@ -61,7 +70,6 @@ var ( Name: "pages-domain", Usage: "specifies the main domain (starting with a dot) for which subdomains shall be served as static pages", EnvVars: []string{"PAGES_DOMAIN"}, - Value: "codeberg.page", }, // RawDomain specifies the domain from which raw repository content shall be served in the following format: // https://{RawDomain}/{owner}/{repo}[/{branch|tag|commit}/{version}]/{filepath...} @@ -70,7 +78,6 @@ var ( Name: "raw-domain", Usage: "specifies the domain from which raw repository content shall be served, not set disable raw content hosting", EnvVars: []string{"RAW_DOMAIN"}, - Value: "raw.codeberg.page", }, // ######################### @@ -98,19 +105,38 @@ var ( Name: "enable-http-server", Usage: "start a http server to redirect to https and respond to http acme challenges", EnvVars: []string{"ENABLE_HTTP_SERVER"}, + Value: false, }, + // Default branches to fetch assets from + &cli.StringSliceFlag{ + Name: "pages-branch", + Usage: "define a branch to fetch assets from. Use this flag multiple times for multiple branches.", + EnvVars: []string{"PAGES_BRANCHES"}, + Value: cli.NewStringSlice("pages"), + }, + + &cli.StringSliceFlag{ + Name: "allowed-cors-domains", + Usage: "specify allowed CORS domains. Use this flag multiple times for multiple domains.", + EnvVars: []string{"ALLOWED_CORS_DOMAINS"}, + }, + &cli.StringSliceFlag{ + Name: "blacklisted-paths", + Usage: "return an error on these url paths.Use this flag multiple times for multiple paths.", + EnvVars: []string{"BLACKLISTED_PATHS"}, + }, + &cli.StringFlag{ Name: "log-level", Value: "warn", Usage: "specify at which log level should be logged. Possible options: info, warn, error, fatal", EnvVars: []string{"LOG_LEVEL"}, }, - // Default branches to fetch assets from - &cli.StringSliceFlag{ - Name: "pages-branch", - Usage: "define a branch to fetch assets from", - EnvVars: []string{"PAGES_BRANCHES"}, - Value: cli.NewStringSlice("pages"), + &cli.StringFlag{ + Name: "config-file", + Usage: "specify the location of the config file", + Aliases: []string{"config"}, + EnvVars: []string{"CONFIG_FILE"}, }, // ############################ diff --git a/cli/setup.go b/cli/setup.go new file mode 100644 index 0000000..6bbff40 --- /dev/null +++ b/cli/setup.go @@ -0,0 +1,39 @@ +package cli + +import ( + "fmt" + + "github.com/rs/zerolog/log" + "github.com/urfave/cli/v2" + + "codeberg.org/codeberg/pages/server/database" + "codeberg.org/codeberg/pages/server/version" +) + +func CreatePagesApp() *cli.App { + app := cli.NewApp() + app.Name = "pages-server" + app.Version = version.Version + app.Usage = "pages server" + app.Flags = ServerFlags + app.Commands = []*cli.Command{ + Certs, + } + + return app +} + +func OpenCertDB(ctx *cli.Context) (certDB database.CertDB, closeFn func(), err error) { + certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn")) + if err != nil { + return nil, nil, fmt.Errorf("could not connect to database: %w", err) + } + + closeFn = func() { + if err := certDB.Close(); err != nil { + log.Error().Err(err) + } + } + + return certDB, closeFn, nil +} diff --git a/cmd/main.go b/cmd/main.go deleted file mode 100644 index 683e859..0000000 --- a/cmd/main.go +++ /dev/null @@ -1,150 +0,0 @@ -package cmd - -import ( - "context" - "crypto/tls" - "fmt" - "net" - "net/http" - "os" - "strings" - "time" - - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" - "github.com/urfave/cli/v2" - - "codeberg.org/codeberg/pages/server/cache" - "codeberg.org/codeberg/pages/server/certificates" - "codeberg.org/codeberg/pages/server/gitea" - "codeberg.org/codeberg/pages/server/handler" -) - -// AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed. -// TODO: make it a flag -var AllowedCorsDomains = []string{ - "fonts.codeberg.org", - "design.codeberg.org", -} - -// BlacklistedPaths specifies forbidden path prefixes for all Codeberg Pages. -// TODO: Make it a flag too -var BlacklistedPaths = []string{ - "/.well-known/acme-challenge/", -} - -// Serve sets up and starts the web server. -func Serve(ctx *cli.Context) error { - // Initialize the logger. - logLevel, err := zerolog.ParseLevel(ctx.String("log-level")) - if err != nil { - return err - } - log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger().Level(logLevel) - - giteaRoot := ctx.String("gitea-root") - giteaAPIToken := ctx.String("gitea-api-token") - rawDomain := ctx.String("raw-domain") - defaultBranches := ctx.StringSlice("pages-branch") - mainDomainSuffix := ctx.String("pages-domain") - listeningHost := ctx.String("host") - listeningSSLPort := ctx.Uint("port") - listeningSSLAddress := fmt.Sprintf("%s:%d", listeningHost, listeningSSLPort) - listeningHTTPAddress := fmt.Sprintf("%s:%d", listeningHost, ctx.Uint("http-port")) - enableHTTPServer := ctx.Bool("enable-http-server") - - allowedCorsDomains := AllowedCorsDomains - if rawDomain != "" { - allowedCorsDomains = append(allowedCorsDomains, rawDomain) - } - - // Make sure MainDomain has a trailing dot - if !strings.HasPrefix(mainDomainSuffix, ".") { - mainDomainSuffix = "." + mainDomainSuffix - } - - if len(defaultBranches) == 0 { - return fmt.Errorf("no default branches set (PAGES_BRANCHES)") - } - - // Init ssl cert database - certDB, closeFn, err := openCertDB(ctx) - if err != nil { - return err - } - defer closeFn() - - keyCache := cache.NewKeyValueCache() - challengeCache := cache.NewKeyValueCache() - // canonicalDomainCache stores canonical domains - canonicalDomainCache := cache.NewKeyValueCache() - // dnsLookupCache stores DNS lookups for custom domains - dnsLookupCache := cache.NewKeyValueCache() - // redirectsCache stores redirects in _redirects files - redirectsCache := cache.NewKeyValueCache() - // clientResponseCache stores responses from the Gitea server - clientResponseCache := cache.NewKeyValueCache() - - giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken, clientResponseCache, ctx.Bool("enable-symlink-support"), ctx.Bool("enable-lfs-support")) - if err != nil { - return fmt.Errorf("could not create new gitea client: %v", err) - } - - acmeClient, err := createAcmeClient(ctx, enableHTTPServer, challengeCache) - if err != nil { - return err - } - - if err := certificates.SetupMainDomainCertificates(mainDomainSuffix, acmeClient, certDB); err != nil { - return err - } - - // Create listener for SSL connections - log.Info().Msgf("Create TCP listener for SSL on %s", listeningSSLAddress) - listener, err := net.Listen("tcp", listeningSSLAddress) - if err != nil { - return fmt.Errorf("couldn't create listener: %v", err) - } - - // Setup listener for SSL connections - listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix, - giteaClient, - acmeClient, - defaultBranches[0], - keyCache, challengeCache, dnsLookupCache, canonicalDomainCache, - certDB)) - - interval := 12 * time.Hour - certMaintainCtx, cancelCertMaintain := context.WithCancel(context.Background()) - defer cancelCertMaintain() - go certificates.MaintainCertDB(certMaintainCtx, interval, acmeClient, mainDomainSuffix, certDB) - - if enableHTTPServer { - // Create handler for http->https redirect and http acme challenges - httpHandler := certificates.SetupHTTPACMEChallengeServer(challengeCache, listeningSSLPort) - - // Create listener for http and start listening - go func() { - log.Info().Msgf("Start HTTP server listening on %s", listeningHTTPAddress) - err := http.ListenAndServe(listeningHTTPAddress, httpHandler) - if err != nil { - log.Panic().Err(err).Msg("Couldn't start HTTP fastServer") - } - }() - } - - // Create ssl handler based on settings - sslHandler := handler.Handler(mainDomainSuffix, rawDomain, - giteaClient, - BlacklistedPaths, allowedCorsDomains, - defaultBranches, - dnsLookupCache, canonicalDomainCache, redirectsCache) - - // Start the ssl listener - log.Info().Msgf("Start SSL server using TCP listener on %s", listener.Addr()) - if err := http.Serve(listener, sslHandler); err != nil { - log.Panic().Err(err).Msg("Couldn't start fastServer") - } - - return nil -} diff --git a/cmd/setup.go b/cmd/setup.go deleted file mode 100644 index cde4bc9..0000000 --- a/cmd/setup.go +++ /dev/null @@ -1,64 +0,0 @@ -package cmd - -import ( - "errors" - "fmt" - - "github.com/rs/zerolog/log" - "github.com/urfave/cli/v2" - - "codeberg.org/codeberg/pages/server/cache" - "codeberg.org/codeberg/pages/server/certificates" - "codeberg.org/codeberg/pages/server/database" -) - -var ErrAcmeMissConfig = errors.New("ACME client has wrong config") - -func openCertDB(ctx *cli.Context) (certDB database.CertDB, closeFn func(), err error) { - certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn")) - if err != nil { - return nil, nil, fmt.Errorf("could not connect to database: %w", err) - } - - closeFn = func() { - if err := certDB.Close(); err != nil { - log.Error().Err(err) - } - } - - return certDB, closeFn, nil -} - -func createAcmeClient(ctx *cli.Context, enableHTTPServer bool, challengeCache cache.SetGetKey) (*certificates.AcmeClient, error) { - acmeAPI := ctx.String("acme-api-endpoint") - acmeMail := ctx.String("acme-email") - acmeEabHmac := ctx.String("acme-eab-hmac") - acmeEabKID := ctx.String("acme-eab-kid") - acmeAcceptTerms := ctx.Bool("acme-accept-terms") - dnsProvider := ctx.String("dns-provider") - acmeUseRateLimits := ctx.Bool("acme-use-rate-limits") - acmeAccountConf := ctx.String("acme-account-config") - - // check config - if (!acmeAcceptTerms || dnsProvider == "") && acmeAPI != "https://acme.mock.directory" { - return nil, fmt.Errorf("%w: you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER, unless $ACME_API is set to https://acme.mock.directory", ErrAcmeMissConfig) - } - if acmeEabHmac != "" && acmeEabKID == "" { - return nil, fmt.Errorf("%w: ACME_EAB_HMAC also needs ACME_EAB_KID to be set", ErrAcmeMissConfig) - } else if acmeEabHmac == "" && acmeEabKID != "" { - return nil, fmt.Errorf("%w: ACME_EAB_KID also needs ACME_EAB_HMAC to be set", ErrAcmeMissConfig) - } - - return certificates.NewAcmeClient( - acmeAccountConf, - acmeAPI, - acmeMail, - acmeEabHmac, - acmeEabKID, - dnsProvider, - acmeAcceptTerms, - enableHTTPServer, - acmeUseRateLimits, - challengeCache, - ) -} diff --git a/config/assets/test_config.toml b/config/assets/test_config.toml new file mode 100644 index 0000000..6a2f0d0 --- /dev/null +++ b/config/assets/test_config.toml @@ -0,0 +1,33 @@ +logLevel = 'trace' + +[server] +host = '127.0.0.1' +port = 443 +httpPort = 80 +httpServerEnabled = true +mainDomain = 'codeberg.page' +rawDomain = 'raw.codeberg.page' +allowedCorsDomains = ['fonts.codeberg.org', 'design.codeberg.org'] +blacklistedPaths = ['do/not/use'] + +[gitea] +root = 'codeberg.org' +token = 'XXXXXXXX' +lfsEnabled = true +followSymlinks = true +defaultMimeType = "application/wasm" +forbiddenMimeTypes = ["text/html"] + +[database] +type = 'sqlite' +conn = 'certs.sqlite' + +[ACME] +email = 'a@b.c' +apiEndpoint = 'https://example.com' +acceptTerms = false +useRateLimits = true +eab_hmac = 'asdf' +eab_kid = 'qwer' +dnsProvider = 'cloudflare.com' +accountConfigFile = 'nope' diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..6cb972b --- /dev/null +++ b/config/config.go @@ -0,0 +1,46 @@ +package config + +type Config struct { + LogLevel string `default:"warn"` + Server ServerConfig + Gitea GiteaConfig + Database DatabaseConfig + ACME ACMEConfig +} + +type ServerConfig struct { + Host string `default:"[::]"` + Port uint16 `default:"443"` + HttpPort uint16 `default:"80"` + HttpServerEnabled bool `default:"true"` + MainDomain string + RawDomain string + PagesBranches []string + AllowedCorsDomains []string + BlacklistedPaths []string +} + +type GiteaConfig struct { + Root string + Token string + LFSEnabled bool `default:"false"` + FollowSymlinks bool `default:"false"` + DefaultMimeType string `default:"application/octet-stream"` + ForbiddenMimeTypes []string +} + +type DatabaseConfig struct { + Type string `default:"sqlite3"` + Conn string `default:"certs.sqlite"` +} + +type ACMEConfig struct { + Email string + APIEndpoint string `default:"https://acme-v02.api.letsencrypt.org/directory"` + AcceptTerms bool `default:"false"` + UseRateLimits bool `default:"true"` + EAB_HMAC string + EAB_KID string + DNSProvider string + AccountConfigFile string `default:"acme-account.json"` +} diff --git a/config/setup.go b/config/setup.go new file mode 100644 index 0000000..e774084 --- /dev/null +++ b/config/setup.go @@ -0,0 +1,147 @@ +package config + +import ( + "os" + "path" + + "github.com/creasty/defaults" + "github.com/pelletier/go-toml/v2" + "github.com/rs/zerolog/log" + "github.com/urfave/cli/v2" +) + +var ALWAYS_BLACKLISTED_PATHS = []string{ + "/.well-known/acme-challenge/", +} + +func NewDefaultConfig() Config { + config := Config{} + if err := defaults.Set(&config); err != nil { + panic(err) + } + + // defaults does not support setting arrays from strings + config.Server.PagesBranches = []string{"main", "master", "pages"} + + return config +} + +func ReadConfig(ctx *cli.Context) (*Config, error) { + config := NewDefaultConfig() + // if config is not given as argument return empty config + if !ctx.IsSet("config-file") { + return &config, nil + } + + configFile := path.Clean(ctx.String("config-file")) + + log.Debug().Str("config-file", configFile).Msg("reading config file") + content, err := os.ReadFile(configFile) + if err != nil { + return nil, err + } + + err = toml.Unmarshal(content, &config) + return &config, err +} + +func MergeConfig(ctx *cli.Context, config *Config) { + if ctx.IsSet("log-level") { + config.LogLevel = ctx.String("log-level") + } + + mergeServerConfig(ctx, &config.Server) + mergeGiteaConfig(ctx, &config.Gitea) + mergeDatabaseConfig(ctx, &config.Database) + mergeACMEConfig(ctx, &config.ACME) +} + +func mergeServerConfig(ctx *cli.Context, config *ServerConfig) { + if ctx.IsSet("host") { + config.Host = ctx.String("host") + } + if ctx.IsSet("port") { + config.Port = uint16(ctx.Uint("port")) + } + if ctx.IsSet("http-port") { + config.HttpPort = uint16(ctx.Uint("http-port")) + } + if ctx.IsSet("enable-http-server") { + config.HttpServerEnabled = ctx.Bool("enable-http-server") + } + if ctx.IsSet("pages-domain") { + config.MainDomain = ctx.String("pages-domain") + } + if ctx.IsSet("raw-domain") { + config.RawDomain = ctx.String("raw-domain") + } + if ctx.IsSet("pages-branch") { + config.PagesBranches = ctx.StringSlice("pages-branch") + } + if ctx.IsSet("allowed-cors-domains") { + config.AllowedCorsDomains = ctx.StringSlice("allowed-cors-domains") + } + if ctx.IsSet("blacklisted-paths") { + config.BlacklistedPaths = ctx.StringSlice("blacklisted-paths") + } + + // add the paths that should always be blacklisted + config.BlacklistedPaths = append(config.BlacklistedPaths, ALWAYS_BLACKLISTED_PATHS...) +} + +func mergeGiteaConfig(ctx *cli.Context, config *GiteaConfig) { + if ctx.IsSet("gitea-root") { + config.Root = ctx.String("gitea-root") + } + if ctx.IsSet("gitea-api-token") { + config.Token = ctx.String("gitea-api-token") + } + if ctx.IsSet("enable-lfs-support") { + config.LFSEnabled = ctx.Bool("enable-lfs-support") + } + if ctx.IsSet("enable-symlink-support") { + config.FollowSymlinks = ctx.Bool("enable-symlink-support") + } + if ctx.IsSet("default-mime-type") { + config.DefaultMimeType = ctx.String("default-mime-type") + } + if ctx.IsSet("forbidden-mime-types") { + config.ForbiddenMimeTypes = ctx.StringSlice("forbidden-mime-types") + } +} + +func mergeDatabaseConfig(ctx *cli.Context, config *DatabaseConfig) { + if ctx.IsSet("db-type") { + config.Type = ctx.String("db-type") + } + if ctx.IsSet("db-conn") { + config.Conn = ctx.String("db-conn") + } +} + +func mergeACMEConfig(ctx *cli.Context, config *ACMEConfig) { + if ctx.IsSet("acme-email") { + config.Email = ctx.String("acme-email") + } + if ctx.IsSet("acme-api-endpoint") { + config.APIEndpoint = ctx.String("acme-api-endpoint") + } + if ctx.IsSet("acme-accept-terms") { + config.AcceptTerms = ctx.Bool("acme-accept-terms") + } + if ctx.IsSet("acme-use-rate-limits") { + config.UseRateLimits = ctx.Bool("acme-use-rate-limits") + } + if ctx.IsSet("acme-eab-hmac") { + config.EAB_HMAC = ctx.String("acme-eab-hmac") + } + if ctx.IsSet("acme-eab-kid") { + config.EAB_KID = ctx.String("acme-eab-kid") + } + if ctx.IsSet("dns-provider") { + config.DNSProvider = ctx.String("dns-provider") + } + if ctx.IsSet("acme-account-config") { + config.AccountConfigFile = ctx.String("acme-account-config") + } +} diff --git a/config/setup_test.go b/config/setup_test.go new file mode 100644 index 0000000..a863e2f --- /dev/null +++ b/config/setup_test.go @@ -0,0 +1,596 @@ +package config + +import ( + "context" + "os" + "testing" + + "github.com/pelletier/go-toml/v2" + "github.com/stretchr/testify/assert" + "github.com/urfave/cli/v2" + + cmd "codeberg.org/codeberg/pages/cli" +) + +func runApp(t *testing.T, fn func(*cli.Context) error, args []string) { + app := cmd.CreatePagesApp() + app.Action = fn + + appCtx, appCancel := context.WithCancel(context.Background()) + defer appCancel() + + // os.Args always contains the binary name + args = append([]string{"testing"}, args...) + + err := app.RunContext(appCtx, args) + assert.NoError(t, err) +} + +// fixArrayFromCtx fixes the number of "changed" strings in a string slice according to the number of values in the context. +// This is a workaround because the cli library has a bug where the number of values in the context gets bigger the more tests are run. +func fixArrayFromCtx(ctx *cli.Context, key string, expected []string) []string { + if ctx.IsSet(key) { + ctxSlice := ctx.StringSlice(key) + + if len(ctxSlice) > 1 { + for i := 1; i < len(ctxSlice); i++ { + expected = append([]string{"changed"}, expected...) + } + } + } + + return expected +} + +func readTestConfig() (*Config, error) { + content, err := os.ReadFile("assets/test_config.toml") + if err != nil { + return nil, err + } + + expectedConfig := NewDefaultConfig() + err = toml.Unmarshal(content, &expectedConfig) + if err != nil { + return nil, err + } + + return &expectedConfig, nil +} + +func TestReadConfigShouldReturnEmptyConfigWhenConfigArgEmpty(t *testing.T) { + runApp( + t, + func(ctx *cli.Context) error { + cfg, err := ReadConfig(ctx) + expected := NewDefaultConfig() + assert.Equal(t, &expected, cfg) + + return err + }, + []string{}, + ) +} + +func TestReadConfigShouldReturnConfigFromFileWhenConfigArgPresent(t *testing.T) { + runApp( + t, + func(ctx *cli.Context) error { + cfg, err := ReadConfig(ctx) + if err != nil { + return err + } + + expectedConfig, err := readTestConfig() + if err != nil { + return err + } + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + []string{"--config-file", "assets/test_config.toml"}, + ) +} + +func TestValuesReadFromConfigFileShouldBeOverwrittenByArgs(t *testing.T) { + runApp( + t, + func(ctx *cli.Context) error { + cfg, err := ReadConfig(ctx) + if err != nil { + return err + } + + MergeConfig(ctx, cfg) + + expectedConfig, err := readTestConfig() + if err != nil { + return err + } + + expectedConfig.LogLevel = "debug" + expectedConfig.Gitea.Root = "not-codeberg.org" + expectedConfig.ACME.AcceptTerms = true + expectedConfig.Server.Host = "172.17.0.2" + expectedConfig.Server.BlacklistedPaths = append(expectedConfig.Server.BlacklistedPaths, ALWAYS_BLACKLISTED_PATHS...) + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + []string{ + "--config-file", "assets/test_config.toml", + "--log-level", "debug", + "--gitea-root", "not-codeberg.org", + "--acme-accept-terms", + "--host", "172.17.0.2", + }, + ) +} + +func TestMergeConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T) { + runApp( + t, + func(ctx *cli.Context) error { + cfg := &Config{ + LogLevel: "original", + Server: ServerConfig{ + Host: "original", + Port: 8080, + HttpPort: 80, + HttpServerEnabled: false, + MainDomain: "original", + RawDomain: "original", + PagesBranches: []string{"original"}, + AllowedCorsDomains: []string{"original"}, + BlacklistedPaths: []string{"original"}, + }, + Gitea: GiteaConfig{ + Root: "original", + Token: "original", + LFSEnabled: false, + FollowSymlinks: false, + DefaultMimeType: "original", + ForbiddenMimeTypes: []string{"original"}, + }, + Database: DatabaseConfig{ + Type: "original", + Conn: "original", + }, + ACME: ACMEConfig{ + Email: "original", + APIEndpoint: "original", + AcceptTerms: false, + UseRateLimits: false, + EAB_HMAC: "original", + EAB_KID: "original", + DNSProvider: "original", + AccountConfigFile: "original", + }, + } + + MergeConfig(ctx, cfg) + + expectedConfig := &Config{ + LogLevel: "changed", + Server: ServerConfig{ + Host: "changed", + Port: 8443, + HttpPort: 443, + HttpServerEnabled: true, + MainDomain: "changed", + RawDomain: "changed", + PagesBranches: []string{"changed"}, + AllowedCorsDomains: []string{"changed"}, + BlacklistedPaths: append([]string{"changed"}, ALWAYS_BLACKLISTED_PATHS...), + }, + Gitea: GiteaConfig{ + Root: "changed", + Token: "changed", + LFSEnabled: true, + FollowSymlinks: true, + DefaultMimeType: "changed", + ForbiddenMimeTypes: []string{"changed"}, + }, + Database: DatabaseConfig{ + Type: "changed", + Conn: "changed", + }, + ACME: ACMEConfig{ + Email: "changed", + APIEndpoint: "changed", + AcceptTerms: true, + UseRateLimits: true, + EAB_HMAC: "changed", + EAB_KID: "changed", + DNSProvider: "changed", + AccountConfigFile: "changed", + }, + } + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + []string{ + "--log-level", "changed", + // Server + "--pages-domain", "changed", + "--raw-domain", "changed", + "--allowed-cors-domains", "changed", + "--blacklisted-paths", "changed", + "--pages-branch", "changed", + "--host", "changed", + "--port", "8443", + "--http-port", "443", + "--enable-http-server", + // Gitea + "--gitea-root", "changed", + "--gitea-api-token", "changed", + "--enable-lfs-support", + "--enable-symlink-support", + "--default-mime-type", "changed", + "--forbidden-mime-types", "changed", + // Database + "--db-type", "changed", + "--db-conn", "changed", + // ACME + "--acme-email", "changed", + "--acme-api-endpoint", "changed", + "--acme-accept-terms", + "--acme-use-rate-limits", + "--acme-eab-hmac", "changed", + "--acme-eab-kid", "changed", + "--dns-provider", "changed", + "--acme-account-config", "changed", + }, + ) +} + +func TestMergeServerConfigShouldAddDefaultBlacklistedPathsToBlacklistedPaths(t *testing.T) { + runApp( + t, + func(ctx *cli.Context) error { + cfg := &ServerConfig{} + mergeServerConfig(ctx, cfg) + + expected := ALWAYS_BLACKLISTED_PATHS + assert.Equal(t, expected, cfg.BlacklistedPaths) + + return nil + }, + []string{}, + ) +} + +func TestMergeServerConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T) { + for range []uint8{0, 1} { + runApp( + t, + func(ctx *cli.Context) error { + cfg := &ServerConfig{ + Host: "original", + Port: 8080, + HttpPort: 80, + HttpServerEnabled: false, + MainDomain: "original", + RawDomain: "original", + AllowedCorsDomains: []string{"original"}, + BlacklistedPaths: []string{"original"}, + } + + mergeServerConfig(ctx, cfg) + + expectedConfig := &ServerConfig{ + Host: "changed", + Port: 8443, + HttpPort: 443, + HttpServerEnabled: true, + MainDomain: "changed", + RawDomain: "changed", + AllowedCorsDomains: fixArrayFromCtx(ctx, "allowed-cors-domains", []string{"changed"}), + BlacklistedPaths: fixArrayFromCtx(ctx, "blacklisted-paths", append([]string{"changed"}, ALWAYS_BLACKLISTED_PATHS...)), + } + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + []string{ + "--pages-domain", "changed", + "--raw-domain", "changed", + "--allowed-cors-domains", "changed", + "--blacklisted-paths", "changed", + "--host", "changed", + "--port", "8443", + "--http-port", "443", + "--enable-http-server", + }, + ) + } +} + +func TestMergeServerConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgExists(t *testing.T) { + type testValuePair struct { + args []string + callback func(*ServerConfig) + } + testValuePairs := []testValuePair{ + {args: []string{"--host", "changed"}, callback: func(sc *ServerConfig) { sc.Host = "changed" }}, + {args: []string{"--port", "8443"}, callback: func(sc *ServerConfig) { sc.Port = 8443 }}, + {args: []string{"--http-port", "443"}, callback: func(sc *ServerConfig) { sc.HttpPort = 443 }}, + {args: []string{"--enable-http-server"}, callback: func(sc *ServerConfig) { sc.HttpServerEnabled = true }}, + {args: []string{"--pages-domain", "changed"}, callback: func(sc *ServerConfig) { sc.MainDomain = "changed" }}, + {args: []string{"--raw-domain", "changed"}, callback: func(sc *ServerConfig) { sc.RawDomain = "changed" }}, + {args: []string{"--pages-branch", "changed"}, callback: func(sc *ServerConfig) { sc.PagesBranches = []string{"changed"} }}, + {args: []string{"--allowed-cors-domains", "changed"}, callback: func(sc *ServerConfig) { sc.AllowedCorsDomains = []string{"changed"} }}, + {args: []string{"--blacklisted-paths", "changed"}, callback: func(sc *ServerConfig) { sc.BlacklistedPaths = []string{"changed"} }}, + } + + for _, pair := range testValuePairs { + runApp( + t, + func(ctx *cli.Context) error { + cfg := ServerConfig{ + Host: "original", + Port: 8080, + HttpPort: 80, + HttpServerEnabled: false, + MainDomain: "original", + RawDomain: "original", + PagesBranches: []string{"original"}, + AllowedCorsDomains: []string{"original"}, + BlacklistedPaths: []string{"original"}, + } + + expectedConfig := cfg + pair.callback(&expectedConfig) + expectedConfig.BlacklistedPaths = append(expectedConfig.BlacklistedPaths, ALWAYS_BLACKLISTED_PATHS...) + + expectedConfig.PagesBranches = fixArrayFromCtx(ctx, "pages-branch", expectedConfig.PagesBranches) + expectedConfig.AllowedCorsDomains = fixArrayFromCtx(ctx, "allowed-cors-domains", expectedConfig.AllowedCorsDomains) + expectedConfig.BlacklistedPaths = fixArrayFromCtx(ctx, "blacklisted-paths", expectedConfig.BlacklistedPaths) + + mergeServerConfig(ctx, &cfg) + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + pair.args, + ) + } +} + +func TestMergeGiteaConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T) { + runApp( + t, + func(ctx *cli.Context) error { + cfg := &GiteaConfig{ + Root: "original", + Token: "original", + LFSEnabled: false, + FollowSymlinks: false, + DefaultMimeType: "original", + ForbiddenMimeTypes: []string{"original"}, + } + + mergeGiteaConfig(ctx, cfg) + + expectedConfig := &GiteaConfig{ + Root: "changed", + Token: "changed", + LFSEnabled: true, + FollowSymlinks: true, + DefaultMimeType: "changed", + ForbiddenMimeTypes: fixArrayFromCtx(ctx, "forbidden-mime-types", []string{"changed"}), + } + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + []string{ + "--gitea-root", "changed", + "--gitea-api-token", "changed", + "--enable-lfs-support", + "--enable-symlink-support", + "--default-mime-type", "changed", + "--forbidden-mime-types", "changed", + }, + ) +} + +func TestMergeGiteaConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgExists(t *testing.T) { + type testValuePair struct { + args []string + callback func(*GiteaConfig) + } + testValuePairs := []testValuePair{ + {args: []string{"--gitea-root", "changed"}, callback: func(gc *GiteaConfig) { gc.Root = "changed" }}, + {args: []string{"--gitea-api-token", "changed"}, callback: func(gc *GiteaConfig) { gc.Token = "changed" }}, + {args: []string{"--enable-lfs-support"}, callback: func(gc *GiteaConfig) { gc.LFSEnabled = true }}, + {args: []string{"--enable-symlink-support"}, callback: func(gc *GiteaConfig) { gc.FollowSymlinks = true }}, + {args: []string{"--default-mime-type", "changed"}, callback: func(gc *GiteaConfig) { gc.DefaultMimeType = "changed" }}, + {args: []string{"--forbidden-mime-types", "changed"}, callback: func(gc *GiteaConfig) { gc.ForbiddenMimeTypes = []string{"changed"} }}, + } + + for _, pair := range testValuePairs { + runApp( + t, + func(ctx *cli.Context) error { + cfg := GiteaConfig{ + Root: "original", + Token: "original", + LFSEnabled: false, + FollowSymlinks: false, + DefaultMimeType: "original", + ForbiddenMimeTypes: []string{"original"}, + } + + expectedConfig := cfg + pair.callback(&expectedConfig) + + mergeGiteaConfig(ctx, &cfg) + + expectedConfig.ForbiddenMimeTypes = fixArrayFromCtx(ctx, "forbidden-mime-types", expectedConfig.ForbiddenMimeTypes) + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + pair.args, + ) + } +} + +func TestMergeDatabaseConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T) { + runApp( + t, + func(ctx *cli.Context) error { + cfg := &DatabaseConfig{ + Type: "original", + Conn: "original", + } + + mergeDatabaseConfig(ctx, cfg) + + expectedConfig := &DatabaseConfig{ + Type: "changed", + Conn: "changed", + } + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + []string{ + "--db-type", "changed", + "--db-conn", "changed", + }, + ) +} + +func TestMergeDatabaseConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgExists(t *testing.T) { + type testValuePair struct { + args []string + callback func(*DatabaseConfig) + } + testValuePairs := []testValuePair{ + {args: []string{"--db-type", "changed"}, callback: func(gc *DatabaseConfig) { gc.Type = "changed" }}, + {args: []string{"--db-conn", "changed"}, callback: func(gc *DatabaseConfig) { gc.Conn = "changed" }}, + } + + for _, pair := range testValuePairs { + runApp( + t, + func(ctx *cli.Context) error { + cfg := DatabaseConfig{ + Type: "original", + Conn: "original", + } + + expectedConfig := cfg + pair.callback(&expectedConfig) + + mergeDatabaseConfig(ctx, &cfg) + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + pair.args, + ) + } +} + +func TestMergeACMEConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T) { + runApp( + t, + func(ctx *cli.Context) error { + cfg := &ACMEConfig{ + Email: "original", + APIEndpoint: "original", + AcceptTerms: false, + UseRateLimits: false, + EAB_HMAC: "original", + EAB_KID: "original", + DNSProvider: "original", + AccountConfigFile: "original", + } + + mergeACMEConfig(ctx, cfg) + + expectedConfig := &ACMEConfig{ + Email: "changed", + APIEndpoint: "changed", + AcceptTerms: true, + UseRateLimits: true, + EAB_HMAC: "changed", + EAB_KID: "changed", + DNSProvider: "changed", + AccountConfigFile: "changed", + } + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + []string{ + "--acme-email", "changed", + "--acme-api-endpoint", "changed", + "--acme-accept-terms", + "--acme-use-rate-limits", + "--acme-eab-hmac", "changed", + "--acme-eab-kid", "changed", + "--dns-provider", "changed", + "--acme-account-config", "changed", + }, + ) +} + +func TestMergeACMEConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgExists(t *testing.T) { + type testValuePair struct { + args []string + callback func(*ACMEConfig) + } + testValuePairs := []testValuePair{ + {args: []string{"--acme-email", "changed"}, callback: func(gc *ACMEConfig) { gc.Email = "changed" }}, + {args: []string{"--acme-api-endpoint", "changed"}, callback: func(gc *ACMEConfig) { gc.APIEndpoint = "changed" }}, + {args: []string{"--acme-accept-terms"}, callback: func(gc *ACMEConfig) { gc.AcceptTerms = true }}, + {args: []string{"--acme-use-rate-limits"}, callback: func(gc *ACMEConfig) { gc.UseRateLimits = true }}, + {args: []string{"--acme-eab-hmac", "changed"}, callback: func(gc *ACMEConfig) { gc.EAB_HMAC = "changed" }}, + {args: []string{"--acme-eab-kid", "changed"}, callback: func(gc *ACMEConfig) { gc.EAB_KID = "changed" }}, + {args: []string{"--dns-provider", "changed"}, callback: func(gc *ACMEConfig) { gc.DNSProvider = "changed" }}, + {args: []string{"--acme-account-config", "changed"}, callback: func(gc *ACMEConfig) { gc.AccountConfigFile = "changed" }}, + } + + for _, pair := range testValuePairs { + runApp( + t, + func(ctx *cli.Context) error { + cfg := ACMEConfig{ + Email: "original", + APIEndpoint: "original", + AcceptTerms: false, + UseRateLimits: false, + EAB_HMAC: "original", + EAB_KID: "original", + DNSProvider: "original", + AccountConfigFile: "original", + } + + expectedConfig := cfg + pair.callback(&expectedConfig) + + mergeACMEConfig(ctx, &cfg) + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + pair.args, + ) + } +} diff --git a/example_config.toml b/example_config.toml new file mode 100644 index 0000000..30e77c4 --- /dev/null +++ b/example_config.toml @@ -0,0 +1,32 @@ +logLevel = 'debug' + +[server] +host = '[::]' +port = 443 +httpPort = 80 +httpServerEnabled = true +mainDomain = 'codeberg.page' +rawDomain = 'raw.codeberg.page' +pagesBranches = ["pages"] +allowedCorsDomains = [] +blacklistedPaths = [] + +[gitea] +root = 'https://codeberg.org' +token = 'ASDF1234' +lfsEnabled = true +followSymlinks = true + +[database] +type = 'sqlite' +conn = 'certs.sqlite' + +[ACME] +email = 'noreply@example.email' +apiEndpoint = 'https://acme-v02.api.letsencrypt.org/directory' +acceptTerms = false +useRateLimits = false +eab_hmac = '' +eab_kid = '' +dnsProvider = '' +accountConfigFile = 'acme-account.json' diff --git a/go.mod b/go.mod index eba292e..47e1e71 100644 --- a/go.mod +++ b/go.mod @@ -13,9 +13,10 @@ require ( github.com/lib/pq v1.10.7 github.com/mattn/go-sqlite3 v1.14.16 github.com/microcosm-cc/bluemonday v1.0.26 + github.com/pelletier/go-toml/v2 v2.1.0 github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad github.com/rs/zerolog v1.27.0 - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.8.4 github.com/urfave/cli/v2 v2.3.0 golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb xorm.io/xorm v1.3.2 @@ -44,6 +45,7 @@ require ( github.com/cloudflare/cloudflare-go v0.20.0 // indirect github.com/cpu/goacmedns v0.1.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect + github.com/creasty/defaults v1.7.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidmz/go-pageant v1.0.2 // indirect github.com/deepmap/oapi-codegen v1.6.1 // indirect @@ -113,7 +115,7 @@ require ( github.com/softlayer/softlayer-go v1.0.3 // indirect github.com/softlayer/xmlrpc v0.0.0-20200409220501-5f089df7cb7e // indirect github.com/spf13/cast v1.3.1 // indirect - github.com/stretchr/objx v0.3.0 // indirect + github.com/stretchr/objx v0.5.0 // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/transip/gotransip/v6 v6.6.1 // indirect github.com/vinyldns/go-vinyldns v0.0.0-20200917153823-148a5f6b8f14 // indirect @@ -135,6 +137,6 @@ require ( gopkg.in/ns1/ns1-go.v2 v2.6.2 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect xorm.io/builder v0.3.12 // indirect ) diff --git a/go.sum b/go.sum index 7ea8b78..1a10599 100644 --- a/go.sum +++ b/go.sum @@ -131,6 +131,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creasty/defaults v1.7.0 h1:eNdqZvc5B509z18lD8yc212CAqJNvfT1Jq6L8WowdBA= +github.com/creasty/defaults v1.7.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -570,6 +572,8 @@ github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTK github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= +github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= +github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -680,14 +684,19 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= @@ -1079,8 +1088,9 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/integration/main_test.go b/integration/main_test.go index 6566f78..86fd9d3 100644 --- a/integration/main_test.go +++ b/integration/main_test.go @@ -10,9 +10,10 @@ import ( "testing" "time" - "codeberg.org/codeberg/pages/cmd" - "github.com/urfave/cli/v2" + + cmd "codeberg.org/codeberg/pages/cli" + "codeberg.org/codeberg/pages/server" ) func TestMain(m *testing.M) { @@ -32,10 +33,7 @@ func TestMain(m *testing.M) { } func startServer(ctx context.Context) error { - args := []string{ - "--verbose", - "--acme-accept-terms", "true", - } + args := []string{"integration"} setEnvIfNotSet("ACME_API", "https://acme.mock.directory") setEnvIfNotSet("PAGES_DOMAIN", "localhost.mock.directory") setEnvIfNotSet("RAW_DOMAIN", "raw.localhost.mock.directory") @@ -44,10 +42,15 @@ func startServer(ctx context.Context) error { setEnvIfNotSet("HTTP_PORT", "8880") setEnvIfNotSet("ENABLE_HTTP_SERVER", "true") setEnvIfNotSet("DB_TYPE", "sqlite3") + setEnvIfNotSet("GITEA_ROOT", "https://codeberg.org") + setEnvIfNotSet("LOG_LEVEL", "trace") + setEnvIfNotSet("ENABLE_LFS_SUPPORT", "true") + setEnvIfNotSet("ENABLE_SYMLINK_SUPPORT", "true") + setEnvIfNotSet("ACME_ACCOUNT_CONFIG", "integration/acme-account.json") app := cli.NewApp() app.Name = "pages-server" - app.Action = cmd.Serve + app.Action = server.Serve app.Flags = cmd.ServerFlags go func() { diff --git a/main.go b/main.go index 6c1d0cc..87e21f3 100644 --- a/main.go +++ b/main.go @@ -1,29 +1,21 @@ package main import ( - "fmt" "os" _ "github.com/joho/godotenv/autoload" - "github.com/urfave/cli/v2" + "github.com/rs/zerolog/log" - "codeberg.org/codeberg/pages/cmd" - "codeberg.org/codeberg/pages/server/version" + "codeberg.org/codeberg/pages/cli" + "codeberg.org/codeberg/pages/server" ) func main() { - app := cli.NewApp() - app.Name = "pages-server" - app.Version = version.Version - app.Usage = "pages server" - app.Action = cmd.Serve - app.Flags = cmd.ServerFlags - app.Commands = []*cli.Command{ - cmd.Certs, - } + app := cli.CreatePagesApp() + app.Action = server.Serve if err := app.Run(os.Args); err != nil { - _, _ = fmt.Fprintln(os.Stderr, err) + log.Error().Err(err).Msg("A fatal error occurred") os.Exit(1) } } diff --git a/server/acme/client.go b/server/acme/client.go new file mode 100644 index 0000000..38e2785 --- /dev/null +++ b/server/acme/client.go @@ -0,0 +1,26 @@ +package acme + +import ( + "errors" + "fmt" + + "codeberg.org/codeberg/pages/config" + "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/certificates" +) + +var ErrAcmeMissConfig = errors.New("ACME client has wrong config") + +func CreateAcmeClient(cfg config.ACMEConfig, enableHTTPServer bool, challengeCache cache.ICache) (*certificates.AcmeClient, error) { + // check config + if (!cfg.AcceptTerms || cfg.DNSProvider == "") && cfg.APIEndpoint != "https://acme.mock.directory" { + return nil, fmt.Errorf("%w: you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER, unless $ACME_API is set to https://acme.mock.directory", ErrAcmeMissConfig) + } + if cfg.EAB_HMAC != "" && cfg.EAB_KID == "" { + return nil, fmt.Errorf("%w: ACME_EAB_HMAC also needs ACME_EAB_KID to be set", ErrAcmeMissConfig) + } else if cfg.EAB_HMAC == "" && cfg.EAB_KID != "" { + return nil, fmt.Errorf("%w: ACME_EAB_KID also needs ACME_EAB_HMAC to be set", ErrAcmeMissConfig) + } + + return certificates.NewAcmeClient(cfg, enableHTTPServer, challengeCache) +} diff --git a/server/cache/interface.go b/server/cache/interface.go index 2952b29..b3412cc 100644 --- a/server/cache/interface.go +++ b/server/cache/interface.go @@ -2,7 +2,8 @@ package cache import "time" -type SetGetKey interface { +// ICache is an interface that defines how the pages server interacts with the cache. +type ICache interface { Set(key string, value interface{}, ttl time.Duration) error Get(key string) (interface{}, bool) Remove(key string) diff --git a/server/cache/setup.go b/server/cache/memory.go similarity index 69% rename from server/cache/setup.go rename to server/cache/memory.go index a5928b0..093696f 100644 --- a/server/cache/setup.go +++ b/server/cache/memory.go @@ -2,6 +2,6 @@ package cache import "github.com/OrlovEvgeny/go-mcache" -func NewKeyValueCache() SetGetKey { +func NewInMemoryCache() ICache { return mcache.New() } diff --git a/server/certificates/acme_client.go b/server/certificates/acme_client.go index ba83e50..d53e854 100644 --- a/server/certificates/acme_client.go +++ b/server/certificates/acme_client.go @@ -10,6 +10,7 @@ import ( "github.com/reugn/equalizer" "github.com/rs/zerolog/log" + "codeberg.org/codeberg/pages/config" "codeberg.org/codeberg/pages/server/cache" ) @@ -28,8 +29,8 @@ type AcmeClient struct { acmeClientCertificateLimitPerUser map[string]*equalizer.TokenBucket } -func NewAcmeClient(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider string, acmeAcceptTerms, enableHTTPServer, acmeUseRateLimits bool, challengeCache cache.SetGetKey) (*AcmeClient, error) { - acmeConfig, err := setupAcmeConfig(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, acmeAcceptTerms) +func NewAcmeClient(cfg config.ACMEConfig, enableHTTPServer bool, challengeCache cache.ICache) (*AcmeClient, error) { + acmeConfig, err := setupAcmeConfig(cfg) if err != nil { return nil, err } @@ -54,7 +55,7 @@ func NewAcmeClient(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, if err != nil { log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only") } else { - if dnsProvider == "" { + if cfg.DNSProvider == "" { // using mock server, don't use wildcard certs err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache}) if err != nil { @@ -62,7 +63,7 @@ func NewAcmeClient(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, } } else { // use DNS-Challenge https://go-acme.github.io/lego/dns/ - provider, err := dns.NewDNSChallengeProviderByName(dnsProvider) + provider, err := dns.NewDNSChallengeProviderByName(cfg.DNSProvider) if err != nil { return nil, fmt.Errorf("can not create DNS Challenge provider: %w", err) } @@ -76,7 +77,7 @@ func NewAcmeClient(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, legoClient: acmeClient, dnsChallengerLegoClient: mainDomainAcmeClient, - acmeUseRateLimits: acmeUseRateLimits, + acmeUseRateLimits: cfg.UseRateLimits, obtainLocks: sync.Map{}, diff --git a/server/certificates/acme_config.go b/server/certificates/acme_config.go index 12ad7c6..2b5151d 100644 --- a/server/certificates/acme_config.go +++ b/server/certificates/acme_config.go @@ -8,6 +8,7 @@ import ( "fmt" "os" + "codeberg.org/codeberg/pages/config" "github.com/go-acme/lego/v4/certcrypto" "github.com/go-acme/lego/v4/lego" "github.com/go-acme/lego/v4/registration" @@ -16,21 +17,27 @@ import ( const challengePath = "/.well-known/acme-challenge/" -func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcceptTerms bool) (*lego.Config, error) { +func setupAcmeConfig(cfg config.ACMEConfig) (*lego.Config, error) { var myAcmeAccount AcmeAccount var myAcmeConfig *lego.Config - if account, err := os.ReadFile(configFile); err == nil { - log.Info().Msgf("found existing acme account config file '%s'", configFile) + if cfg.AccountConfigFile == "" { + return nil, fmt.Errorf("invalid acme config file: '%s'", cfg.AccountConfigFile) + } + + if account, err := os.ReadFile(cfg.AccountConfigFile); err == nil { + log.Info().Msgf("found existing acme account config file '%s'", cfg.AccountConfigFile) if err := json.Unmarshal(account, &myAcmeAccount); err != nil { return nil, err } + myAcmeAccount.Key, err = certcrypto.ParsePEMPrivateKey([]byte(myAcmeAccount.KeyPEM)) if err != nil { return nil, err } + myAcmeConfig = lego.NewConfig(&myAcmeAccount) - myAcmeConfig.CADirURL = acmeAPI + myAcmeConfig.CADirURL = cfg.APIEndpoint myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048 // Validate Config @@ -39,6 +46,7 @@ func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID stri log.Info().Err(err).Msg("config validation failed, you might just delete the config file and let it recreate") return nil, fmt.Errorf("acme config validation failed: %w", err) } + return myAcmeConfig, nil } else if !os.IsNotExist(err) { return nil, err @@ -51,20 +59,20 @@ func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID stri return nil, err } myAcmeAccount = AcmeAccount{ - Email: acmeMail, + Email: cfg.Email, Key: privateKey, KeyPEM: string(certcrypto.PEMEncode(privateKey)), } myAcmeConfig = lego.NewConfig(&myAcmeAccount) - myAcmeConfig.CADirURL = acmeAPI + myAcmeConfig.CADirURL = cfg.APIEndpoint myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048 tempClient, err := lego.NewClient(myAcmeConfig) if err != nil { log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only") } else { // accept terms & log in to EAB - if acmeEabKID == "" || acmeEabHmac == "" { - reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: acmeAcceptTerms}) + if cfg.EAB_KID == "" || cfg.EAB_HMAC == "" { + reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: cfg.AcceptTerms}) if err != nil { log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only") } else { @@ -72,9 +80,9 @@ func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID stri } } else { reg, err := tempClient.Registration.RegisterWithExternalAccountBinding(registration.RegisterEABOptions{ - TermsOfServiceAgreed: acmeAcceptTerms, - Kid: acmeEabKID, - HmacEncoded: acmeEabHmac, + TermsOfServiceAgreed: cfg.AcceptTerms, + Kid: cfg.EAB_KID, + HmacEncoded: cfg.EAB_HMAC, }) if err != nil { log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only") @@ -89,8 +97,8 @@ func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID stri log.Error().Err(err).Msg("json.Marshalfailed, waiting for manual restart to avoid rate limits") select {} } - log.Info().Msgf("new acme account created. write to config file '%s'", configFile) - err = os.WriteFile(configFile, acmeAccountJSON, 0o600) + log.Info().Msgf("new acme account created. write to config file '%s'", cfg.AccountConfigFile) + err = os.WriteFile(cfg.AccountConfigFile, acmeAccountJSON, 0o600) if err != nil { log.Error().Err(err).Msg("os.WriteFile failed, waiting for manual restart to avoid rate limits") select {} diff --git a/server/certificates/cached_challengers.go b/server/certificates/cached_challengers.go index bc9ea67..39439fb 100644 --- a/server/certificates/cached_challengers.go +++ b/server/certificates/cached_challengers.go @@ -15,7 +15,7 @@ import ( ) type AcmeTLSChallengeProvider struct { - challengeCache cache.SetGetKey + challengeCache cache.ICache } // make sure AcmeTLSChallengeProvider match Provider interface @@ -31,7 +31,7 @@ func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error { } type AcmeHTTPChallengeProvider struct { - challengeCache cache.SetGetKey + challengeCache cache.ICache } // make sure AcmeHTTPChallengeProvider match Provider interface @@ -46,7 +46,7 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error { return nil } -func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey, sslPort uint) http.HandlerFunc { +func SetupHTTPACMEChallengeServer(challengeCache cache.ICache, sslPort uint) http.HandlerFunc { // handle custom-ssl-ports to be added on https redirects portPart := "" if sslPort != 443 { diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index 4fb99fa..b638755 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -31,7 +31,7 @@ func TLSConfig(mainDomainSuffix string, giteaClient *gitea.Client, acmeClient *AcmeClient, firstDefaultBranch string, - keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey, + keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.ICache, certDB database.CertDB, ) *tls.Config { return &tls.Config{ diff --git a/server/dns/dns.go b/server/dns/dns.go index c11b278..970f0c0 100644 --- a/server/dns/dns.go +++ b/server/dns/dns.go @@ -15,7 +15,7 @@ var defaultPagesRepo = "pages" // GetTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix. // If everything is fine, it returns the target data. -func GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) { +func GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch string, dnsLookupCache cache.ICache) (targetOwner, targetRepo, targetBranch string) { // Get CNAME or TXT var cname string var err error diff --git a/server/gitea/cache.go b/server/gitea/cache.go index af61edf..267c3d8 100644 --- a/server/gitea/cache.go +++ b/server/gitea/cache.go @@ -74,7 +74,7 @@ type writeCacheReader struct { buffer *bytes.Buffer rileResponse *FileResponse cacheKey string - cache cache.SetGetKey + cache cache.ICache hasError bool } @@ -99,7 +99,7 @@ func (t *writeCacheReader) Close() error { return t.originalReader.Close() } -func (f FileResponse) CreateCacheReader(r io.ReadCloser, cache cache.SetGetKey, cacheKey string) io.ReadCloser { +func (f FileResponse) CreateCacheReader(r io.ReadCloser, cache cache.ICache, cacheKey string) io.ReadCloser { if r == nil || cache == nil || cacheKey == "" { log.Error().Msg("could not create CacheReader") return nil diff --git a/server/gitea/client.go b/server/gitea/client.go index f3bda54..42cf065 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/sdk/gitea" "github.com/rs/zerolog/log" + "codeberg.org/codeberg/pages/config" "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/version" ) @@ -44,7 +45,7 @@ const ( type Client struct { sdkClient *gitea.Client - responseCache cache.SetGetKey + responseCache cache.ICache giteaRoot string @@ -55,24 +56,21 @@ type Client struct { defaultMimeType string } -func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, followSymlinks, supportLFS bool) (*Client, error) { - rootURL, err := url.Parse(giteaRoot) +func NewClient(cfg config.GiteaConfig, respCache cache.ICache) (*Client, error) { + rootURL, err := url.Parse(cfg.Root) if err != nil { return nil, err } - giteaRoot = strings.Trim(rootURL.String(), "/") + giteaRoot := strings.Trim(rootURL.String(), "/") stdClient := http.Client{Timeout: 10 * time.Second} - // TODO: pass down - var ( - forbiddenMimeTypes map[string]bool - defaultMimeType string - ) - - if forbiddenMimeTypes == nil { - forbiddenMimeTypes = make(map[string]bool) + forbiddenMimeTypes := make(map[string]bool, len(cfg.ForbiddenMimeTypes)) + for _, mimeType := range cfg.ForbiddenMimeTypes { + forbiddenMimeTypes[mimeType] = true } + + defaultMimeType := cfg.DefaultMimeType if defaultMimeType == "" { defaultMimeType = "application/octet-stream" } @@ -80,7 +78,7 @@ func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, follo sdk, err := gitea.NewClient( giteaRoot, gitea.SetHTTPClient(&stdClient), - gitea.SetToken(giteaAPIToken), + gitea.SetToken(cfg.Token), gitea.SetUserAgent("pages-server/"+version.Version), ) @@ -90,8 +88,8 @@ func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, follo giteaRoot: giteaRoot, - followSymlinks: followSymlinks, - supportLFS: supportLFS, + followSymlinks: cfg.FollowSymlinks, + supportLFS: cfg.LFSEnabled, forbiddenMimeTypes: forbiddenMimeTypes, defaultMimeType: defaultMimeType, diff --git a/server/handler/handler.go b/server/handler/handler.go index 7da5d39..96788e4 100644 --- a/server/handler/handler.go +++ b/server/handler/handler.go @@ -6,6 +6,7 @@ import ( "github.com/rs/zerolog/log" + "codeberg.org/codeberg/pages/config" "codeberg.org/codeberg/pages/html" "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/context" @@ -19,11 +20,10 @@ const ( ) // Handler handles a single HTTP request to the web server. -func Handler(mainDomainSuffix, rawDomain string, +func Handler( + cfg config.ServerConfig, giteaClient *gitea.Client, - blacklistedPaths, allowedCorsDomains []string, - defaultPagesBranches []string, - dnsLookupCache, canonicalDomainCache, redirectsCache cache.SetGetKey, + dnsLookupCache, canonicalDomainCache, redirectsCache cache.ICache, ) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { log := log.With().Strs("Handler", []string{req.Host, req.RequestURI}).Logger() @@ -39,8 +39,8 @@ func Handler(mainDomainSuffix, rawDomain string, trimmedHost := ctx.TrimHostPort() - // Add HSTS for RawDomain and MainDomainSuffix - if hsts := getHSTSHeader(trimmedHost, mainDomainSuffix, rawDomain); hsts != "" { + // Add HSTS for RawDomain and MainDomain + if hsts := getHSTSHeader(trimmedHost, cfg.MainDomain, cfg.RawDomain); hsts != "" { ctx.RespWriter.Header().Set("Strict-Transport-Security", hsts) } @@ -62,7 +62,7 @@ func Handler(mainDomainSuffix, rawDomain string, } // Block blacklisted paths (like ACME challenges) - for _, blacklistedPath := range blacklistedPaths { + for _, blacklistedPath := range cfg.BlacklistedPaths { if strings.HasPrefix(ctx.Path(), blacklistedPath) { html.ReturnErrorPage(ctx, "requested path is blacklisted", http.StatusForbidden) return @@ -71,7 +71,7 @@ func Handler(mainDomainSuffix, rawDomain string, // Allow CORS for specified domains allowCors := false - for _, allowedCorsDomain := range allowedCorsDomains { + for _, allowedCorsDomain := range cfg.AllowedCorsDomains { if strings.EqualFold(trimmedHost, allowedCorsDomain) { allowCors = true break @@ -85,28 +85,28 @@ func Handler(mainDomainSuffix, rawDomain string, // Prepare request information to Gitea pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/") - if rawDomain != "" && strings.EqualFold(trimmedHost, rawDomain) { + if cfg.RawDomain != "" && strings.EqualFold(trimmedHost, cfg.RawDomain) { log.Debug().Msg("raw domain request detected") handleRaw(log, ctx, giteaClient, - mainDomainSuffix, + cfg.MainDomain, trimmedHost, pathElements, canonicalDomainCache, redirectsCache) - } else if strings.HasSuffix(trimmedHost, mainDomainSuffix) { + } else if strings.HasSuffix(trimmedHost, cfg.MainDomain) { log.Debug().Msg("subdomain request detected") handleSubDomain(log, ctx, giteaClient, - mainDomainSuffix, - defaultPagesBranches, + cfg.MainDomain, + cfg.PagesBranches, trimmedHost, pathElements, canonicalDomainCache, redirectsCache) } else { log.Debug().Msg("custom domain request detected") handleCustomDomain(log, ctx, giteaClient, - mainDomainSuffix, + cfg.MainDomain, trimmedHost, pathElements, - defaultPagesBranches[0], + cfg.PagesBranches[0], dnsLookupCache, canonicalDomainCache, redirectsCache) } } diff --git a/server/handler/handler_custom_domain.go b/server/handler/handler_custom_domain.go index 8742be4..82953f9 100644 --- a/server/handler/handler_custom_domain.go +++ b/server/handler/handler_custom_domain.go @@ -19,7 +19,7 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g trimmedHost string, pathElements []string, firstDefaultBranch string, - dnsLookupCache, canonicalDomainCache, redirectsCache cache.SetGetKey, + dnsLookupCache, canonicalDomainCache, redirectsCache cache.ICache, ) { // Serve pages from custom domains targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, mainDomainSuffix, firstDefaultBranch, dnsLookupCache) diff --git a/server/handler/handler_raw_domain.go b/server/handler/handler_raw_domain.go index caa8209..f48e8e4 100644 --- a/server/handler/handler_raw_domain.go +++ b/server/handler/handler_raw_domain.go @@ -19,7 +19,7 @@ func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Clie mainDomainSuffix string, trimmedHost string, pathElements []string, - canonicalDomainCache, redirectsCache cache.SetGetKey, + canonicalDomainCache, redirectsCache cache.ICache, ) { // Serve raw content from RawDomain log.Debug().Msg("raw domain") diff --git a/server/handler/handler_sub_domain.go b/server/handler/handler_sub_domain.go index 6c14393..806fe7f 100644 --- a/server/handler/handler_sub_domain.go +++ b/server/handler/handler_sub_domain.go @@ -21,7 +21,7 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite defaultPagesBranches []string, trimmedHost string, pathElements []string, - canonicalDomainCache, redirectsCache cache.SetGetKey, + canonicalDomainCache, redirectsCache cache.ICache, ) { // Serve pages from subdomains of MainDomainSuffix log.Debug().Msg("main domain suffix") diff --git a/server/handler/handler_test.go b/server/handler/handler_test.go index d04ebda..4cb859a 100644 --- a/server/handler/handler_test.go +++ b/server/handler/handler_test.go @@ -6,23 +6,30 @@ import ( "testing" "time" + "codeberg.org/codeberg/pages/config" "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/gitea" "github.com/rs/zerolog/log" ) func TestHandlerPerformance(t *testing.T) { - giteaClient, _ := gitea.NewClient("https://codeberg.org", "", cache.NewKeyValueCache(), false, false) - testHandler := Handler( - "codeberg.page", "raw.codeberg.org", - giteaClient, - []string{"/.well-known/acme-challenge/"}, - []string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"}, - []string{"pages"}, - cache.NewKeyValueCache(), - cache.NewKeyValueCache(), - cache.NewKeyValueCache(), - ) + cfg := config.GiteaConfig{ + Root: "https://codeberg.org", + Token: "", + LFSEnabled: false, + FollowSymlinks: false, + } + giteaClient, _ := gitea.NewClient(cfg, cache.NewInMemoryCache()) + serverCfg := config.ServerConfig{ + MainDomain: "codeberg.page", + RawDomain: "raw.codeberg.page", + BlacklistedPaths: []string{ + "/.well-known/acme-challenge/", + }, + AllowedCorsDomains: []string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"}, + PagesBranches: []string{"pages"}, + } + testHandler := Handler(serverCfg, giteaClient, cache.NewInMemoryCache(), cache.NewInMemoryCache(), cache.NewInMemoryCache()) testCase := func(uri string, status int) { t.Run(uri, func(t *testing.T) { diff --git a/server/handler/try.go b/server/handler/try.go index bb79dce..145b1a9 100644 --- a/server/handler/try.go +++ b/server/handler/try.go @@ -17,8 +17,8 @@ import ( func tryUpstream(ctx *context.Context, giteaClient *gitea.Client, mainDomainSuffix, trimmedHost string, options *upstream.Options, - canonicalDomainCache cache.SetGetKey, - redirectsCache cache.SetGetKey, + canonicalDomainCache cache.ICache, + redirectsCache cache.ICache, ) { // check if a canonical domain exists on a request on MainDomain if strings.HasSuffix(trimmedHost, mainDomainSuffix) && !options.ServeRaw { diff --git a/server/startup.go b/server/startup.go new file mode 100644 index 0000000..ffdabb7 --- /dev/null +++ b/server/startup.go @@ -0,0 +1,141 @@ +package server + +import ( + "context" + "crypto/tls" + "encoding/json" + "fmt" + "net" + "net/http" + "os" + "strings" + "time" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/urfave/cli/v2" + + cmd "codeberg.org/codeberg/pages/cli" + "codeberg.org/codeberg/pages/config" + "codeberg.org/codeberg/pages/server/acme" + "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/certificates" + "codeberg.org/codeberg/pages/server/gitea" + "codeberg.org/codeberg/pages/server/handler" +) + +// Serve sets up and starts the web server. +func Serve(ctx *cli.Context) error { + // initialize logger with Trace, overridden later with actual level + log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger().Level(zerolog.TraceLevel) + + cfg, err := config.ReadConfig(ctx) + if err != nil { + log.Error().Err(err).Msg("could not read config") + } + + config.MergeConfig(ctx, cfg) + + // Initialize the logger. + logLevel, err := zerolog.ParseLevel(cfg.LogLevel) + if err != nil { + return err + } + log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger().Level(logLevel) + + foo, _ := json.Marshal(cfg) + log.Trace().RawJSON("config", foo).Msg("starting server with config") + + listeningSSLAddress := fmt.Sprintf("%s:%d", cfg.Server.Host, cfg.Server.Port) + listeningHTTPAddress := fmt.Sprintf("%s:%d", cfg.Server.Host, cfg.Server.HttpPort) + + if cfg.Server.RawDomain != "" { + cfg.Server.AllowedCorsDomains = append(cfg.Server.AllowedCorsDomains, cfg.Server.RawDomain) + } + + // Make sure MainDomain has a leading dot + if !strings.HasPrefix(cfg.Server.MainDomain, ".") { + // TODO make this better + cfg.Server.MainDomain = "." + cfg.Server.MainDomain + } + + if len(cfg.Server.PagesBranches) == 0 { + return fmt.Errorf("no default branches set (PAGES_BRANCHES)") + } + + // Init ssl cert database + certDB, closeFn, err := cmd.OpenCertDB(ctx) + if err != nil { + return err + } + defer closeFn() + + keyCache := cache.NewInMemoryCache() + challengeCache := cache.NewInMemoryCache() + // canonicalDomainCache stores canonical domains + canonicalDomainCache := cache.NewInMemoryCache() + // dnsLookupCache stores DNS lookups for custom domains + dnsLookupCache := cache.NewInMemoryCache() + // redirectsCache stores redirects in _redirects files + redirectsCache := cache.NewInMemoryCache() + // clientResponseCache stores responses from the Gitea server + clientResponseCache := cache.NewInMemoryCache() + + giteaClient, err := gitea.NewClient(cfg.Gitea, clientResponseCache) + if err != nil { + return fmt.Errorf("could not create new gitea client: %v", err) + } + + acmeClient, err := acme.CreateAcmeClient(cfg.ACME, cfg.Server.HttpServerEnabled, challengeCache) + if err != nil { + return err + } + + if err := certificates.SetupMainDomainCertificates(cfg.Server.MainDomain, acmeClient, certDB); err != nil { + return err + } + + // Create listener for SSL connections + log.Info().Msgf("Create TCP listener for SSL on %s", listeningSSLAddress) + listener, err := net.Listen("tcp", listeningSSLAddress) + if err != nil { + return fmt.Errorf("couldn't create listener: %v", err) + } + + // Setup listener for SSL connections + listener = tls.NewListener(listener, certificates.TLSConfig( + cfg.Server.MainDomain, + giteaClient, + acmeClient, + cfg.Server.PagesBranches[0], + keyCache, challengeCache, dnsLookupCache, canonicalDomainCache, + certDB, + )) + + interval := 12 * time.Hour + certMaintainCtx, cancelCertMaintain := context.WithCancel(context.Background()) + defer cancelCertMaintain() + go certificates.MaintainCertDB(certMaintainCtx, interval, acmeClient, cfg.Server.MainDomain, certDB) + + if cfg.Server.HttpServerEnabled { + // Create handler for http->https redirect and http acme challenges + httpHandler := certificates.SetupHTTPACMEChallengeServer(challengeCache, uint(cfg.Server.Port)) + + // Create listener for http and start listening + go func() { + log.Info().Msgf("Start HTTP server listening on %s", listeningHTTPAddress) + err := http.ListenAndServe(listeningHTTPAddress, httpHandler) + if err != nil { + log.Error().Err(err).Msg("Couldn't start HTTP server") + } + }() + } + + // Create ssl handler based on settings + sslHandler := handler.Handler(cfg.Server, giteaClient, dnsLookupCache, canonicalDomainCache, redirectsCache) + + // Start the ssl listener + log.Info().Msgf("Start SSL server using TCP listener on %s", listener.Addr()) + + return http.Serve(listener, sslHandler) +} diff --git a/server/upstream/domains.go b/server/upstream/domains.go index 230f268..d53d586 100644 --- a/server/upstream/domains.go +++ b/server/upstream/domains.go @@ -17,7 +17,7 @@ var canonicalDomainCacheTimeout = 15 * time.Minute const canonicalDomainConfig = ".domains" // CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file). -func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (domain string, valid bool) { +func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.ICache) (domain string, valid bool) { // Check if this request is cached. if cachedValue, ok := canonicalDomainCache.Get(o.TargetOwner + "/" + o.TargetRepo + "/" + o.TargetBranch); ok { domains := cachedValue.([]string) diff --git a/server/upstream/redirects.go b/server/upstream/redirects.go index ab6c971..dd36a84 100644 --- a/server/upstream/redirects.go +++ b/server/upstream/redirects.go @@ -23,7 +23,7 @@ var redirectsCacheTimeout = 10 * time.Minute const redirectsConfig = "_redirects" // getRedirects returns redirects specified in the _redirects file. -func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.SetGetKey) []Redirect { +func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.ICache) []Redirect { var redirects []Redirect cacheKey := o.TargetOwner + "/" + o.TargetRepo + "/" + o.TargetBranch @@ -63,7 +63,7 @@ func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.S return redirects } -func (o *Options) matchRedirects(ctx *context.Context, giteaClient *gitea.Client, redirects []Redirect, redirectsCache cache.SetGetKey) (final bool) { +func (o *Options) matchRedirects(ctx *context.Context, giteaClient *gitea.Client, redirects []Redirect, redirectsCache cache.ICache) (final bool) { if len(redirects) > 0 { for _, redirect := range redirects { reqUrl := ctx.Req.RequestURI diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index 4a85503..07b6ad2 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -53,7 +53,7 @@ type Options struct { } // Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context. -func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redirectsCache cache.SetGetKey) bool { +func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redirectsCache cache.ICache) bool { log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger() if o.TargetOwner == "" || o.TargetRepo == "" { From a6e9510c078758794da5f6a1983b595bfc415de9 Mon Sep 17 00:00:00 2001 From: Hoernschen Date: Mon, 26 Feb 2024 22:21:42 +0000 Subject: [PATCH 13/19] FIX blank internal pages (#164) (#292) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hello 👋 since it affected my deployment of the pages server I started to look into the problem of the blank pages and think I found a solution for it: 1. There is no check if the file response is empty, neither in cache retrieval nor in writing of a cache. Also the provided method for checking for empty responses had a bug. 2. I identified the redirect response to be the issue here. There is a cache write with the full cache key (e. g. rawContent/user/repo|branch|route/index.html) happening in the handling of the redirect response. But the written body here is empty. In the triggered request from the redirect response the server then finds a cache item to the key and serves the empty body. A quick fix is the check for empty file responses mentioned in 1. 3. The decision to redirect the user comes quite far down in the upstream function. Before that happens a lot of stuff that may not be important since after the redirect response comes a new request anyway. Also, I suspect that this causes the caching problem because there is a request to the forge server and its error handling with some recursions happening before. I propose to move two of the redirects before "Preparing" 4. The recursion in the upstream function makes it difficult to understand what is actually happening. I added some more logging to have an easier time with that. 5. I changed the default behaviour to append a trailing slash to the path to true. In my tested scenarios it happened anyway. This way there is no recursion happening before the redirect. I am not developing in go frequently and rarely contribute to open source -> so feedback of all kind is appreciated closes #164 Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/292 Reviewed-by: 6543 <6543@obermui.de> Reviewed-by: crapStone Co-authored-by: Hoernschen Co-committed-by: Hoernschen --- server/gitea/cache.go | 25 +++++++++++++++++-------- server/gitea/client.go | 19 +++++++++++-------- server/handler/handler.go | 1 + server/upstream/upstream.go | 12 ++++++++++++ 4 files changed, 41 insertions(+), 16 deletions(-) diff --git a/server/gitea/cache.go b/server/gitea/cache.go index 267c3d8..4ecc5e0 100644 --- a/server/gitea/cache.go +++ b/server/gitea/cache.go @@ -39,7 +39,7 @@ type FileResponse struct { } func (f FileResponse) IsEmpty() bool { - return len(f.Body) != 0 + return len(f.Body) == 0 } func (f FileResponse) createHttpResponse(cacheKey string) (header http.Header, statusCode int) { @@ -72,13 +72,14 @@ type BranchTimestamp struct { type writeCacheReader struct { originalReader io.ReadCloser buffer *bytes.Buffer - rileResponse *FileResponse + fileResponse *FileResponse cacheKey string cache cache.ICache hasError bool } func (t *writeCacheReader) Read(p []byte) (n int, err error) { + log.Trace().Msgf("[cache] read %q", t.cacheKey) n, err = t.originalReader.Read(p) if err != nil && err != io.EOF { log.Trace().Err(err).Msgf("[cache] original reader for %q has returned an error", t.cacheKey) @@ -90,12 +91,20 @@ func (t *writeCacheReader) Read(p []byte) (n int, err error) { } func (t *writeCacheReader) Close() error { - if !t.hasError { - fc := *t.rileResponse - fc.Body = t.buffer.Bytes() - _ = t.cache.Set(t.cacheKey, fc, fileCacheTimeout) + doWrite := !t.hasError + fc := *t.fileResponse + fc.Body = t.buffer.Bytes() + if fc.IsEmpty() { + log.Trace().Msg("[cache] file response is empty") + doWrite = false } - log.Trace().Msgf("cacheReader for %q saved=%t closed", t.cacheKey, !t.hasError) + if doWrite { + err := t.cache.Set(t.cacheKey, fc, fileCacheTimeout) + if err != nil { + log.Trace().Err(err).Msgf("[cache] writer for %q has returned an error", t.cacheKey) + } + } + log.Trace().Msgf("cacheReader for %q saved=%t closed", t.cacheKey, doWrite) return t.originalReader.Close() } @@ -108,7 +117,7 @@ func (f FileResponse) CreateCacheReader(r io.ReadCloser, cache cache.ICache, cac return &writeCacheReader{ originalReader: r, buffer: bytes.NewBuffer(make([]byte, 0)), - rileResponse: &f, + fileResponse: &f, cache: cache, cacheKey: cacheKey, } diff --git a/server/gitea/client.go b/server/gitea/client.go index 42cf065..4f7eaa6 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -112,26 +112,27 @@ func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource str func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource string) (io.ReadCloser, http.Header, int, error) { cacheKey := fmt.Sprintf("%s/%s/%s|%s|%s", rawContentCacheKeyPrefix, targetOwner, targetRepo, ref, resource) log := log.With().Str("cache_key", cacheKey).Logger() - + log.Trace().Msg("try file in cache") // handle if cache entry exist if cache, ok := client.responseCache.Get(cacheKey); ok { cache := cache.(FileResponse) cachedHeader, cachedStatusCode := cache.createHttpResponse(cacheKey) // TODO: check against some timestamp mismatch?!? if cache.Exists { + log.Debug().Msg("[cache] exists") if cache.IsSymlink { linkDest := string(cache.Body) log.Debug().Msgf("[cache] follow symlink from %q to %q", resource, linkDest) return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest) - } else { - log.Debug().Msg("[cache] return bytes") + } else if !cache.IsEmpty() { + log.Debug().Msgf("[cache] return %d bytes", len(cache.Body)) return io.NopCloser(bytes.NewReader(cache.Body)), cachedHeader, cachedStatusCode, nil + } else if cache.IsEmpty() { + log.Debug().Msg("[cache] is empty") } - } else { - return nil, cachedHeader, cachedStatusCode, ErrorNotFound } } - + log.Trace().Msg("file not in cache") // not in cache, open reader via gitea api reader, resp, err := client.sdkClient.GetFileReader(targetOwner, targetRepo, ref, resource, client.supportLFS) if resp != nil { @@ -155,12 +156,14 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str linkDest = path.Join(path.Dir(resource), linkDest) // we store symlink not content to reduce duplicates in cache - if err := client.responseCache.Set(cacheKey, FileResponse{ + fileResponse := FileResponse{ Exists: true, IsSymlink: true, Body: []byte(linkDest), ETag: resp.Header.Get(ETagHeader), - }, fileCacheTimeout); err != nil { + } + log.Trace().Msgf("file response has %d bytes", len(fileResponse.Body)) + if err := client.responseCache.Set(cacheKey, fileResponse, fileCacheTimeout); err != nil { log.Error().Err(err).Msg("[cache] error on cache write") } diff --git a/server/handler/handler.go b/server/handler/handler.go index 96788e4..ffc3400 100644 --- a/server/handler/handler.go +++ b/server/handler/handler.go @@ -26,6 +26,7 @@ func Handler( dnsLookupCache, canonicalDomainCache, redirectsCache cache.ICache, ) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { + log.Debug().Msg("\n----------------------------------------------------------") log := log.With().Strs("Handler", []string{req.Host, req.RequestURI}).Logger() ctx := context.New(w, req) diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index 07b6ad2..d9c131e 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -56,6 +56,8 @@ type Options struct { func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redirectsCache cache.ICache) bool { log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger() + log.Debug().Msg("Start") + if o.TargetOwner == "" || o.TargetRepo == "" { html.ReturnErrorPage(ctx, "forge client: either repo owner or name info is missing", http.StatusBadRequest) return true @@ -104,13 +106,16 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi // Handle not found error if err != nil && errors.Is(err, gitea.ErrorNotFound) { + log.Debug().Msg("Handling not found error") // Get and match redirects redirects := o.getRedirects(giteaClient, redirectsCache) if o.matchRedirects(ctx, giteaClient, redirects, redirectsCache) { + log.Trace().Msg("redirect") return true } if o.TryIndexPages { + log.Trace().Msg("try index page") // copy the o struct & try if an index page exists optionsForIndexPages := *o optionsForIndexPages.TryIndexPages = false @@ -121,6 +126,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi return true } } + log.Trace().Msg("try html file with path name") // compatibility fix for GitHub Pages (/example → /example.html) optionsForIndexPages.appendTrailingSlash = false optionsForIndexPages.redirectIfExists = strings.TrimSuffix(ctx.Path(), "/") + ".html" @@ -130,8 +136,11 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi } } + log.Trace().Msg("not found") + ctx.StatusCode = http.StatusNotFound if o.TryIndexPages { + log.Trace().Msg("try not found page") // copy the o struct & try if a not found page exists optionsForNotFoundPages := *o optionsForNotFoundPages.TryIndexPages = false @@ -142,6 +151,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi return true } } + log.Trace().Msg("not found page missing") } return false @@ -173,10 +183,12 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi // Append trailing slash if missing (for index files), and redirect to fix filenames in general // o.appendTrailingSlash is only true when looking for index pages if o.appendTrailingSlash && !strings.HasSuffix(ctx.Path(), "/") { + log.Trace().Msg("append trailing slash and redirect") ctx.Redirect(ctx.Path()+"/", http.StatusTemporaryRedirect) return true } if strings.HasSuffix(ctx.Path(), "/index.html") && !o.ServeRaw { + log.Trace().Msg("remove index.html from path and redirect") ctx.Redirect(strings.TrimSuffix(ctx.Path(), "index.html"), http.StatusTemporaryRedirect) return true } From dd6d8bd60fe0e318602042d1ae8129654f4f5186 Mon Sep 17 00:00:00 2001 From: caelandb Date: Tue, 12 Mar 2024 21:50:17 +0000 Subject: [PATCH 14/19] fixed one grammar error. (#297) Just a small grammar change in the README. Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/297 Reviewed-by: crapStone Co-authored-by: caelandb Co-committed-by: caelandb --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fb2a4b9..7193691 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ Thank you very much. Make sure you have [golang](https://go.dev) v1.21 or newer and [just](https://just.systems/man/en/) installed. run `just dev` -now this pages should work: +now these pages should work: - - From 03881382a4c286e3a7dc351f095f9147e89c2d6d Mon Sep 17 00:00:00 2001 From: Jean-Marie 'Histausse' Mineau Date: Thu, 18 Apr 2024 17:05:20 +0000 Subject: [PATCH 15/19] Add option to disable DNS ACME provider (#290) This PR add the `$NO_DNS_01` option (disabled by default) that removes the DNS ACME provider, and replaces the wildcard certificate by individual certificates obtained using the TLS ACME provider. This option allows an instance to work without having to manage access tokens for the DNS provider. On the flip side, this means that a certificate can be requested for each subdomains. To limit the risk of DOS, the existence of the user/org corresponding to a subdomain is checked before requesting a cert, however, this limitation is not enough for an forge with a high number of users/orgs. Co-authored-by: 6543 <6543@obermui.de> Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/290 Reviewed-by: Moritz Marquardt Co-authored-by: Jean-Marie 'Histausse' Mineau Co-committed-by: Jean-Marie 'Histausse' Mineau --- README.md | 1 + cli/flags.go | 5 +++++ config/config.go | 1 + config/setup.go | 3 +++ config/setup_test.go | 7 ++++++ server/acme/client.go | 4 ++-- server/certificates/acme_client.go | 7 ++---- server/certificates/certificates.go | 30 ++++++++++++++++++++------ server/database/xorm.go | 13 ------------ server/gitea/cache.go | 3 +++ server/gitea/client.go | 33 +++++++++++++++++++++++++++++ server/startup.go | 2 ++ 12 files changed, 83 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 7193691..f47a196 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,7 @@ and especially have a look at [this section of the haproxy.cfg](https://codeberg - `ENABLE_HTTP_SERVER` (default: false): Set this to true to enable the HTTP-01 challenge and redirect all other HTTP requests to HTTPS. Currently only works with port 80. - `DNS_PROVIDER` (default: use self-signed certificate): Code of the ACME DNS provider for the main domain wildcard. See for available values & additional environment variables. +- `NO_DNS_01` (default: `false`): Disable the use of ACME DNS. This means that the wildcard certificate is self-signed and all domains and subdomains will have a distinct certificate. Because this may lead to a rate limit from the ACME provider, this option is not recommended for Gitea/Forgejo instances with open registrations or a great number of users/orgs. - `LOG_LEVEL` (default: warn): Set this to specify the level of logging. ## Contributing to the development diff --git a/cli/flags.go b/cli/flags.go index 097cf4f..52e1c1c 100644 --- a/cli/flags.go +++ b/cli/flags.go @@ -178,6 +178,11 @@ var ( Usage: "Use DNS-Challenge for main domain. Read more at: https://go-acme.github.io/lego/dns/", EnvVars: []string{"DNS_PROVIDER"}, }, + &cli.BoolFlag{ + Name: "no-dns-01", + Usage: "Always use individual certificates instead of a DNS-01 wild card certificate", + EnvVars: []string{"NO_DNS_01"}, + }, &cli.StringFlag{ Name: "acme-account-config", Usage: "json file of acme account", diff --git a/config/config.go b/config/config.go index 6cb972b..0146e0f 100644 --- a/config/config.go +++ b/config/config.go @@ -42,5 +42,6 @@ type ACMEConfig struct { EAB_HMAC string EAB_KID string DNSProvider string + NoDNS01 bool `default:"false"` AccountConfigFile string `default:"acme-account.json"` } diff --git a/config/setup.go b/config/setup.go index e774084..6a2aa62 100644 --- a/config/setup.go +++ b/config/setup.go @@ -141,6 +141,9 @@ func mergeACMEConfig(ctx *cli.Context, config *ACMEConfig) { if ctx.IsSet("dns-provider") { config.DNSProvider = ctx.String("dns-provider") } + if ctx.IsSet("no-dns-01") { + config.NoDNS01 = ctx.Bool("no-dns-01") + } if ctx.IsSet("acme-account-config") { config.AccountConfigFile = ctx.String("acme-account-config") } diff --git a/config/setup_test.go b/config/setup_test.go index a863e2f..1a32740 100644 --- a/config/setup_test.go +++ b/config/setup_test.go @@ -166,6 +166,7 @@ func TestMergeConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T EAB_HMAC: "original", EAB_KID: "original", DNSProvider: "original", + NoDNS01: false, AccountConfigFile: "original", }, } @@ -205,6 +206,7 @@ func TestMergeConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T EAB_HMAC: "changed", EAB_KID: "changed", DNSProvider: "changed", + NoDNS01: true, AccountConfigFile: "changed", }, } @@ -243,6 +245,7 @@ func TestMergeConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T "--acme-eab-hmac", "changed", "--acme-eab-kid", "changed", "--dns-provider", "changed", + "--no-dns-01", "--acme-account-config", "changed", }, ) @@ -517,6 +520,7 @@ func TestMergeACMEConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testi EAB_HMAC: "original", EAB_KID: "original", DNSProvider: "original", + NoDNS01: false, AccountConfigFile: "original", } @@ -530,6 +534,7 @@ func TestMergeACMEConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testi EAB_HMAC: "changed", EAB_KID: "changed", DNSProvider: "changed", + NoDNS01: true, AccountConfigFile: "changed", } @@ -545,6 +550,7 @@ func TestMergeACMEConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testi "--acme-eab-hmac", "changed", "--acme-eab-kid", "changed", "--dns-provider", "changed", + "--no-dns-01", "--acme-account-config", "changed", }, ) @@ -563,6 +569,7 @@ func TestMergeACMEConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgExi {args: []string{"--acme-eab-hmac", "changed"}, callback: func(gc *ACMEConfig) { gc.EAB_HMAC = "changed" }}, {args: []string{"--acme-eab-kid", "changed"}, callback: func(gc *ACMEConfig) { gc.EAB_KID = "changed" }}, {args: []string{"--dns-provider", "changed"}, callback: func(gc *ACMEConfig) { gc.DNSProvider = "changed" }}, + {args: []string{"--no-dns-01"}, callback: func(gc *ACMEConfig) { gc.NoDNS01 = true }}, {args: []string{"--acme-account-config", "changed"}, callback: func(gc *ACMEConfig) { gc.AccountConfigFile = "changed" }}, } diff --git a/server/acme/client.go b/server/acme/client.go index 38e2785..d5c83d0 100644 --- a/server/acme/client.go +++ b/server/acme/client.go @@ -13,8 +13,8 @@ var ErrAcmeMissConfig = errors.New("ACME client has wrong config") func CreateAcmeClient(cfg config.ACMEConfig, enableHTTPServer bool, challengeCache cache.ICache) (*certificates.AcmeClient, error) { // check config - if (!cfg.AcceptTerms || cfg.DNSProvider == "") && cfg.APIEndpoint != "https://acme.mock.directory" { - return nil, fmt.Errorf("%w: you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER, unless $ACME_API is set to https://acme.mock.directory", ErrAcmeMissConfig) + if (!cfg.AcceptTerms || (cfg.DNSProvider == "" && !cfg.NoDNS01)) && cfg.APIEndpoint != "https://acme.mock.directory" { + return nil, fmt.Errorf("%w: you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER or $NO_DNS_01, unless $ACME_API is set to https://acme.mock.directory", ErrAcmeMissConfig) } if cfg.EAB_HMAC != "" && cfg.EAB_KID == "" { return nil, fmt.Errorf("%w: ACME_EAB_HMAC also needs ACME_EAB_KID to be set", ErrAcmeMissConfig) diff --git a/server/certificates/acme_client.go b/server/certificates/acme_client.go index d53e854..f42fd8f 100644 --- a/server/certificates/acme_client.go +++ b/server/certificates/acme_client.go @@ -56,11 +56,8 @@ func NewAcmeClient(cfg config.ACMEConfig, enableHTTPServer bool, challengeCache log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only") } else { if cfg.DNSProvider == "" { - // using mock server, don't use wildcard certs - err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache}) - if err != nil { - log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider") - } + // using mock wildcard certs + mainDomainAcmeClient = nil } else { // use DNS-Challenge https://go-acme.github.io/lego/dns/ provider, err := dns.NewDNSChallengeProviderByName(cfg.DNSProvider) diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index b638755..67219dd 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -33,6 +33,8 @@ func TLSConfig(mainDomainSuffix string, firstDefaultBranch string, keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.ICache, certDB database.CertDB, + noDNS01 bool, + rawDomain string, ) *tls.Config { return &tls.Config{ // check DNS name & get certificate from Let's Encrypt @@ -64,9 +66,24 @@ func TLSConfig(mainDomainSuffix string, targetOwner := "" mayObtainCert := true + if strings.HasSuffix(domain, mainDomainSuffix) || strings.EqualFold(domain, mainDomainSuffix[1:]) { - // deliver default certificate for the main domain (*.codeberg.page) - domain = mainDomainSuffix + if noDNS01 { + // Limit the domains allowed to request a certificate to pages-server domains + // and domains for an existing user of org + if !strings.EqualFold(domain, mainDomainSuffix[1:]) && !strings.EqualFold(domain, rawDomain) { + targetOwner := strings.TrimSuffix(domain, mainDomainSuffix) + owner_exist, err := giteaClient.GiteaCheckIfOwnerExists(targetOwner) + mayObtainCert = owner_exist + if err != nil { + log.Error().Err(err).Msgf("Failed to check '%s' existence on the forge: %s", targetOwner, err) + mayObtainCert = false + } + } + } else { + // deliver default certificate for the main domain (*.codeberg.page) + domain = mainDomainSuffix + } } else { var targetRepo, targetBranch string targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch, dnsLookupCache) @@ -199,9 +216,6 @@ func (c *AcmeClient) retrieveCertFromDB(sni, mainDomainSuffix string, useDnsProv func (c *AcmeClient) obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user string, useDnsProvider bool, mainDomainSuffix string, keyDatabase database.CertDB) (*tls.Certificate, error) { name := strings.TrimPrefix(domains[0], "*") - if useDnsProvider && domains[0] != "" && domains[0][0] == '*' { - domains = domains[1:] - } // lock to avoid simultaneous requests _, working := c.obtainLocks.LoadOrStore(name, struct{}{}) @@ -219,7 +233,11 @@ func (c *AcmeClient) obtainCert(acmeClient *lego.Client, domains []string, renew defer c.obtainLocks.Delete(name) if acmeClient == nil { - return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!", mainDomainSuffix, keyDatabase) + if useDnsProvider { + return mockCert(domains[0], "DNS ACME client is not defined", mainDomainSuffix, keyDatabase) + } else { + return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!", mainDomainSuffix, keyDatabase) + } } // request actual cert diff --git a/server/database/xorm.go b/server/database/xorm.go index 217b6d1..63fa39e 100644 --- a/server/database/xorm.go +++ b/server/database/xorm.go @@ -52,7 +52,6 @@ func (x xDB) Close() error { func (x xDB) Put(domain string, cert *certificate.Resource) error { log.Trace().Str("domain", cert.Domain).Msg("inserting cert to db") - domain = integrationTestReplacements(domain) c, err := toCert(domain, cert) if err != nil { return err @@ -82,7 +81,6 @@ func (x xDB) Get(domain string) (*certificate.Resource, error) { if domain[:1] == "." { domain = "*" + domain } - domain = integrationTestReplacements(domain) cert := new(Cert) log.Trace().Str("domain", domain).Msg("get cert from db") @@ -99,7 +97,6 @@ func (x xDB) Delete(domain string) error { if domain[:1] == "." { domain = "*" + domain } - domain = integrationTestReplacements(domain) log.Trace().Str("domain", domain).Msg("delete cert from db") _, err := x.engine.ID(domain).Delete(new(Cert)) @@ -139,13 +136,3 @@ func supportedDriver(driver string) bool { return false } } - -// integrationTestReplacements is needed because integration tests use a single domain cert, -// while production use a wildcard cert -// TODO: find a better way to handle this -func integrationTestReplacements(domainKey string) string { - if domainKey == "*.localhost.mock.directory" { - return "localhost.mock.directory" - } - return domainKey -} diff --git a/server/gitea/cache.go b/server/gitea/cache.go index 4ecc5e0..97024a1 100644 --- a/server/gitea/cache.go +++ b/server/gitea/cache.go @@ -26,6 +26,9 @@ const ( // TODO: move as option into cache interface fileCacheTimeout = 5 * time.Minute + // ownerExistenceCacheTimeout specifies the timeout for the existence of a repo/org + ownerExistenceCacheTimeout = 5 * time.Minute + // fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default. fileCacheSizeLimit = int64(1000 * 1000) ) diff --git a/server/gitea/client.go b/server/gitea/client.go index 4f7eaa6..5955bfb 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -28,6 +28,7 @@ const ( branchTimestampCacheKeyPrefix = "branchTime" defaultBranchCacheKeyPrefix = "defaultBranch" rawContentCacheKeyPrefix = "rawContent" + ownerExistenceKeyPrefix = "ownerExist" // pages server PagesCacheIndicatorHeader = "X-Pages-Cache" @@ -266,6 +267,38 @@ func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (str return branch, nil } +func (client *Client) GiteaCheckIfOwnerExists(owner string) (bool, error) { + cacheKey := fmt.Sprintf("%s/%s", ownerExistenceKeyPrefix, owner) + + if exist, ok := client.responseCache.Get(cacheKey); ok && exist != nil { + return exist.(bool), nil + } + + _, resp, err := client.sdkClient.GetUserInfo(owner) + if resp.StatusCode == http.StatusOK && err == nil { + if err := client.responseCache.Set(cacheKey, true, ownerExistenceCacheTimeout); err != nil { + log.Error().Err(err).Msg("[cache] error on cache write") + } + return true, nil + } else if resp.StatusCode != http.StatusNotFound { + return false, err + } + + _, resp, err = client.sdkClient.GetOrg(owner) + if resp.StatusCode == http.StatusOK && err == nil { + if err := client.responseCache.Set(cacheKey, true, ownerExistenceCacheTimeout); err != nil { + log.Error().Err(err).Msg("[cache] error on cache write") + } + return true, nil + } else if resp.StatusCode != http.StatusNotFound { + return false, err + } + if err := client.responseCache.Set(cacheKey, false, ownerExistenceCacheTimeout); err != nil { + log.Error().Err(err).Msg("[cache] error on cache write") + } + return false, nil +} + func (client *Client) getMimeTypeByExtension(resource string) string { mimeType := mime.TypeByExtension(path.Ext(resource)) mimeTypeSplit := strings.SplitN(mimeType, ";", 2) diff --git a/server/startup.go b/server/startup.go index ffdabb7..149a07d 100644 --- a/server/startup.go +++ b/server/startup.go @@ -110,6 +110,8 @@ func Serve(ctx *cli.Context) error { cfg.Server.PagesBranches[0], keyCache, challengeCache, dnsLookupCache, canonicalDomainCache, certDB, + cfg.ACME.NoDNS01, + cfg.Server.RawDomain, )) interval := 12 * time.Hour From 9ffdc9d4f9406b0cb2e1d07346f65c557a58568a Mon Sep 17 00:00:00 2001 From: Daniel Erat Date: Thu, 18 Apr 2024 21:03:16 +0000 Subject: [PATCH 16/19] Refactor redirect code and add tests (#304) Move repetitive code from Options.matchRedirects into a new Redirect.rewriteURL method and add a new test file. No functional changes are intended; this is in preparation for a later change to address #269. Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/304 Reviewed-by: crapStone Co-authored-by: Daniel Erat Co-committed-by: Daniel Erat --- server/upstream/redirects.go | 75 +++++++++++++------------------ server/upstream/redirects_test.go | 37 +++++++++++++++ 2 files changed, 68 insertions(+), 44 deletions(-) create mode 100644 server/upstream/redirects_test.go diff --git a/server/upstream/redirects.go b/server/upstream/redirects.go index dd36a84..e0601d8 100644 --- a/server/upstream/redirects.go +++ b/server/upstream/redirects.go @@ -17,6 +17,24 @@ type Redirect struct { StatusCode int } +// rewriteURL returns the destination URL and true if r matches reqURL. +func (r *Redirect) rewriteURL(reqURL string) (dstURL string, ok bool) { + // check if from url matches request url + if strings.TrimSuffix(r.From, "/") == strings.TrimSuffix(reqURL, "/") { + return r.To, true + } + // handle wildcard redirects + trimmedFromURL := strings.TrimSuffix(r.From, "/*") + if strings.HasSuffix(r.From, "/*") && strings.HasPrefix(reqURL, trimmedFromURL) { + if strings.Contains(r.To, ":splat") { + splatURL := strings.ReplaceAll(r.To, ":splat", strings.TrimPrefix(reqURL, trimmedFromURL)) + return splatURL, true + } + return r.To, true + } + return "", false +} + // redirectsCacheTimeout specifies the timeout for the redirects cache. var redirectsCacheTimeout = 10 * time.Minute @@ -64,52 +82,21 @@ func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.I } func (o *Options) matchRedirects(ctx *context.Context, giteaClient *gitea.Client, redirects []Redirect, redirectsCache cache.ICache) (final bool) { - if len(redirects) > 0 { - for _, redirect := range redirects { - reqUrl := ctx.Req.RequestURI - // remove repo and branch from request url - reqUrl = strings.TrimPrefix(reqUrl, "/"+o.TargetRepo) - reqUrl = strings.TrimPrefix(reqUrl, "/@"+o.TargetBranch) + reqURL := ctx.Req.RequestURI + // remove repo and branch from request url + reqURL = strings.TrimPrefix(reqURL, "/"+o.TargetRepo) + reqURL = strings.TrimPrefix(reqURL, "/@"+o.TargetBranch) - // check if from url matches request url - if strings.TrimSuffix(redirect.From, "/") == strings.TrimSuffix(reqUrl, "/") { - // do rewrite if status code is 200 - if redirect.StatusCode == 200 { - o.TargetPath = redirect.To - o.Upstream(ctx, giteaClient, redirectsCache) - return true - } else { - ctx.Redirect(redirect.To, redirect.StatusCode) - return true - } - } - - // handle wildcard redirects - trimmedFromUrl := strings.TrimSuffix(redirect.From, "/*") - if strings.HasSuffix(redirect.From, "/*") && strings.HasPrefix(reqUrl, trimmedFromUrl) { - if strings.Contains(redirect.To, ":splat") { - splatUrl := strings.ReplaceAll(redirect.To, ":splat", strings.TrimPrefix(reqUrl, trimmedFromUrl)) - // do rewrite if status code is 200 - if redirect.StatusCode == 200 { - o.TargetPath = splatUrl - o.Upstream(ctx, giteaClient, redirectsCache) - return true - } else { - ctx.Redirect(splatUrl, redirect.StatusCode) - return true - } - } else { - // do rewrite if status code is 200 - if redirect.StatusCode == 200 { - o.TargetPath = redirect.To - o.Upstream(ctx, giteaClient, redirectsCache) - return true - } else { - ctx.Redirect(redirect.To, redirect.StatusCode) - return true - } - } + for _, redirect := range redirects { + if dstURL, ok := redirect.rewriteURL(reqURL); ok { + // do rewrite if status code is 200 + if redirect.StatusCode == 200 { + o.TargetPath = dstURL + o.Upstream(ctx, giteaClient, redirectsCache) + } else { + ctx.Redirect(dstURL, redirect.StatusCode) } + return true } } diff --git a/server/upstream/redirects_test.go b/server/upstream/redirects_test.go new file mode 100644 index 0000000..c1088a9 --- /dev/null +++ b/server/upstream/redirects_test.go @@ -0,0 +1,37 @@ +package upstream + +import ( + "testing" +) + +func TestRedirect_rewriteURL(t *testing.T) { + for _, tc := range []struct { + redirect Redirect + reqURL string + wantDstURL string + wantOk bool + }{ + {Redirect{"/", "/dst", 200}, "/", "/dst", true}, + {Redirect{"/", "/dst", 200}, "/foo", "", false}, + {Redirect{"/src", "/dst", 200}, "/src", "/dst", true}, + {Redirect{"/src", "/dst", 200}, "/foo", "", false}, + {Redirect{"/src", "/dst", 200}, "/src/foo", "", false}, + {Redirect{"/*", "/dst", 200}, "/", "/dst", true}, + {Redirect{"/*", "/dst", 200}, "/src", "/dst", true}, + {Redirect{"/src/*", "/dst/:splat", 200}, "/src", "/dst/", true}, + // TODO: There shouldn't be double-slashes in these URLs: + // https://codeberg.org/Codeberg/pages-server/issues/269 + {Redirect{"/src/*", "/dst/:splat", 200}, "/src/", "/dst//", true}, + {Redirect{"/src/*", "/dst/:splat", 200}, "/src/foo", "/dst//foo", true}, + {Redirect{"/src/*", "/dst/:splat", 200}, "/src/foo/bar", "/dst//foo/bar", true}, + // TODO: This is a workaround for the above bug. Should it be preserved? + {Redirect{"/src/*", "/dst:splat", 200}, "/src/foo", "/dst/foo", true}, + // TODO: This behavior is incorrect; no redirect should be performed. + {Redirect{"/src/*", "/dst", 200}, "/srcfoo", "/dst", true}, + } { + if dstURL, ok := tc.redirect.rewriteURL(tc.reqURL); dstURL != tc.wantDstURL || ok != tc.wantOk { + t.Errorf("%#v.rewriteURL(%q) = %q, %v; want %q, %v", + tc.redirect, tc.reqURL, dstURL, ok, tc.wantDstURL, tc.wantOk) + } + } +} From a986a52755aba425b9a69c94ed48ceb01c56776a Mon Sep 17 00:00:00 2001 From: Moritz Marquardt Date: Thu, 18 Apr 2024 21:19:45 +0000 Subject: [PATCH 17/19] Fix masked error message from Gitea (#306) This would yield to the error "forge client failed" instead of e.g. "404 Not Found". The issue was introduced in cbb2ce6d0732bcf2372cbc201fc1c1f2733aadba. Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/306 Reviewed-by: crapStone Co-authored-by: Moritz Marquardt Co-committed-by: Moritz Marquardt --- server/handler/try.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/handler/try.go b/server/handler/try.go index 145b1a9..e76891d 100644 --- a/server/handler/try.go +++ b/server/handler/try.go @@ -1,6 +1,7 @@ package handler import ( + "fmt" "net/http" "strings" @@ -41,7 +42,7 @@ func tryUpstream(ctx *context.Context, giteaClient *gitea.Client, // Try to request the file from the Gitea API if !options.Upstream(ctx, giteaClient, redirectsCache) { - html.ReturnErrorPage(ctx, "forge client failed", ctx.StatusCode) + html.ReturnErrorPage(ctx, fmt.Sprintf("Forge returned %d %s", ctx.StatusCode, http.StatusText(ctx.StatusCode)), ctx.StatusCode) } } From 69fb22a9e793e92e23e74900950ac9f60a049f9d Mon Sep 17 00:00:00 2001 From: Daniel Erat Date: Sat, 20 Apr 2024 11:00:15 +0000 Subject: [PATCH 18/19] Avoid extra slashes in redirects with :splat (#308) Remove leading slashes from captured portions of paths when redirecting using splats. This makes a directive like "/articles/* /posts/:splat 302" behave as described in FEATURES.md, i.e. "/articles/foo" now redirects to "/posts/foo" rather than to "/posts//foo". Fixes #269. This also changes the behavior of a redirect like "/articles/* /posts:splat 302". "/articles/foo" will now redirect to "/postsfoo" rather than to "/posts/foo". This change also fixes an issue where paths like "/articles123" would be incorrectly matched by the above patterns. Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/308 Reviewed-by: crapStone Co-authored-by: Daniel Erat Co-committed-by: Daniel Erat --- server/upstream/redirects.go | 15 +++++++++------ server/upstream/redirects_test.go | 17 ++++++++--------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/server/upstream/redirects.go b/server/upstream/redirects.go index e0601d8..ddde375 100644 --- a/server/upstream/redirects.go +++ b/server/upstream/redirects.go @@ -24,13 +24,16 @@ func (r *Redirect) rewriteURL(reqURL string) (dstURL string, ok bool) { return r.To, true } // handle wildcard redirects - trimmedFromURL := strings.TrimSuffix(r.From, "/*") - if strings.HasSuffix(r.From, "/*") && strings.HasPrefix(reqURL, trimmedFromURL) { - if strings.Contains(r.To, ":splat") { - splatURL := strings.ReplaceAll(r.To, ":splat", strings.TrimPrefix(reqURL, trimmedFromURL)) - return splatURL, true + if strings.HasSuffix(r.From, "/*") { + trimmedFromURL := strings.TrimSuffix(r.From, "/*") + if reqURL == trimmedFromURL || strings.HasPrefix(reqURL, trimmedFromURL+"/") { + if strings.Contains(r.To, ":splat") { + matched := strings.TrimPrefix(reqURL, trimmedFromURL) + matched = strings.TrimPrefix(matched, "/") + return strings.ReplaceAll(r.To, ":splat", matched), true + } + return r.To, true } - return r.To, true } return "", false } diff --git a/server/upstream/redirects_test.go b/server/upstream/redirects_test.go index c1088a9..6118a70 100644 --- a/server/upstream/redirects_test.go +++ b/server/upstream/redirects_test.go @@ -19,15 +19,14 @@ func TestRedirect_rewriteURL(t *testing.T) { {Redirect{"/*", "/dst", 200}, "/", "/dst", true}, {Redirect{"/*", "/dst", 200}, "/src", "/dst", true}, {Redirect{"/src/*", "/dst/:splat", 200}, "/src", "/dst/", true}, - // TODO: There shouldn't be double-slashes in these URLs: - // https://codeberg.org/Codeberg/pages-server/issues/269 - {Redirect{"/src/*", "/dst/:splat", 200}, "/src/", "/dst//", true}, - {Redirect{"/src/*", "/dst/:splat", 200}, "/src/foo", "/dst//foo", true}, - {Redirect{"/src/*", "/dst/:splat", 200}, "/src/foo/bar", "/dst//foo/bar", true}, - // TODO: This is a workaround for the above bug. Should it be preserved? - {Redirect{"/src/*", "/dst:splat", 200}, "/src/foo", "/dst/foo", true}, - // TODO: This behavior is incorrect; no redirect should be performed. - {Redirect{"/src/*", "/dst", 200}, "/srcfoo", "/dst", true}, + {Redirect{"/src/*", "/dst/:splat", 200}, "/src/", "/dst/", true}, + {Redirect{"/src/*", "/dst/:splat", 200}, "/src/foo", "/dst/foo", true}, + {Redirect{"/src/*", "/dst/:splat", 200}, "/src/foo/bar", "/dst/foo/bar", true}, + {Redirect{"/src/*", "/dst/:splatsuffix", 200}, "/src/foo", "/dst/foosuffix", true}, + {Redirect{"/src/*", "/dst:splat", 200}, "/src/foo", "/dstfoo", true}, + {Redirect{"/src/*", "/dst", 200}, "/srcfoo", "", false}, + // This is the example from FEATURES.md: + {Redirect{"/articles/*", "/posts/:splat", 302}, "/articles/2022/10/12/post-1/", "/posts/2022/10/12/post-1/", true}, } { if dstURL, ok := tc.redirect.rewriteURL(tc.reqURL); dstURL != tc.wantDstURL || ok != tc.wantOk { t.Errorf("%#v.rewriteURL(%q) = %q, %v; want %q, %v", From 019e85a0d0c8315248732e7e04eeab26bb3308cc Mon Sep 17 00:00:00 2001 From: Dependency bot Date: Sat, 27 Apr 2024 18:47:42 +0000 Subject: [PATCH 19/19] chore: Configure Renovate (#284) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Welcome to [Renovate](https://github.com/renovatebot/renovate)! This is an onboarding PR to help you understand and configure settings before regular Pull Requests begin. 🚦 To activate Renovate, merge this Pull Request. To disable Renovate, simply close this Pull Request unmerged. --- ### Detected Package Files * `Dockerfile` (dockerfile) * `go.mod` (gomod) * `.woodpecker.yml` (woodpecker) ### Configuration Summary Based on the default config's presets, Renovate will: - Start dependency updates only once this onboarding PR is merged - Enable Renovate Dependency Dashboard creation. - Use semantic commit type `fix` for dependencies and `chore` for all others if semantic commits are in use. - Ignore `node_modules`, `bower_components`, `vendor` and various test/tests directories. - Group known monorepo packages together. - Use curated list of recommended non-monorepo package groupings. - Apply crowd-sourced package replacement rules. - Apply crowd-sourced workarounds for known problems with packages. - Run lock file maintenance (updates) early Monday mornings. - Schedule automerge daily. - Schedule for weekends. - Enable Renovate Dependency Dashboard creation. - Use semantic commit type `fix` for dependencies and `chore` for all others if semantic commits are in use. - Ignore `node_modules`, `bower_components`, `vendor` and various test/tests directories. - Group known monorepo packages together. - Use curated list of recommended non-monorepo package groupings. - Apply crowd-sourced package replacement rules. - Apply crowd-sourced workarounds for known problems with packages. - Run lock file maintenance (updates) early Monday mornings. - Enable the pre-commit manager. - Schedule automerge daily. - Schedule for weekends. - Run Renovate on following schedule: every weekend 🔡 Do you want to change how Renovate upgrades your dependencies? Add your custom config to `renovate.json` in this branch. Renovate will update the Pull Request description the next time it runs. --- ### What to Expect With your current configuration, Renovate will create 3 Pull Requests:
fix(deps): update golang.org/x/exp digest to fe59bbe - Schedule: ["every weekend"] - Branch name: `renovate/golang.org-x-exp-digest` - Merge into: `main` - Upgrade golang.org/x/exp to `fe59bbe5cc7f158318a9631d96683d2df264a3c1`
chore(deps): update golang docker tag to v1.22 - Schedule: ["every weekend"] - Branch name: `renovate/golang-1.x` - Merge into: `main` - Upgrade golang to `1.22`
fix(deps): update golang deps non-major - Schedule: ["before 4am"] - Branch name: `renovate/golang-deps-non-major` - Merge into: `main` - Upgrade [github.com/go-acme/lego/v4](https://github.com/go-acme/lego) to `v4.16.1` - Upgrade [github.com/go-sql-driver/mysql](https://github.com/go-sql-driver/mysql) to `v1.8.1` - Upgrade [github.com/joho/godotenv](https://github.com/joho/godotenv) to `v1.5.1` - Upgrade [github.com/lib/pq](https://github.com/lib/pq) to `v1.10.9` - Upgrade [github.com/mattn/go-sqlite3](https://github.com/mattn/go-sqlite3) to `v1.14.22` - Upgrade [github.com/pelletier/go-toml/v2](https://github.com/pelletier/go-toml) to `v2.2.1` - Upgrade [github.com/reugn/equalizer](https://github.com/reugn/equalizer) to `11d4adaf94377c6ab5d3fc2d434cd539de78cb73` - Upgrade [github.com/rs/zerolog](https://github.com/rs/zerolog) to `v1.32.0` - Upgrade [github.com/stretchr/testify](https://github.com/stretchr/testify) to `v1.9.0` - Upgrade [github.com/urfave/cli/v2](https://github.com/urfave/cli) to `v2.27.1` - Upgrade [go](https://github.com/golang/go) to `1.22.2` - Upgrade xorm.io/xorm to `v1.3.9`

🚸 Branch creation will be limited to maximum 2 per hour, so it doesn't swamp any CI resources or overwhelm the project. See docs for `prhourlylimit` for details. --- > > ⚠ **Warning** > > Please correct - or verify that you can safely ignore - these dependency lookup failures before you merge this PR. > > - `Could not determine new digest for update (go package code.gitea.io/sdk/gitea)` > > Files affected: `go.mod` --- ❓ Got questions? Check out Renovate's [Docs](https://docs.renovatebot.com/), particularly the Getting Started section. If you need any further assistance then you can also [request help here](https://github.com/renovatebot/renovate/discussions). --- This PR has been generated by [Renovate Bot](https://github.com/renovatebot/renovate). Co-authored-by: woodpecker-bot Co-authored-by: pat-s Co-authored-by: Patrick Schratz Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/284 Co-authored-by: Dependency bot Co-committed-by: Dependency bot --- renovate.json | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 renovate.json diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000..9dd1cd7 --- /dev/null +++ b/renovate.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:recommended", + ":maintainLockFilesWeekly", + ":enablePreCommit", + "schedule:automergeDaily", + "schedule:weekends" + ], + "automergeType": "branch", + "automergeMajor": false, + "automerge": true, + "prConcurrentLimit": 5, + "labels": ["dependencies"], + "packageRules": [ + { + "matchManagers": ["gomod", "dockerfile"] + }, + { + "groupName": "golang deps non-major", + "matchManagers": ["gomod"], + "matchUpdateTypes": ["minor", "patch"], + "extends": ["schedule:daily"] + } + ], + "postUpdateOptions": ["gomodTidy", "gomodUpdateImportPaths"] +}