chore(deps): update woodpeckerci/plugin-release docker tag to v0.2.5 (#449)

This PR contains the following updates:

| Package | Update | Change |
|---|---|---|
| [docker.io/woodpeckerci/plugin-prettier](https://codeberg.org/woodpecker-plugins/prettier) ([source](https://codeberg.org/woodpecker-plugins/prettier.git)) | minor | `1.1.0` -> `1.3.0` |

---

### Release Notes

<details>
<summary>woodpecker-plugins/prettier (docker.io/woodpeckerci/plugin-prettier)</summary>

### [`v1.3.0`](https://codeberg.org/woodpecker-plugins/prettier/blob/HEAD/CHANGELOG.md#130---2025-02-18)

[Compare Source](https://codeberg.org/woodpecker-plugins/prettier/compare/1.2.1...1.3.0)

##### ❤️ Thanks to all contributors! ❤️

[@&#8203;OCram85](https://github.com/OCram85)

##### 📈 Enhancement

-   adds output parameter for prettier results \[[#&#8203;25](https://codeberg.org/woodpecker-plugins/prettier/pulls/25)]

### [`v1.2.1`](https://codeberg.org/woodpecker-plugins/prettier/blob/HEAD/CHANGELOG.md#121---2025-02-13)

[Compare Source](https://codeberg.org/woodpecker-plugins/prettier/compare/1.1.0...1.2.1)

##### ❤️ Thanks to all contributors! ❤️

[@&#8203;woodpecker-bot](https://github.com/woodpecker-bot)

##### 📦️ Dependency

-   chore(deps): update woodpeckerci/plugin-docker-buildx docker tag to v5.2.1 \[[#&#8203;24](https://codeberg.org/woodpecker-plugins/prettier/pulls/24)]
-   chore(deps): update woodpeckerci/plugin-docker-buildx docker tag to v5.2.0 \[[#&#8203;22](https://codeberg.org/woodpecker-plugins/prettier/pulls/22)]
-   chore(deps): update woodpeckerci/plugin-ready-release-go docker tag to v3.1.3 \[[#&#8203;21](https://codeberg.org/woodpecker-plugins/prettier/pulls/21)]

</details>

---

### Configuration

📅 **Schedule**: Branch creation - "* * * * 0,6" (UTC), Automerge - "* 0-3 * * *" (UTC).

🚦 **Automerge**: Enabled.

♻ **Rebasing**: Whenever PR is behind base branch, or you tick the rebase/retry checkbox.

🔕 **Ignore**: Close this PR and you won't be reminded about this update again.

---

 - [ ] <!-- rebase-check -->If you want to rebase/retry this PR, check this box

---

This PR has been generated by [Renovate Bot](https://github.com/renovatebot/renovate).
<!--renovate-debug:eyJjcmVhdGVkSW5WZXIiOiIzOS4xNzMuMSIsInVwZGF0ZWRJblZlciI6IjM5LjE3Ni40IiwidGFyZ2V0QnJhbmNoIjoibWFpbiIsImxhYmVscyI6WyJkZXBlbmRlbmNpZXMiXX0=-->

Co-authored-by: woodpecker-bot <woodpecker-bot@obermui.de>
Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/435
Co-authored-by: Dependency bot <renovate-bot@noreply.codeberg.org>
Co-committed-by: Dependency bot <renovate-bot@noreply.codeberg.org>

.env-dev

@@ -0,0 +1,11 @@
+ACME_API=https://acme.mock.directory
+ACME_ACCEPT_TERMS=true
+PAGES_DOMAIN=localhost.mock.directory
+RAW_DOMAIN=raw.localhost.mock.directory
+PAGES_BRANCHES=pages,master,main
+GITEA_ROOT=https://codeberg.org
+PORT=4430
+HTTP_PORT=8880
+ENABLE_HTTP_SERVER=true
+LOG_LEVEL=trace
+ACME_ACCOUNT_CONFIG=integration/acme-account.json

.envrc

@@ -0,0 +1 @@
+use_flake

.gitea/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Codeberg Pages Usage Support
+    url: https://codeberg.org/Codeberg/Community/issues/
+    about: If you need help with configuring Codeberg Pages on codeberg.org, please go here.

.gitignore

@@ -7,3 +7,6 @@ build/
 vendor/
 pages
 certs.sqlite
+.bash_history
+pkg/
+.direnv/

.prettierrc.json

@@ -0,0 +1,8 @@
+{
+  "semi": true,
+  "trailingComma": "all",
+  "singleQuote": true,
+  "printWidth": 120,
+  "tabWidth": 2,
+  "endOfLine": "lf"
+}

.vscode/launch.json

@@ -0,0 +1,26 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Launch PagesServer",
+      "type": "go",
+      "request": "launch",
+      "mode": "auto",
+      "program": "${workspaceFolder}/main.go",
+      "args": ["sqlite", "sqlite_unlock_notify", "netgo"],
+      "envFile": "${workspaceFolder}/.env-dev"
+    },
+    {
+      "name": "Launch PagesServer integration test",
+      "type": "go",
+      "request": "launch",
+      "mode": "auto",
+      "program": "${workspaceFolder}/integration/main_test.go",
+      "args": ["codeberg.org/codeberg/pages/integration/..."],
+      "buildFlags": ["-tags", "'integration sqlite sqlite_unlock_notify netgo'"]
+    }
+  ]
+}

.woodpecker.yml

@@ -1,124 +0,0 @@
-pipeline:
-  # use vendor to cache dependencies
-  vendor:
-    image: golang:1.20
-    commands:
-      - go mod vendor
-
-  lint:
-    image: golangci/golangci-lint:latest
-    group: compliant
-    pull: true
-    commands:
-      - go version
-      - go install mvdan.cc/gofumpt@latest
-      - "[ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }"
-      - golangci-lint run --timeout 5m --build-tags integration
-
-  editor-config:
-    group: compliant
-    image: mstruebing/editorconfig-checker
-
-  build:
-    group: compliant
-    image: codeberg.org/6543/docker-images/golang_just
-    commands:
-      - go version
-      - just build
-    when:
-      event: [ "pull_request", "push" ]
-
-  docker-dryrun:
-    group: compliant
-    image: plugins/kaniko
-    settings:
-      dockerfile: Dockerfile
-      no_push: true
-      tags: latest
-    when:
-      event: [ "pull_request", "push" ]
-      path: Dockerfile
-
-  build-tag:
-    group: compliant
-    image: codeberg.org/6543/docker-images/golang_just
-    commands:
-      - go version
-      - just build-tag ${CI_COMMIT_TAG##v}
-    when:
-      event: [ "tag" ]
-
-  test:
-    group: test
-    image: codeberg.org/6543/docker-images/golang_just
-    commands:
-      - just test
-
-  integration-tests:
-    group: test
-    image: codeberg.org/6543/docker-images/golang_just
-    commands:
-      - just integration
-    environment:
-      - ACME_API=https://acme.mock.directory
-      - PAGES_DOMAIN=localhost.mock.directory
-      - RAW_DOMAIN=raw.localhost.mock.directory
-      - PORT=4430
-
-  # TODO: remove in next version
-  integration-tests-legacy:
-    group: test
-    image: codeberg.org/6543/docker-images/golang_just
-    commands:
-      - just integration
-    environment:
-      - ACME_API=https://acme.mock.directory
-      - PAGES_DOMAIN=localhost.mock.directory
-      - RAW_DOMAIN=raw.localhost.mock.directory
-      - PORT=4430
-      - DB_TYPE=
-
-  release:
-    image: plugins/gitea-release
-    settings:
-      base_url: https://codeberg.org
-      file_exists: overwrite
-      files: build/codeberg-pages-server
-      api_key:
-        from_secret: bot_token
-    environment:
-      - DRONE_REPO_OWNER=${CI_REPO_OWNER}
-      - DRONE_REPO_NAME=${CI_REPO_NAME}
-      - DRONE_BUILD_EVENT=${CI_BUILD_EVENT}
-      - DRONE_COMMIT_REF=${CI_COMMIT_REF}
-    when:
-      event: [ "tag" ]
-
-  docker-next:
-    image: plugins/kaniko
-    settings:
-      registry: codeberg.org
-      dockerfile: Dockerfile
-      repo: codeberg.org/codeberg/pages-server
-      tags: next
-      username:
-        from_secret: bot_user
-      password:
-        from_secret: bot_token
-    when:
-      event: [ "push" ]
-      branch: ${CI_REPO_DEFAULT_BRANCH}
-
-  docker-tag:
-    image: plugins/kaniko
-    settings:
-      registry: codeberg.org
-      dockerfile: Dockerfile
-      repo: codeberg.org/codeberg/pages-server
-      tags: [ latest, "${CI_COMMIT_TAG}" ]
-      username:
-        from_secret: bot_user
-      password:
-        from_secret: bot_token
-    when:
-      event: [ "tag" ]

.woodpecker/build.yml

@@ -0,0 +1,123 @@
+when:
+  - event: [push, pull_request, tag, cron]
+    branch: ${CI_REPO_DEFAULT_BRANCH}
+
+steps:
+  # use vendor to cache dependencies
+  vendor:
+    image: golang:1.24
+    commands:
+      - go mod vendor
+
+  build:
+    depends_on: vendor
+    image: codeberg.org/6543/docker-images/golang_just:go-1.24
+    commands:
+      - go version
+      - just build
+    when:
+      - event: [push, pull_request, tag]
+
+  docker-dryrun:
+    depends_on: vendor
+    image: woodpeckerci/plugin-docker-buildx:5.2.1
+    settings:
+      dockerfile: Dockerfile
+      platforms: linux/amd64
+      dry-run: true
+      tags: latest
+    when:
+      - event: [pull_request]
+        path: Dockerfile
+
+  build-tag:
+    depends_on: vendor
+    image: codeberg.org/6543/docker-images/golang_just:go-1.24
+    commands:
+      - go version
+      - just build-tag ${CI_COMMIT_TAG##v}
+    when:
+      - event: [tag]
+
+  test:
+    depends_on: build
+    image: codeberg.org/6543/docker-images/golang_just:go-1.24
+    commands:
+      - just test
+    when:
+      - event: [pull_request]
+
+  integration-tests:
+    failure: ignore
+    depends_on: build
+    image: codeberg.org/6543/docker-images/golang_just:go-1.24
+    commands:
+      - just integration
+    environment:
+      ACME_API: https://acme.mock.directory
+      PAGES_DOMAIN: localhost.mock.directory
+      RAW_DOMAIN: raw.localhost.mock.directory
+      PORT: 4430
+    when:
+      - event: [pull_request]
+
+  release:
+    depends_on: build
+    image: woodpeckerci/plugin-release:0.2.5
+    settings:
+      base_url: https://codeberg.org
+      file_exists: overwrite
+      files: build/codeberg-pages-server
+      api_key:
+        from_secret: bot_token
+    when:
+      - event: [tag]
+
+  docker-next:
+    depends_on: vendor
+    image: woodpeckerci/plugin-docker-buildx:5.2.1
+    settings:
+      registry: codeberg.org
+      dockerfile: Dockerfile
+      platforms: linux/amd64,arm64
+      repo: codeberg.org/codeberg/pages-server
+      tags: next
+      username:
+        from_secret: bot_user
+      password:
+        from_secret: bot_token
+    when:
+      - event: [push]
+
+  'Publish PR image':
+    image: woodpeckerci/plugin-docker-buildx:5.2.1
+    depends_on: test
+    settings:
+      registry: codeberg.org
+      dockerfile: Dockerfile
+      platforms: linux/amd64
+      repo: codeberg.org/codeberg/pages-server
+      tags: next
+      username:
+        from_secret: bot_user
+      password:
+        from_secret: bot_token
+    when:
+      evaluate: 'CI_COMMIT_PULL_REQUEST_LABELS contains "build_pr_image"'
+      event: pull_request
+
+  docker-release:
+    depends_on: vendor
+    image: woodpeckerci/plugin-docker-buildx:5.2.1
+    settings:
+      registry: codeberg.org
+      dockerfile: Dockerfile
+      platforms: linux/amd64,arm64
+      repo: codeberg.org/codeberg/pages-server
+      tags: [latest, '${CI_COMMIT_TAG}']
+      username:
+        from_secret: bot_user
+      password:
+        from_secret: bot_token
+    when:
+      - event: [tag]

.woodpecker/lint.yml

@@ -0,0 +1,30 @@
+when:
+  - event: pull_request
+    branch:
+      - ${CI_REPO_DEFAULT_BRANCH}
+
+steps:
+  lint:
+    depends_on: []
+    image: golangci/golangci-lint:v1.64.8
+    commands:
+      - go version
+      - go install mvdan.cc/gofumpt@latest
+      - "[ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }"
+      - golangci-lint run --timeout 10m --build-tags integration
+
+  editor-config:
+    depends_on: []
+    image: mstruebing/editorconfig-checker:v3.2.0
+
+  yamllint:
+    image: pipelinecomponents/yamllint:0.33.0
+    depends_on: []
+    commands:
+      - yamllint .
+
+  prettier:
+    image: docker.io/woodpeckerci/plugin-prettier:1.3.0
+    depends_on: []
+    settings:
+      version: 3.2.5

.yamllint.yaml

@@ -0,0 +1,19 @@
+extends: default
+
+rules:
+  comments:
+    require-starting-space: false
+    ignore-shebangs: true
+    min-spaces-from-content: 1
+  braces:
+    min-spaces-inside: 1
+    max-spaces-inside: 1
+  document-start:
+    present: false
+  indentation:
+    spaces: 2
+    indent-sequences: true
+  line-length:
+    max: 256
+  new-lines:
+    type: unix

Dockerfile

@@ -1,14 +1,36 @@
-FROM techknowlogick/xgo as build
+# Set the default Go version as a build argument
+ARG XGO="go-1.24.x"
 
-WORKDIR /workspace
+# Use xgo (a Go cross-compiler tool) as build image
+FROM --platform=$BUILDPLATFORM techknowlogick/xgo:${XGO} AS build
 
-COPY . .
-RUN CGO_ENABLED=1 go build -tags 'sqlite sqlite_unlock_notify netgo' -ldflags '-s -w -extldflags "-static" -linkmode external' .
+# Set the working directory and copy the source code
+WORKDIR /go/src/codeberg.org/codeberg/pages
+COPY . /go/src/codeberg.org/codeberg/pages
 
+# Set the target architecture (can be set using --build-arg), buildx set it automatically
+ARG TARGETOS TARGETARCH
+
+# Build the binary using xgo
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg \
+    GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=1 \
+    xgo -x -v --targets=${TARGETOS}/${TARGETARCH} -tags='sqlite sqlite_unlock_notify netgo' -ldflags='-s -w -extldflags "-static" -linkmode external' -out pages .
+RUN mv -vf /build/pages-* /go/src/codeberg.org/codeberg/pages/pages
+
+# Use a scratch image as the base image for the final container,
+# which will contain only the built binary and the CA certificates
 FROM scratch
-COPY --from=build /workspace/pages /pages
+
+# Copy the built binary and the CA certificates from the build container to the final container
+COPY --from=build /go/src/codeberg.org/codeberg/pages/pages /pages
 COPY --from=build \
     /etc/ssl/certs/ca-certificates.crt \
     /etc/ssl/certs/ca-certificates.crt
 
+# Expose ports 80 and 443 for the built binary to listen on
+EXPOSE 80/tcp
+EXPOSE 443/tcp
+
+# Set the entrypoint for the container to the built binary
 ENTRYPOINT ["/pages"]

FEATURES.md

@@ -0,0 +1,51 @@
+# Features
+
+## Custom domains
+
+Custom domains can be used by creating a `.domains` file with the domain name, e.g.:
+
+```text
+codeberg.page
+```
+
+You also have to set some DNS records, see the [Codeberg Documentation](https://docs.codeberg.org/codeberg-pages/using-custom-domain/).
+
+## Redirects
+
+Redirects can be created with a `_redirects` file with the following format:
+
+```text
+# Comment
+from  to  [status]
+```
+
+- Lines starting with `#` are ignored
+- `from` - the path to redirect from (Note: repository and branch names are removed from request URLs)
+- `to` - the path or URL to redirect to
+- `status` - status code to use when redirecting (default 301)
+
+### Status codes
+
+- `200` - returns content from specified path (no external URLs) without changing the URL (rewrite)
+- `301` - Moved Permanently (Permanent redirect)
+- `302` - Found (Temporary redirect)
+
+### Examples
+
+#### SPA (single-page application) rewrite
+
+Redirects all paths to `/index.html` for single-page apps.
+
+```text
+/*  /index.html 200
+```
+
+#### Splats
+
+Redirects every path under `/articles` to `/posts` while keeping the path.
+
+```text
+/articles/*  /posts/:splat  302
+```
+
+Example: `/articles/2022/10/12/post-1/` -> `/posts/2022/10/12/post-1/`

Justfile

@@ -1,16 +1,13 @@
 CGO_FLAGS := '-extldflags "-static" -linkmode external'
 TAGS      := 'sqlite sqlite_unlock_notify netgo'
 
-dev:
+dev *FLAGS:
     #!/usr/bin/env bash
     set -euxo pipefail
-    export ACME_API=https://acme.mock.directory
-    export ACME_ACCEPT_TERMS=true
-    export PAGES_DOMAIN=localhost.mock.directory
-    export RAW_DOMAIN=raw.localhost.mock.directory
-    export PORT=4430
-    export LOG_LEVEL=trace
-    go run -tags '{{TAGS}}' .
+    set -a # automatically export all variables
+    source .env-dev
+    set +a
+    go run -tags '{{TAGS}}' . {{FLAGS}}
 
 build:
     CGO_ENABLED=1 go build -tags '{{TAGS}}' -ldflags '-s -w {{CGO_FLAGS}}' -v -o build/codeberg-pages-server ./
@@ -27,7 +24,7 @@ fmt: tool-gofumpt
 
 clean:
     go clean ./...
-    rm -rf build/ integration/certs.sqlite integration/key-database.pogreb/ integration/acme-account.json
+    rm -rf build/ integration/certs.sqlite integration/acme-account.json
 
 tool-golangci:
     @hash golangci-lint> /dev/null 2>&1; if [ $? -ne 0 ]; then \
@@ -40,13 +37,16 @@ tool-gofumpt:
     fi
 
 test:
-    go test -race -cover -tags '{{TAGS}}' codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/
+    go test -race -cover -tags '{{TAGS}}' codeberg.org/codeberg/pages/config/ codeberg.org/codeberg/pages/html/ codeberg.org/codeberg/pages/server/...
 
 test-run TEST:
-    go test -race -tags '{{TAGS}}' -run "^{{TEST}}$" codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/
+    go test -race -tags '{{TAGS}}' -run "^{{TEST}}$" codeberg.org/codeberg/pages/config/ codeberg.org/codeberg/pages/html/ codeberg.org/codeberg/pages/server/...
 
 integration:
     go test -race -tags 'integration {{TAGS}}' codeberg.org/codeberg/pages/integration/...
 
 integration-run TEST:
     go test -race -tags 'integration {{TAGS}}' -run "^{{TEST}}$" codeberg.org/codeberg/pages/integration/...
+
+docker:
+    docker run --rm -it --user $(id -u) -v $(pwd):/work --workdir /work -e HOME=/work codeberg.org/6543/docker-images/golang_just

README.md

@@ -1,5 +1,11 @@
 # Codeberg Pages
 
+[![License: EUPL-1.2](https://img.shields.io/badge/License-EUPL--1.2-blue)](https://opensource.org/license/eupl-1-2/)
+[![status-badge](https://ci.codeberg.org/api/badges/Codeberg/pages-server/status.svg)](https://ci.codeberg.org/Codeberg/pages-server)
+<a href="https://matrix.to/#/#gitea-pages-server:matrix.org" title="Join the Matrix room at https://matrix.to/#/#gitea-pages-server:matrix.org">
+<img src="https://img.shields.io/matrix/gitea-pages-server:matrix.org?label=matrix">
+</a>
+
 Gitea lacks the ability to host static pages from Git.
 The Codeberg Pages Server addresses this lack by implementing a standalone service
 that connects to Gitea via API.
@@ -8,30 +14,37 @@ It is suitable to be deployed by other Gitea instances, too, to offer static pag
 **End user documentation** can mainly be found at the [Wiki](https://codeberg.org/Codeberg/pages-server/wiki/Overview)
 and the [Codeberg Documentation](https://docs.codeberg.org/codeberg-pages/).
 
+<a href="https://codeberg.org/Codeberg/pages-server"> <img src="https://codeberg.org/Codeberg/GetItOnCodeberg/raw/branch/main/get-it-on-blue-on-white.svg" alt="Get It On Codeberg" width="250"/> </a>
+
 ## Quickstart
 
 This is the new Codeberg Pages server, a solution for serving static pages from Gitea repositories.
 Mapping custom domains is not static anymore, but can be done with DNS:
 
-1) add a `.domains` text file to your repository, containing the allowed domains, separated by new lines. The
-first line will be the canonical domain/URL; all other occurrences will be redirected to it.
+1. add a `.domains` text file to your repository, containing the allowed domains, separated by new lines. The
+   first line will be the canonical domain/URL; all other occurrences will be redirected to it.
 
-2) add a CNAME entry to your domain, pointing to `[[{branch}.]{repo}.]{owner}.codeberg.page` (repo defaults to
-"pages", "branch" defaults to the default branch if "repo" is "pages", or to "pages" if "repo" is something else.
-If the branch name contains slash characters, you need to replace "/" in the branch name to "~"):
-  `www.example.org. IN CNAME main.pages.example.codeberg.page.`
+2. add a CNAME entry to your domain, pointing to `[[{branch}.]{repo}.]{owner}.codeberg.page` (repo defaults to
+   "pages", "branch" defaults to the default branch if "repo" is "pages", or to "pages" if "repo" is something else.
+   If the branch name contains slash characters, you need to replace "/" in the branch name to "~"):
+   `www.example.org. IN CNAME main.pages.example.codeberg.page.`
 
-3) if a CNAME is set for "www.example.org", you can redirect there from the naked domain by adding an ALIAS record
-for "example.org" (if your provider allows ALIAS or similar records, otherwise use A/AAAA), together with a TXT
-record that points to your repo (just like the CNAME record):
-  `example.org IN ALIAS codeberg.page.`
-  `example.org IN TXT main.pages.example.codeberg.page.`
+3. if a CNAME is set for "www.example.org", you can redirect there from the naked domain by adding an ALIAS record
+   for "example.org" (if your provider allows ALIAS or similar records, otherwise use A/AAAA), together with a TXT
+   record that points to your repo (just like the CNAME record):
+   `example.org IN ALIAS codeberg.page.`
+   `example.org IN TXT main.pages.example.codeberg.page.`
 
 Certificates are generated, updated and cleaned up automatically via Let's Encrypt through a TLS challenge.
 
+## Chat for admins & devs
+
+[matrix: #gitea-pages-server:matrix.org](https://matrix.to/#/#gitea-pages-server:matrix.org)
+
 ## Deployment
 
-**Warning: Some Caveats Apply**  
+**Warning: Some Caveats Apply**
+
 > Currently, the deployment requires you to have some knowledge of system administration as well as understanding and building code,
 > so you can eventually edit non-configurable and codeberg-specific settings.
 > In the future, we'll try to reduce these and make hosting Codeberg Pages as easy as setting up Gitea.
@@ -48,28 +61,37 @@ but if you want to run it on a shared IP address (and not a standalone),
 you'll need to configure your reverse proxy not to terminate the TLS connections,
 but forward the requests on the IP level to the Pages Server.
 
-You can check out a proof of concept in the `haproxy-sni` folder,
-and especially have a look at [this section of the haproxy.cfg](https://codeberg.org/Codeberg/pages-server/src/branch/main/haproxy-sni/haproxy.cfg#L38).
+You can check out a proof of concept in the `examples/haproxy-sni` folder,
+and especially have a look at [this section of the haproxy.cfg](https://codeberg.org/Codeberg/pages-server/src/branch/main/examples/haproxy-sni/haproxy.cfg#L38).
 
-### Environment
+If you want to test a change, you can open a PR and ask for the label `build_pr_image` to be added.
+This will trigger a build of the PR which will build a docker image to be used for testing.
+
+### Environment Variables
 
-- `HOST` & `PORT` (default: `[::]` & `443`): listen address.
-- `PAGES_DOMAIN` (default: `codeberg.page`): main domain for pages.
-- `RAW_DOMAIN` (default: `raw.codeberg.page`): domain for raw resources.
-- `GITEA_ROOT` (default: `https://codeberg.org`): root of the upstream Gitea instance.
-- `GITEA_API_TOKEN` (default: empty): API token for the Gitea instance to access non-public (e.g. limited) repos.
-- `RAW_INFO_PAGE` (default: https://docs.codeberg.org/pages/raw-content/): info page for raw resources, shown if no resource is provided.
-- `ACME_API` (default: https://acme-v02.api.letsencrypt.org/directory): set this to https://acme.mock.director to use invalid certificates without any verification (great for debugging).  
-  ZeroSSL might be better in the future as it doesn't have rate limits and doesn't clash with the official Codeberg certificates (which are using Let's Encrypt), but I couldn't get it to work yet.
-- `ACME_EMAIL` (default: `noreply@example.email`): Set this to "true" to accept the Terms of Service of your ACME provider.
-- `ACME_EAB_KID` &  `ACME_EAB_HMAC` (default: don't use EAB): EAB credentials, for example for ZeroSSL.
 - `ACME_ACCEPT_TERMS` (default: use self-signed certificate): Set this to "true" to accept the Terms of Service of your ACME provider.
+- `ACME_API` (default: <https://acme-v02.api.letsencrypt.org/directory>): set this to <https://acme.mock.directory> to use invalid certificates without any verification (great for debugging). ZeroSSL might be better in the future as it doesn't have rate limits and doesn't clash with the official Codeberg certificates (which are using Let's Encrypt), but I couldn't get it to work yet.
+- `ACME_EAB_KID` & `ACME_EAB_HMAC` (default: don't use EAB): EAB credentials, for example for ZeroSSL.
+- `ACME_EMAIL` (default: `noreply@example.email`): Set the email sent to the ACME API server to receive, for example, renewal reminders.
 - `ACME_USE_RATE_LIMITS` (default: true): Set this to false to disable rate limits, e.g. with ZeroSSL.
+- `DNS_PROVIDER` (default: use self-signed certificate): Code of the ACME DNS provider for the main domain wildcard. See <https://go-acme.github.io/lego/dns/> for available values & additional environment variables.
 - `ENABLE_HTTP_SERVER` (default: false): Set this to true to enable the HTTP-01 challenge and redirect all other HTTP requests to HTTPS. Currently only works with port 80.
-- `DNS_PROVIDER` (default: use self-signed certificate): Code of the ACME DNS provider for the main domain wildcard.  
-  See https://go-acme.github.io/lego/dns/ for available values & additional environment variables.
+- `GITEA_API_TOKEN` (default: empty): API token for the Gitea instance to access non-public (e.g. limited) repos.
+- `GITEA_ROOT` (default: `https://codeberg.org`): root of the upstream Gitea instance.
+- `HOST` & `PORT` (default: `[::]` & `443`): listen address.
 - `LOG_LEVEL` (default: warn): Set this to specify the level of logging.
+- `NO_DNS_01` (default: `false`): Disable the use of ACME DNS. This means that the wildcard certificate is self-signed and all domains and subdomains will have a distinct certificate. Because this may lead to a rate limit from the ACME provider, this option is not recommended for Gitea/Forgejo instances with open registrations or a great number of users/orgs.
+- `PAGES_DOMAIN` (default: `codeberg.page`): main domain for pages.
+- `RAW_DOMAIN` (default: `raw.codeberg.page`): domain for raw resources (must be subdomain of `PAGES_DOMAIN`).
 
+### Custom Error Page
+
+A custom error page template can be served by creating `custom/error.html`.
+Data available to the template includes:
+
+- `{{ .StatusCode }}`: The HTTP status code (e.g. 404)
+- `{{ .StatusText }}`: The textual name associated with the status code (e.g. Not Found)
+- `{{ .Message }}`: The reason for the error
 
 ## Contributing to the development
 
@@ -78,9 +100,13 @@ Since we are working nicely in a team, it might be hard at times to get started
 (still check out the issues, we always aim to have some things to get you started).
 
 If you have any questions, want to work on a feature or could imagine collaborating with us for some time,
-feel free to ping us in an issue or in a general Matrix chatgroup.
+feel free to ping us in an issue or in a general [Matrix chat room](#chat-for-admins--devs).
 
-You can also contact the maintainers of this project:
+You can also contact the maintainer(s) of this project:
+
+- [crapStone](https://codeberg.org/crapStone) [(Matrix)](https://matrix.to/#/@crapstone:obermui.de)
+
+Previous maintainers:
 
 - [momar](https://codeberg.org/momar) [(Matrix)](https://matrix.to/#/@moritz:wuks.space)
 - [6543](https://codeberg.org/6543) [(Matrix)](https://matrix.to/#/@marddl:obermui.de)
@@ -88,21 +114,37 @@ You can also contact the maintainers of this project:
 ### First steps
 
 The code of this repository is split in several modules.
-While heavy refactoring work is currently undergo, you can easily understand the basic structure:
+The [Architecture is explained](https://codeberg.org/Codeberg/pages-server/wiki/Architecture) in the wiki.
+
 The `cmd` folder holds the data necessary for interacting with the service via the cli.
-If you are considering to deploy the service yourself, make sure to check it out.
 The heart of the software lives in the `server` folder and is split in several modules.
-After scanning the code, you should quickly be able to understand their function and start hacking on them.
 
 Again: Feel free to get in touch with us for any questions that might arise.
 Thank you very much.
 
-
 ### Test Server
 
+Make sure you have [golang](https://go.dev) v1.21 or newer and [just](https://just.systems/man/en/) installed.
+
 run `just dev`
-now this pages should work:
- - https://cb_pages_tests.localhost.mock.directory:4430/images/827679288a.jpg
- - https://momar.localhost.mock.directory:4430/ci-testing/
- - https://momar.localhost.mock.directory:4430/pag/@master/
- - https://mock-pages.codeberg-test.org:4430/README.md
+now these pages should work:
+
+- <https://cb_pages_tests.localhost.mock.directory:4430/images/827679288a.jpg>
+- <https://momar.localhost.mock.directory:4430/ci-testing/>
+- <https://momar.localhost.mock.directory:4430/pag/@master/>
+- <https://mock-pages.codeberg-test.org:4430/README.md>
+
+### Profiling
+
+> This section is just a collection of commands for quick reference. If you want to learn more about profiling read [this](https://go.dev/doc/diagnostics) article or google `golang profiling`.
+
+First enable profiling by supplying the cli arg `--enable-profiling` or using the environment variable `EENABLE_PROFILING`.
+
+Get cpu and mem stats:
+
+```bash
+go tool pprof -raw -output=cpu.txt 'http://localhost:9999/debug/pprof/profile?seconds=60' &
+curl -so mem.txt 'http://localhost:9999/debug/pprof/heap?seconds=60'
+```
+
+More endpoints are documented here: <https://pkg.go.dev/net/http/pprof>

cli/certs.go

@@ -0,0 +1,69 @@
+package cli
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/urfave/cli/v2"
+)
+
+var Certs = &cli.Command{
+	Name:  "certs",
+	Usage: "manage certs manually",
+	Subcommands: []*cli.Command{
+		{
+			Name:   "list",
+			Usage:  "list all certificates in the database",
+			Action: listCerts,
+		},
+		{
+			Name:   "remove",
+			Usage:  "remove a certificate from the database",
+			Action: removeCert,
+		},
+	},
+	Flags: CertStorageFlags,
+}
+
+func listCerts(ctx *cli.Context) error {
+	certDB, closeFn, err := OpenCertDB(ctx)
+	if err != nil {
+		return err
+	}
+	defer closeFn()
+
+	items, err := certDB.Items(0, 0)
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("Domain\tValidTill\n\n")
+	for _, cert := range items {
+		fmt.Printf("%s\t%s\n",
+			cert.Domain,
+			time.Unix(cert.ValidTill, 0).Format(time.RFC3339))
+	}
+	return nil
+}
+
+func removeCert(ctx *cli.Context) error {
+	if ctx.Args().Len() < 1 {
+		return fmt.Errorf("'certs remove' requires at least one domain as an argument")
+	}
+
+	domains := ctx.Args().Slice()
+
+	certDB, closeFn, err := OpenCertDB(ctx)
+	if err != nil {
+		return err
+	}
+	defer closeFn()
+
+	for _, domain := range domains {
+		fmt.Printf("Removing domain %s from the database...\n", domain)
+		if err := certDB.Delete(domain); err != nil {
+			return err
+		}
+	}
+	return nil
+}

cli/flags.go

@@ -0,0 +1,214 @@
+package cli
+
+import (
+	"github.com/urfave/cli/v2"
+)
+
+var (
+	CertStorageFlags = []cli.Flag{
+		&cli.StringFlag{
+			Name:    "db-type",
+			Usage:   "Specify the database driver. Valid options are \"sqlite3\", \"mysql\" and \"postgres\". Read more at https://xorm.io",
+			Value:   "sqlite3",
+			EnvVars: []string{"DB_TYPE"},
+		},
+		&cli.StringFlag{
+			Name:    "db-conn",
+			Usage:   "Specify the database connection. For \"sqlite3\" it's the filepath. Read more at https://go.dev/doc/tutorial/database-access",
+			Value:   "certs.sqlite",
+			EnvVars: []string{"DB_CONN"},
+		},
+	}
+
+	ServerFlags = append(CertStorageFlags, []cli.Flag{
+		// #############
+		// ### Forge ###
+		// #############
+		// ForgeRoot specifies the root URL of the Forge instance, without a trailing slash.
+		&cli.StringFlag{
+			Name:    "forge-root",
+			Aliases: []string{"gitea-root"},
+			Usage:   "specifies the root URL of the Forgejo/Gitea instance, without a trailing slash.",
+			EnvVars: []string{"FORGE_ROOT", "GITEA_ROOT"},
+		},
+		// ForgeApiToken specifies an api token for the Forge instance
+		&cli.StringFlag{
+			Name:    "forge-api-token",
+			Aliases: []string{"gitea-api-token"},
+			Usage:   "specifies an api token for the Forgejo/Gitea instance",
+			EnvVars: []string{"FORGE_API_TOKEN", "GITEA_API_TOKEN"},
+		},
+		&cli.BoolFlag{
+			Name:    "enable-lfs-support",
+			Usage:   "enable lfs support, gitea must be version v1.17.0 or higher",
+			EnvVars: []string{"ENABLE_LFS_SUPPORT"},
+			Value:   false,
+		},
+		&cli.BoolFlag{
+			Name:    "enable-symlink-support",
+			Usage:   "follow symlinks if enabled, gitea must be version v1.18.0 or higher",
+			EnvVars: []string{"ENABLE_SYMLINK_SUPPORT"},
+			Value:   false,
+		},
+		&cli.StringFlag{
+			Name:    "default-mime-type",
+			Usage:   "specifies the default mime type for files that don't have a specific mime type.",
+			EnvVars: []string{"DEFAULT_MIME_TYPE"},
+			Value:   "application/octet-stream",
+		},
+		&cli.StringSliceFlag{
+			Name:    "forbidden-mime-types",
+			Usage:   "specifies the forbidden mime types. Use this flag multiple times for multiple mime types.",
+			EnvVars: []string{"FORBIDDEN_MIME_TYPES"},
+		},
+
+		// ###########################
+		// ### Page Server Domains ###
+		// ###########################
+		// MainDomainSuffix specifies the main domain (starting with a dot) for which subdomains shall be served as static
+		// pages, or used for comparison in CNAME lookups. Static pages can be accessed through
+		// https://{owner}.{MainDomain}[/{repo}], with repo defaulting to "pages".
+		&cli.StringFlag{
+			Name:    "pages-domain",
+			Usage:   "specifies the main domain (starting with a dot) for which subdomains shall be served as static pages",
+			EnvVars: []string{"PAGES_DOMAIN"},
+		},
+		// RawDomain specifies the domain from which raw repository content shall be served in the following format:
+		// https://{RawDomain}/{owner}/{repo}[/{branch|tag|commit}/{version}]/{filepath...}
+		// (set to []byte(nil) to disable raw content hosting)
+		&cli.StringFlag{
+			Name:    "raw-domain",
+			Usage:   "specifies the domain from which raw repository content shall be served, not set disable raw content hosting",
+			EnvVars: []string{"RAW_DOMAIN"},
+		},
+
+		// #########################
+		// ### Page Server Setup ###
+		// #########################
+		&cli.StringFlag{
+			Name:    "host",
+			Usage:   "specifies host of listening address",
+			EnvVars: []string{"HOST"},
+			Value:   "[::]",
+		},
+		&cli.UintFlag{
+			Name:    "port",
+			Usage:   "specifies the https port to listen to ssl requests",
+			EnvVars: []string{"PORT", "HTTPS_PORT"},
+			Value:   443,
+		},
+		&cli.UintFlag{
+			Name:    "http-port",
+			Usage:   "specifies the http port, you also have to enable http server via ENABLE_HTTP_SERVER=true",
+			EnvVars: []string{"HTTP_PORT"},
+			Value:   80,
+		},
+		&cli.BoolFlag{
+			Name:    "enable-http-server",
+			Usage:   "start a http server to redirect to https and respond to http acme challenges",
+			EnvVars: []string{"ENABLE_HTTP_SERVER"},
+			Value:   false,
+		},
+		&cli.BoolFlag{
+			Name:    "use-proxy-protocol",
+			Usage:   "use the proxy protocol",
+			EnvVars: []string{"USE_PROXY_PROTOCOL"},
+			Value:   false,
+		},
+
+		// Default branches to fetch assets from
+		&cli.StringSliceFlag{
+			Name:    "pages-branch",
+			Usage:   "define a branch to fetch assets from. Use this flag multiple times for multiple branches.",
+			EnvVars: []string{"PAGES_BRANCHES"},
+			Value:   cli.NewStringSlice("pages"),
+		},
+
+		&cli.StringSliceFlag{
+			Name:    "allowed-cors-domains",
+			Usage:   "specify allowed CORS domains. Use this flag multiple times for multiple domains.",
+			EnvVars: []string{"ALLOWED_CORS_DOMAINS"},
+		},
+		&cli.StringSliceFlag{
+			Name:    "blacklisted-paths",
+			Usage:   "return an error on these url paths.Use this flag multiple times for multiple paths.",
+			EnvVars: []string{"BLACKLISTED_PATHS"},
+		},
+
+		&cli.StringFlag{
+			Name:    "log-level",
+			Value:   "warn",
+			Usage:   "specify at which log level should be logged. Possible options: info, warn, error, fatal",
+			EnvVars: []string{"LOG_LEVEL"},
+		},
+		&cli.StringFlag{
+			Name:    "config-file",
+			Usage:   "specify the location of the config file",
+			Aliases: []string{"config"},
+			EnvVars: []string{"CONFIG_FILE"},
+		},
+
+		&cli.BoolFlag{
+			Name:    "enable-profiling",
+			Usage:   "enables the go http profiling endpoints",
+			EnvVars: []string{"ENABLE_PROFILING"},
+		},
+		&cli.StringFlag{
+			Name:    "profiling-address",
+			Usage:   "specify ip address and port the profiling server should listen on",
+			EnvVars: []string{"PROFILING_ADDRESS"},
+			Value:   "localhost:9999",
+		},
+
+		// ############################
+		// ### ACME Client Settings ###
+		// ############################
+		&cli.StringFlag{
+			Name:    "acme-api-endpoint",
+			EnvVars: []string{"ACME_API"},
+			Value:   "https://acme-v02.api.letsencrypt.org/directory",
+		},
+		&cli.StringFlag{
+			Name:    "acme-email",
+			EnvVars: []string{"ACME_EMAIL"},
+			Value:   "noreply@example.email",
+		},
+		&cli.BoolFlag{
+			Name: "acme-use-rate-limits",
+			// TODO: Usage
+			EnvVars: []string{"ACME_USE_RATE_LIMITS"},
+			Value:   true,
+		},
+		&cli.BoolFlag{
+			Name:    "acme-accept-terms",
+			Usage:   "To accept the ACME ToS",
+			EnvVars: []string{"ACME_ACCEPT_TERMS"},
+		},
+		&cli.StringFlag{
+			Name:    "acme-eab-kid",
+			Usage:   "Register the current account to the ACME server with external binding.",
+			EnvVars: []string{"ACME_EAB_KID"},
+		},
+		&cli.StringFlag{
+			Name:    "acme-eab-hmac",
+			Usage:   "Register the current account to the ACME server with external binding.",
+			EnvVars: []string{"ACME_EAB_HMAC"},
+		},
+		&cli.StringFlag{
+			Name:    "dns-provider",
+			Usage:   "Use DNS-Challenge for main domain. Read more at: https://go-acme.github.io/lego/dns/",
+			EnvVars: []string{"DNS_PROVIDER"},
+		},
+		&cli.BoolFlag{
+			Name:    "no-dns-01",
+			Usage:   "Always use individual certificates instead of a DNS-01 wild card certificate",
+			EnvVars: []string{"NO_DNS_01"},
+		},
+		&cli.StringFlag{
+			Name:    "acme-account-config",
+			Usage:   "json file of acme account",
+			Value:   "acme-account.json",
+			EnvVars: []string{"ACME_ACCOUNT_CONFIG"},
+		},
+	}...)
+)

cli/setup.go

@@ -0,0 +1,39 @@
+package cli
+
+import (
+	"fmt"
+
+	"github.com/rs/zerolog/log"
+	"github.com/urfave/cli/v2"
+
+	"codeberg.org/codeberg/pages/server/database"
+	"codeberg.org/codeberg/pages/server/version"
+)
+
+func CreatePagesApp() *cli.App {
+	app := cli.NewApp()
+	app.Name = "pages-server"
+	app.Version = version.Version
+	app.Usage = "pages server"
+	app.Flags = ServerFlags
+	app.Commands = []*cli.Command{
+		Certs,
+	}
+
+	return app
+}
+
+func OpenCertDB(ctx *cli.Context) (certDB database.CertDB, closeFn func(), err error) {
+	certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn"))
+	if err != nil {
+		return nil, nil, fmt.Errorf("could not connect to database: %w", err)
+	}
+
+	closeFn = func() {
+		if err := certDB.Close(); err != nil {
+			log.Error().Err(err)
+		}
+	}
+
+	return certDB, closeFn, nil
+}

cmd/certs.go

@@ -1,128 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"time"
-
-	"github.com/rs/zerolog"
-	"github.com/rs/zerolog/log"
-	"github.com/urfave/cli/v2"
-
-	"codeberg.org/codeberg/pages/server/database"
-)
-
-var Certs = &cli.Command{
-	Name:  "certs",
-	Usage: "manage certs manually",
-	Subcommands: []*cli.Command{
-		{
-			Name:   "list",
-			Usage:  "list all certificates in the database",
-			Action: listCerts,
-		},
-		{
-			Name:   "remove",
-			Usage:  "remove a certificate from the database",
-			Action: removeCert,
-		},
-		{
-			Name:   "migrate",
-			Usage:  "migrate from \"pogreb\" driver to dbms driver",
-			Action: migrateCerts,
-		},
-	},
-	Flags: append(CertStorageFlags, []cli.Flag{
-		&cli.BoolFlag{
-			Name:    "verbose",
-			Usage:   "print trace info",
-			EnvVars: []string{"VERBOSE"},
-			Value:   false,
-		},
-	}...),
-}
-
-func migrateCerts(ctx *cli.Context) error {
-	dbType := ctx.String("db-type")
-	if dbType == "" {
-		dbType = "sqlite3"
-	}
-	dbConn := ctx.String("db-conn")
-	dbPogrebConn := ctx.String("db-pogreb")
-	verbose := ctx.Bool("verbose")
-
-	log.Level(zerolog.InfoLevel)
-	if verbose {
-		log.Level(zerolog.TraceLevel)
-	}
-
-	xormDB, err := database.NewXormDB(dbType, dbConn)
-	if err != nil {
-		return fmt.Errorf("could not connect to database: %w", err)
-	}
-	defer xormDB.Close()
-
-	pogrebDB, err := database.NewPogreb(dbPogrebConn)
-	if err != nil {
-		return fmt.Errorf("could not open database: %w", err)
-	}
-	defer pogrebDB.Close()
-
-	fmt.Printf("Start migration from \"%s\" to \"%s:%s\" ...\n", dbPogrebConn, dbType, dbConn)
-
-	certs, err := pogrebDB.Items(0, 0)
-	if err != nil {
-		return err
-	}
-
-	for _, cert := range certs {
-		if err := xormDB.Put(cert.Domain, cert.Raw()); err != nil {
-			return err
-		}
-	}
-
-	fmt.Println("... done")
-	return nil
-}
-
-func listCerts(ctx *cli.Context) error {
-	certDB, closeFn, err := openCertDB(ctx)
-	if err != nil {
-		return err
-	}
-	defer closeFn()
-
-	items, err := certDB.Items(0, 0)
-	if err != nil {
-		return err
-	}
-
-	fmt.Printf("Domain\tValidTill\n\n")
-	for _, cert := range items {
-		fmt.Printf("%s\t%s\n",
-			cert.Domain,
-			time.Unix(cert.ValidTill, 0).Format(time.RFC3339))
-	}
-	return nil
-}
-
-func removeCert(ctx *cli.Context) error {
-	if ctx.Args().Len() < 1 {
-		return fmt.Errorf("'certs remove' requires at least one domain as an argument")
-	}
-
-	domains := ctx.Args().Slice()
-
-	certDB, closeFn, err := openCertDB(ctx)
-	if err != nil {
-		return err
-	}
-	defer closeFn()
-
-	for _, domain := range domains {
-		fmt.Printf("Removing domain %s from the database...\n", domain)
-		if err := certDB.Delete(domain); err != nil {
-			return err
-		}
-	}
-	return nil
-}

cmd/flags.go

@@ -1,155 +0,0 @@
-package cmd
-
-import (
-	"github.com/urfave/cli/v2"
-)
-
-var (
-	CertStorageFlags = []cli.Flag{
-		&cli.StringFlag{
-			// TODO: remove in next version
-			// DEPRICATED
-			Name:    "db-pogreb",
-			Value:   "key-database.pogreb",
-			EnvVars: []string{"DB_POGREB"},
-		},
-		&cli.StringFlag{
-			Name:    "db-type",
-			Value:   "", // TODO: "sqlite3" in next version
-			EnvVars: []string{"DB_TYPE"},
-		},
-		&cli.StringFlag{
-			Name:    "db-conn",
-			Value:   "certs.sqlite",
-			EnvVars: []string{"DB_CONN"},
-		},
-	}
-
-	ServerFlags = append(CertStorageFlags, []cli.Flag{
-		// #############
-		// ### Gitea ###
-		// #############
-		// GiteaRoot specifies the root URL of the Gitea instance, without a trailing slash.
-		&cli.StringFlag{
-			Name:    "gitea-root",
-			Usage:   "specifies the root URL of the Gitea instance, without a trailing slash.",
-			EnvVars: []string{"GITEA_ROOT"},
-			Value:   "https://codeberg.org",
-		},
-		// GiteaApiToken specifies an api token for the Gitea instance
-		&cli.StringFlag{
-			Name:    "gitea-api-token",
-			Usage:   "specifies an api token for the Gitea instance",
-			EnvVars: []string{"GITEA_API_TOKEN"},
-			Value:   "",
-		},
-		&cli.BoolFlag{
-			Name:    "enable-lfs-support",
-			Usage:   "enable lfs support, require gitea >= v1.17.0 as backend",
-			EnvVars: []string{"ENABLE_LFS_SUPPORT"},
-			Value:   true,
-		},
-		&cli.BoolFlag{
-			Name:    "enable-symlink-support",
-			Usage:   "follow symlinks if enabled, require gitea >= v1.18.0 as backend",
-			EnvVars: []string{"ENABLE_SYMLINK_SUPPORT"},
-			Value:   true,
-		},
-
-		// ###########################
-		// ### Page Server Domains ###
-		// ###########################
-		// MainDomainSuffix specifies the main domain (starting with a dot) for which subdomains shall be served as static
-		// pages, or used for comparison in CNAME lookups. Static pages can be accessed through
-		// https://{owner}.{MainDomain}[/{repo}], with repo defaulting to "pages".
-		&cli.StringFlag{
-			Name:    "pages-domain",
-			Usage:   "specifies the main domain (starting with a dot) for which subdomains shall be served as static pages",
-			EnvVars: []string{"PAGES_DOMAIN"},
-			Value:   "codeberg.page",
-		},
-		// RawDomain specifies the domain from which raw repository content shall be served in the following format:
-		// https://{RawDomain}/{owner}/{repo}[/{branch|tag|commit}/{version}]/{filepath...}
-		// (set to []byte(nil) to disable raw content hosting)
-		&cli.StringFlag{
-			Name:    "raw-domain",
-			Usage:   "specifies the domain from which raw repository content shall be served, not set disable raw content hosting",
-			EnvVars: []string{"RAW_DOMAIN"},
-			Value:   "raw.codeberg.page",
-		},
-		// RawInfoPage will be shown (with a redirect) when trying to access RawDomain directly (or without owner/repo/path).
-		&cli.StringFlag{
-			Name:    "raw-info-page",
-			Usage:   "will be shown (with a redirect) when trying to access $RAW_DOMAIN directly (or without owner/repo/path)",
-			EnvVars: []string{"RAW_INFO_PAGE"},
-			Value:   "https://docs.codeberg.org/codeberg-pages/raw-content/",
-		},
-
-		// #########################
-		// ### Page Server Setup ###
-		// #########################
-		&cli.StringFlag{
-			Name:    "host",
-			Usage:   "specifies host of listening address",
-			EnvVars: []string{"HOST"},
-			Value:   "[::]",
-		},
-		&cli.StringFlag{
-			Name:    "port",
-			Usage:   "specifies port of listening address",
-			EnvVars: []string{"PORT"},
-			Value:   "443",
-		},
-		&cli.BoolFlag{
-			Name: "enable-http-server",
-			// TODO: desc
-			EnvVars: []string{"ENABLE_HTTP_SERVER"},
-		},
-		&cli.StringFlag{
-			Name:    "log-level",
-			Value:   "warn",
-			Usage:   "specify at which log level should be logged. Possible options: info, warn, error, fatal",
-			EnvVars: []string{"LOG_LEVEL"},
-		},
-
-		// ############################
-		// ### ACME Client Settings ###
-		// ############################
-		&cli.StringFlag{
-			Name:    "acme-api-endpoint",
-			EnvVars: []string{"ACME_API"},
-			Value:   "https://acme-v02.api.letsencrypt.org/directory",
-		},
-		&cli.StringFlag{
-			Name:    "acme-email",
-			EnvVars: []string{"ACME_EMAIL"},
-			Value:   "noreply@example.email",
-		},
-		&cli.BoolFlag{
-			Name: "acme-use-rate-limits",
-			// TODO: Usage
-			EnvVars: []string{"ACME_USE_RATE_LIMITS"},
-			Value:   true,
-		},
-		&cli.BoolFlag{
-			Name: "acme-accept-terms",
-			// TODO: Usage
-			EnvVars: []string{"ACME_ACCEPT_TERMS"},
-		},
-		&cli.StringFlag{
-			Name: "acme-eab-kid",
-			// TODO: Usage
-			EnvVars: []string{"ACME_EAB_KID"},
-		},
-		&cli.StringFlag{
-			Name: "acme-eab-hmac",
-			// TODO: Usage
-			EnvVars: []string{"ACME_EAB_HMAC"},
-		},
-		&cli.StringFlag{
-			Name: "dns-provider",
-			// TODO: Usage
-			EnvVars: []string{"DNS_PROVIDER"},
-		},
-	}...)
-)

cmd/main.go

@@ -1,151 +0,0 @@
-package cmd
-
-import (
-	"context"
-	"crypto/tls"
-	"errors"
-	"fmt"
-	"net"
-	"net/http"
-	"os"
-	"strings"
-	"time"
-
-	"github.com/rs/zerolog"
-	"github.com/rs/zerolog/log"
-	"github.com/urfave/cli/v2"
-
-	"codeberg.org/codeberg/pages/server"
-	"codeberg.org/codeberg/pages/server/cache"
-	"codeberg.org/codeberg/pages/server/certificates"
-	"codeberg.org/codeberg/pages/server/gitea"
-	"codeberg.org/codeberg/pages/server/handler"
-)
-
-// AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed.
-// TODO: make it a flag
-var AllowedCorsDomains = []string{
-	"fonts.codeberg.org",
-	"design.codeberg.org",
-}
-
-// BlacklistedPaths specifies forbidden path prefixes for all Codeberg Pages.
-// TODO: Make it a flag too
-var BlacklistedPaths = []string{
-	"/.well-known/acme-challenge/",
-}
-
-// Serve sets up and starts the web server.
-func Serve(ctx *cli.Context) error {
-	// Initialize the logger.
-	logLevel, err := zerolog.ParseLevel(ctx.String("log-level"))
-	if err != nil {
-		return err
-	}
-	log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger().Level(logLevel)
-
-	giteaRoot := ctx.String("gitea-root")
-	giteaAPIToken := ctx.String("gitea-api-token")
-	rawDomain := ctx.String("raw-domain")
-	mainDomainSuffix := ctx.String("pages-domain")
-	rawInfoPage := ctx.String("raw-info-page")
-	listeningAddress := fmt.Sprintf("%s:%s", ctx.String("host"), ctx.String("port"))
-	enableHTTPServer := ctx.Bool("enable-http-server")
-
-	acmeAPI := ctx.String("acme-api-endpoint")
-	acmeMail := ctx.String("acme-email")
-	acmeUseRateLimits := ctx.Bool("acme-use-rate-limits")
-	acmeAcceptTerms := ctx.Bool("acme-accept-terms")
-	acmeEabKID := ctx.String("acme-eab-kid")
-	acmeEabHmac := ctx.String("acme-eab-hmac")
-	dnsProvider := ctx.String("dns-provider")
-	if (!acmeAcceptTerms || dnsProvider == "") && acmeAPI != "https://acme.mock.directory" {
-		return errors.New("you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER, unless $ACME_API is set to https://acme.mock.directory")
-	}
-
-	allowedCorsDomains := AllowedCorsDomains
-	if rawDomain != "" {
-		allowedCorsDomains = append(allowedCorsDomains, rawDomain)
-	}
-
-	// Make sure MainDomain has a trailing dot
-	if !strings.HasPrefix(mainDomainSuffix, ".") {
-		mainDomainSuffix = "." + mainDomainSuffix
-	}
-
-	// Init ssl cert database
-	certDB, closeFn, err := openCertDB(ctx)
-	if err != nil {
-		return err
-	}
-	defer closeFn()
-
-	keyCache := cache.NewKeyValueCache()
-	challengeCache := cache.NewKeyValueCache()
-	// canonicalDomainCache stores canonical domains
-	canonicalDomainCache := cache.NewKeyValueCache()
-	// dnsLookupCache stores DNS lookups for custom domains
-	dnsLookupCache := cache.NewKeyValueCache()
-	// clientResponseCache stores responses from the Gitea server
-	clientResponseCache := cache.NewKeyValueCache()
-
-	giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken, clientResponseCache, ctx.Bool("enable-symlink-support"), ctx.Bool("enable-lfs-support"))
-	if err != nil {
-		return fmt.Errorf("could not create new gitea client: %v", err)
-	}
-
-	// Create handler based on settings
-	httpsHandler := handler.Handler(mainDomainSuffix, rawDomain,
-		giteaClient,
-		rawInfoPage,
-		BlacklistedPaths, allowedCorsDomains,
-		dnsLookupCache, canonicalDomainCache)
-
-	httpHandler := server.SetupHTTPACMEChallengeServer(challengeCache)
-
-	// Setup listener and TLS
-	log.Info().Msgf("Listening on https://%s", listeningAddress)
-	listener, err := net.Listen("tcp", listeningAddress)
-	if err != nil {
-		return fmt.Errorf("couldn't create listener: %v", err)
-	}
-
-	listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix,
-		giteaClient,
-		dnsProvider,
-		acmeUseRateLimits,
-		keyCache, challengeCache, dnsLookupCache, canonicalDomainCache,
-		certDB))
-
-	acmeConfig, err := certificates.SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, acmeAcceptTerms)
-	if err != nil {
-		return err
-	}
-
-	if err := certificates.SetupCertificates(mainDomainSuffix, dnsProvider, acmeConfig, acmeUseRateLimits, enableHTTPServer, challengeCache, certDB); err != nil {
-		return err
-	}
-
-	interval := 12 * time.Hour
-	certMaintainCtx, cancelCertMaintain := context.WithCancel(context.Background())
-	defer cancelCertMaintain()
-	go certificates.MaintainCertDB(certMaintainCtx, interval, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB)
-
-	if enableHTTPServer {
-		go func() {
-			log.Info().Msg("Start HTTP server listening on :80")
-			err := http.ListenAndServe("[::]:80", httpHandler)
-			if err != nil {
-				log.Panic().Err(err).Msg("Couldn't start HTTP fastServer")
-			}
-		}()
-	}
-
-	// Start the web fastServer
-	log.Info().Msgf("Start listening on %s", listener.Addr())
-	if err := http.Serve(listener, httpsHandler); err != nil {
-		log.Panic().Err(err).Msg("Couldn't start fastServer")
-	}
-
-	return nil
-}

cmd/setup.go

@@ -1,45 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-
-	"github.com/rs/zerolog/log"
-	"github.com/urfave/cli/v2"
-
-	"codeberg.org/codeberg/pages/server/database"
-)
-
-func openCertDB(ctx *cli.Context) (certDB database.CertDB, closeFn func(), err error) {
-	if ctx.String("db-type") != "" {
-		log.Trace().Msg("use xorm mode")
-		certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn"))
-		if err != nil {
-			return nil, nil, fmt.Errorf("could not connect to database: %w", err)
-		}
-	} else {
-		// TODO: remove in next version
-		fmt.Println(`
-######################
-## W A R N I N G !!! #
-######################
-
-You use "pogreb" witch is deprecated and will be removed in the next version.
-Please switch to sqlite, mysql or postgres !!!
-
-The simplest way is, to use './pages certs migrate' and set environment var DB_TYPE to 'sqlite' on next start.`)
-		log.Error().Msg("depricated \"pogreb\" used\n")
-
-		certDB, err = database.NewPogreb(ctx.String("db-pogreb"))
-		if err != nil {
-			return nil, nil, fmt.Errorf("could not create database: %w", err)
-		}
-	}
-
-	closeFn = func() {
-		if err := certDB.Close(); err != nil {
-			log.Error().Err(err)
-		}
-	}
-
-	return certDB, closeFn, nil
-}

config/assets/test_config.toml

@@ -0,0 +1,33 @@
+logLevel = 'trace'
+
+[server]
+host = '127.0.0.1'
+port = 443
+httpPort = 80
+httpServerEnabled = true
+mainDomain = 'codeberg.page'
+rawDomain = 'raw.codeberg.page'
+allowedCorsDomains = ['fonts.codeberg.org', 'design.codeberg.org']
+blacklistedPaths = ['do/not/use']
+
+[forge]
+root = 'https://codeberg.org'
+token = 'XXXXXXXX'
+lfsEnabled = true
+followSymlinks = true
+defaultMimeType = "application/wasm"
+forbiddenMimeTypes = ["text/html"]
+
+[database]
+type = 'sqlite'
+conn = 'certs.sqlite'
+
+[ACME]
+email = 'a@b.c'
+apiEndpoint = 'https://example.com'
+acceptTerms = false
+useRateLimits = true
+eab_hmac = 'asdf'
+eab_kid = 'qwer'
+dnsProvider = 'cloudflare.com'
+accountConfigFile = 'nope'

config/config.go

@@ -0,0 +1,48 @@
+package config
+
+type Config struct {
+	LogLevel string `default:"warn"`
+	Server   ServerConfig
+	Forge    ForgeConfig
+	Database DatabaseConfig
+	ACME     ACMEConfig
+}
+
+type ServerConfig struct {
+	Host               string `default:"[::]"`
+	Port               uint16 `default:"443"`
+	HttpPort           uint16 `default:"80"`
+	HttpServerEnabled  bool   `default:"true"`
+	UseProxyProtocol   bool   `default:"false"`
+	MainDomain         string
+	RawDomain          string
+	PagesBranches      []string
+	AllowedCorsDomains []string
+	BlacklistedPaths   []string
+}
+
+type ForgeConfig struct {
+	Root               string
+	Token              string
+	LFSEnabled         bool   `default:"false"`
+	FollowSymlinks     bool   `default:"false"`
+	DefaultMimeType    string `default:"application/octet-stream"`
+	ForbiddenMimeTypes []string
+}
+
+type DatabaseConfig struct {
+	Type string `default:"sqlite3"`
+	Conn string `default:"certs.sqlite"`
+}
+
+type ACMEConfig struct {
+	Email             string
+	APIEndpoint       string `default:"https://acme-v02.api.letsencrypt.org/directory"`
+	AcceptTerms       bool   `default:"false"`
+	UseRateLimits     bool   `default:"true"`
+	EAB_HMAC          string
+	EAB_KID           string
+	DNSProvider       string
+	NoDNS01           bool   `default:"false"`
+	AccountConfigFile string `default:"acme-account.json"`
+}

config/setup.go

@@ -0,0 +1,154 @@
+package config
+
+import (
+	"os"
+	"path"
+
+	"github.com/creasty/defaults"
+	"github.com/pelletier/go-toml/v2"
+	"github.com/rs/zerolog/log"
+	"github.com/urfave/cli/v2"
+)
+
+var ALWAYS_BLACKLISTED_PATHS = []string{
+	"/.well-known/acme-challenge/",
+}
+
+func NewDefaultConfig() Config {
+	config := Config{}
+	if err := defaults.Set(&config); err != nil {
+		panic(err)
+	}
+
+	// defaults does not support setting arrays from strings
+	config.Server.PagesBranches = []string{"main", "master", "pages"}
+
+	return config
+}
+
+func ReadConfig(ctx *cli.Context) (*Config, error) {
+	config := NewDefaultConfig()
+	// if config is not given as argument return empty config
+	if !ctx.IsSet("config-file") {
+		return &config, nil
+	}
+
+	configFile := path.Clean(ctx.String("config-file"))
+
+	log.Debug().Str("config-file", configFile).Msg("reading config file")
+	content, err := os.ReadFile(configFile)
+	if err != nil {
+		return nil, err
+	}
+
+	err = toml.Unmarshal(content, &config)
+	return &config, err
+}
+
+func MergeConfig(ctx *cli.Context, config *Config) {
+	if ctx.IsSet("log-level") {
+		config.LogLevel = ctx.String("log-level")
+	}
+
+	mergeServerConfig(ctx, &config.Server)
+	mergeForgeConfig(ctx, &config.Forge)
+	mergeDatabaseConfig(ctx, &config.Database)
+	mergeACMEConfig(ctx, &config.ACME)
+}
+
+func mergeServerConfig(ctx *cli.Context, config *ServerConfig) {
+	if ctx.IsSet("host") {
+		config.Host = ctx.String("host")
+	}
+	if ctx.IsSet("port") {
+		config.Port = uint16(ctx.Uint("port"))
+	}
+	if ctx.IsSet("http-port") {
+		config.HttpPort = uint16(ctx.Uint("http-port"))
+	}
+	if ctx.IsSet("enable-http-server") {
+		config.HttpServerEnabled = ctx.Bool("enable-http-server")
+	}
+	if ctx.IsSet("use-proxy-protocol") {
+		config.UseProxyProtocol = ctx.Bool("use-proxy-protocol")
+	}
+
+	if ctx.IsSet("pages-domain") {
+		config.MainDomain = ctx.String("pages-domain")
+	}
+	if ctx.IsSet("raw-domain") {
+		config.RawDomain = ctx.String("raw-domain")
+	}
+	if ctx.IsSet("pages-branch") {
+		config.PagesBranches = ctx.StringSlice("pages-branch")
+	}
+	if ctx.IsSet("allowed-cors-domains") {
+		config.AllowedCorsDomains = ctx.StringSlice("allowed-cors-domains")
+	}
+	if ctx.IsSet("blacklisted-paths") {
+		config.BlacklistedPaths = ctx.StringSlice("blacklisted-paths")
+	}
+
+	// add the paths that should always be blacklisted
+	config.BlacklistedPaths = append(config.BlacklistedPaths, ALWAYS_BLACKLISTED_PATHS...)
+}
+
+func mergeForgeConfig(ctx *cli.Context, config *ForgeConfig) {
+	if ctx.IsSet("forge-root") {
+		config.Root = ctx.String("forge-root")
+	}
+	if ctx.IsSet("forge-api-token") {
+		config.Token = ctx.String("forge-api-token")
+	}
+	if ctx.IsSet("enable-lfs-support") {
+		config.LFSEnabled = ctx.Bool("enable-lfs-support")
+	}
+	if ctx.IsSet("enable-symlink-support") {
+		config.FollowSymlinks = ctx.Bool("enable-symlink-support")
+	}
+	if ctx.IsSet("default-mime-type") {
+		config.DefaultMimeType = ctx.String("default-mime-type")
+	}
+	if ctx.IsSet("forbidden-mime-types") {
+		config.ForbiddenMimeTypes = ctx.StringSlice("forbidden-mime-types")
+	}
+}
+
+func mergeDatabaseConfig(ctx *cli.Context, config *DatabaseConfig) {
+	if ctx.IsSet("db-type") {
+		config.Type = ctx.String("db-type")
+	}
+	if ctx.IsSet("db-conn") {
+		config.Conn = ctx.String("db-conn")
+	}
+}
+
+func mergeACMEConfig(ctx *cli.Context, config *ACMEConfig) {
+	if ctx.IsSet("acme-email") {
+		config.Email = ctx.String("acme-email")
+	}
+	if ctx.IsSet("acme-api-endpoint") {
+		config.APIEndpoint = ctx.String("acme-api-endpoint")
+	}
+	if ctx.IsSet("acme-accept-terms") {
+		config.AcceptTerms = ctx.Bool("acme-accept-terms")
+	}
+	if ctx.IsSet("acme-use-rate-limits") {
+		config.UseRateLimits = ctx.Bool("acme-use-rate-limits")
+	}
+	if ctx.IsSet("acme-eab-hmac") {
+		config.EAB_HMAC = ctx.String("acme-eab-hmac")
+	}
+	if ctx.IsSet("acme-eab-kid") {
+		config.EAB_KID = ctx.String("acme-eab-kid")
+	}
+	if ctx.IsSet("dns-provider") {
+		config.DNSProvider = ctx.String("dns-provider")
+	}
+	if ctx.IsSet("no-dns-01") {
+		config.NoDNS01 = ctx.Bool("no-dns-01")
+	}
+	if ctx.IsSet("acme-account-config") {
+		config.AccountConfigFile = ctx.String("acme-account-config")
+	}
+}

config/setup_test.go

@@ -0,0 +1,630 @@
+package config
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"github.com/pelletier/go-toml/v2"
+	"github.com/stretchr/testify/assert"
+	"github.com/urfave/cli/v2"
+
+	cmd "codeberg.org/codeberg/pages/cli"
+)
+
+func runApp(t *testing.T, fn func(*cli.Context) error, args []string) {
+	app := cmd.CreatePagesApp()
+	app.Action = fn
+
+	appCtx, appCancel := context.WithCancel(context.Background())
+	defer appCancel()
+
+	// os.Args always contains the binary name
+	args = append([]string{"testing"}, args...)
+
+	err := app.RunContext(appCtx, args)
+	assert.NoError(t, err)
+}
+
+// fixArrayFromCtx fixes the number of "changed" strings in a string slice according to the number of values in the context.
+// This is a workaround because the cli library has a bug where the number of values in the context gets bigger the more tests are run.
+func fixArrayFromCtx(ctx *cli.Context, key string, expected []string) []string {
+	if ctx.IsSet(key) {
+		ctxSlice := ctx.StringSlice(key)
+
+		if len(ctxSlice) > 1 {
+			for i := 1; i < len(ctxSlice); i++ {
+				expected = append([]string{"changed"}, expected...)
+			}
+		}
+	}
+
+	return expected
+}
+
+func readTestConfig() (*Config, error) {
+	content, err := os.ReadFile("assets/test_config.toml")
+	if err != nil {
+		return nil, err
+	}
+
+	expectedConfig := NewDefaultConfig()
+	err = toml.Unmarshal(content, &expectedConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	return &expectedConfig, nil
+}
+
+func TestReadConfigShouldReturnEmptyConfigWhenConfigArgEmpty(t *testing.T) {
+	runApp(
+		t,
+		func(ctx *cli.Context) error {
+			cfg, err := ReadConfig(ctx)
+			expected := NewDefaultConfig()
+			assert.Equal(t, &expected, cfg)
+
+			return err
+		},
+		[]string{},
+	)
+}
+
+func TestReadConfigShouldReturnConfigFromFileWhenConfigArgPresent(t *testing.T) {
+	runApp(
+		t,
+		func(ctx *cli.Context) error {
+			cfg, err := ReadConfig(ctx)
+			if err != nil {
+				return err
+			}
+
+			expectedConfig, err := readTestConfig()
+			if err != nil {
+				return err
+			}
+
+			assert.Equal(t, expectedConfig, cfg)
+
+			return nil
+		},
+		[]string{"--config-file", "assets/test_config.toml"},
+	)
+}
+
+func TestValuesReadFromConfigFileShouldBeOverwrittenByArgs(t *testing.T) {
+	runApp(
+		t,
+		func(ctx *cli.Context) error {
+			cfg, err := ReadConfig(ctx)
+			if err != nil {
+				return err
+			}
+
+			MergeConfig(ctx, cfg)
+
+			expectedConfig, err := readTestConfig()
+			if err != nil {
+				return err
+			}
+
+			expectedConfig.LogLevel = "debug"
+			expectedConfig.Forge.Root = "not-codeberg.org"
+			expectedConfig.ACME.AcceptTerms = true
+			expectedConfig.Server.Host = "172.17.0.2"
+			expectedConfig.Server.BlacklistedPaths = append(expectedConfig.Server.BlacklistedPaths, ALWAYS_BLACKLISTED_PATHS...)
+
+			assert.Equal(t, expectedConfig, cfg)
+
+			return nil
+		},
+		[]string{
+			"--config-file", "assets/test_config.toml",
+			"--log-level", "debug",
+			"--forge-root", "not-codeberg.org",
+			"--acme-accept-terms",
+			"--host", "172.17.0.2",
+		},
+	)
+}
+
+func TestMergeConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T) {
+	runApp(
+		t,
+		func(ctx *cli.Context) error {
+			cfg := &Config{
+				LogLevel: "original",
+				Server: ServerConfig{
+					Host:               "original",
+					Port:               8080,
+					HttpPort:           80,
+					HttpServerEnabled:  false,
+					MainDomain:         "original",
+					RawDomain:          "original",
+					PagesBranches:      []string{"original"},
+					AllowedCorsDomains: []string{"original"},
+					BlacklistedPaths:   []string{"original"},
+				},
+				Forge: ForgeConfig{
+					Root:               "original",
+					Token:              "original",
+					LFSEnabled:         false,
+					FollowSymlinks:     false,
+					DefaultMimeType:    "original",
+					ForbiddenMimeTypes: []string{"original"},
+				},
+				Database: DatabaseConfig{
+					Type: "original",
+					Conn: "original",
+				},
+				ACME: ACMEConfig{
+					Email:             "original",
+					APIEndpoint:       "original",
+					AcceptTerms:       false,
+					UseRateLimits:     false,
+					EAB_HMAC:          "original",
+					EAB_KID:           "original",
+					DNSProvider:       "original",
+					NoDNS01:           false,
+					AccountConfigFile: "original",
+				},
+			}
+
+			MergeConfig(ctx, cfg)
+
+			expectedConfig := &Config{
+				LogLevel: "changed",
+				Server: ServerConfig{
+					Host:               "changed",
+					Port:               8443,
+					HttpPort:           443,
+					HttpServerEnabled:  true,
+					MainDomain:         "changed",
+					RawDomain:          "changed",
+					PagesBranches:      []string{"changed"},
+					AllowedCorsDomains: []string{"changed"},
+					BlacklistedPaths:   append([]string{"changed"}, ALWAYS_BLACKLISTED_PATHS...),
+				},
+				Forge: ForgeConfig{
+					Root:               "changed",
+					Token:              "changed",
+					LFSEnabled:         true,
+					FollowSymlinks:     true,
+					DefaultMimeType:    "changed",
+					ForbiddenMimeTypes: []string{"changed"},
+				},
+				Database: DatabaseConfig{
+					Type: "changed",
+					Conn: "changed",
+				},
+				ACME: ACMEConfig{
+					Email:             "changed",
+					APIEndpoint:       "changed",
+					AcceptTerms:       true,
+					UseRateLimits:     true,
+					EAB_HMAC:          "changed",
+					EAB_KID:           "changed",
+					DNSProvider:       "changed",
+					NoDNS01:           true,
+					AccountConfigFile: "changed",
+				},
+			}
+
+			assert.Equal(t, expectedConfig, cfg)
+
+			return nil
+		},
+		[]string{
+			"--log-level", "changed",
+			// Server
+			"--pages-domain", "changed",
+			"--raw-domain", "changed",
+			"--allowed-cors-domains", "changed",
+			"--blacklisted-paths", "changed",
+			"--pages-branch", "changed",
+			"--host", "changed",
+			"--port", "8443",
+			"--http-port", "443",
+			"--enable-http-server",
+			// Forge
+			"--forge-root", "changed",
+			"--forge-api-token", "changed",
+			"--enable-lfs-support",
+			"--enable-symlink-support",
+			"--default-mime-type", "changed",
+			"--forbidden-mime-types", "changed",
+			// Database
+			"--db-type", "changed",
+			"--db-conn", "changed",
+			// ACME
+			"--acme-email", "changed",
+			"--acme-api-endpoint", "changed",
+			"--acme-accept-terms",
+			"--acme-use-rate-limits",
+			"--acme-eab-hmac", "changed",
+			"--acme-eab-kid", "changed",
+			"--dns-provider", "changed",
+			"--no-dns-01",
+			"--acme-account-config", "changed",
+		},
+	)
+}
+
+func TestMergeServerConfigShouldAddDefaultBlacklistedPathsToBlacklistedPaths(t *testing.T) {
+	runApp(
+		t,
+		func(ctx *cli.Context) error {
+			cfg := &ServerConfig{}
+			mergeServerConfig(ctx, cfg)
+
+			expected := ALWAYS_BLACKLISTED_PATHS
+			assert.Equal(t, expected, cfg.BlacklistedPaths)
+
+			return nil
+		},
+		[]string{},
+	)
+}
+
+func TestMergeServerConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T) {
+	for range []uint8{0, 1} {
+		runApp(
+			t,
+			func(ctx *cli.Context) error {
+				cfg := &ServerConfig{
+					Host:               "original",
+					Port:               8080,
+					HttpPort:           80,
+					HttpServerEnabled:  false,
+					MainDomain:         "original",
+					RawDomain:          "original",
+					AllowedCorsDomains: []string{"original"},
+					BlacklistedPaths:   []string{"original"},
+				}
+
+				mergeServerConfig(ctx, cfg)
+
+				expectedConfig := &ServerConfig{
+					Host:               "changed",
+					Port:               8443,
+					HttpPort:           443,
+					HttpServerEnabled:  true,
+					MainDomain:         "changed",
+					RawDomain:          "changed",
+					AllowedCorsDomains: fixArrayFromCtx(ctx, "allowed-cors-domains", []string{"changed"}),
+					BlacklistedPaths:   fixArrayFromCtx(ctx, "blacklisted-paths", append([]string{"changed"}, ALWAYS_BLACKLISTED_PATHS...)),
+				}
+
+				assert.Equal(t, expectedConfig, cfg)
+
+				return nil
+			},
+			[]string{
+				"--pages-domain", "changed",
+				"--raw-domain", "changed",
+				"--allowed-cors-domains", "changed",
+				"--blacklisted-paths", "changed",
+				"--host", "changed",
+				"--port", "8443",
+				"--http-port", "443",
+				"--enable-http-server",
+			},
+		)
+	}
+}
+
+func TestMergeServerConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgExists(t *testing.T) {
+	type testValuePair struct {
+		args     []string
+		callback func(*ServerConfig)
+	}
+	testValuePairs := []testValuePair{
+		{args: []string{"--host", "changed"}, callback: func(sc *ServerConfig) { sc.Host = "changed" }},
+		{args: []string{"--port", "8443"}, callback: func(sc *ServerConfig) { sc.Port = 8443 }},
+		{args: []string{"--http-port", "443"}, callback: func(sc *ServerConfig) { sc.HttpPort = 443 }},
+		{args: []string{"--enable-http-server"}, callback: func(sc *ServerConfig) { sc.HttpServerEnabled = true }},
+		{args: []string{"--pages-domain", "changed"}, callback: func(sc *ServerConfig) { sc.MainDomain = "changed" }},
+		{args: []string{"--raw-domain", "changed"}, callback: func(sc *ServerConfig) { sc.RawDomain = "changed" }},
+		{args: []string{"--pages-branch", "changed"}, callback: func(sc *ServerConfig) { sc.PagesBranches = []string{"changed"} }},
+		{args: []string{"--allowed-cors-domains", "changed"}, callback: func(sc *ServerConfig) { sc.AllowedCorsDomains = []string{"changed"} }},
+		{args: []string{"--blacklisted-paths", "changed"}, callback: func(sc *ServerConfig) { sc.BlacklistedPaths = []string{"changed"} }},
+	}
+
+	for _, pair := range testValuePairs {
+		runApp(
+			t,
+			func(ctx *cli.Context) error {
+				cfg := ServerConfig{
+					Host:               "original",
+					Port:               8080,
+					HttpPort:           80,
+					HttpServerEnabled:  false,
+					MainDomain:         "original",
+					RawDomain:          "original",
+					PagesBranches:      []string{"original"},
+					AllowedCorsDomains: []string{"original"},
+					BlacklistedPaths:   []string{"original"},
+				}
+
+				expectedConfig := cfg
+				pair.callback(&expectedConfig)
+				expectedConfig.BlacklistedPaths = append(expectedConfig.BlacklistedPaths, ALWAYS_BLACKLISTED_PATHS...)
+
+				expectedConfig.PagesBranches = fixArrayFromCtx(ctx, "pages-branch", expectedConfig.PagesBranches)
+				expectedConfig.AllowedCorsDomains = fixArrayFromCtx(ctx, "allowed-cors-domains", expectedConfig.AllowedCorsDomains)
+				expectedConfig.BlacklistedPaths = fixArrayFromCtx(ctx, "blacklisted-paths", expectedConfig.BlacklistedPaths)
+
+				mergeServerConfig(ctx, &cfg)
+
+				assert.Equal(t, expectedConfig, cfg)
+
+				return nil
+			},
+			pair.args,
+		)
+	}
+}
+
+func TestMergeForgeConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T) {
+	runApp(
+		t,
+		func(ctx *cli.Context) error {
+			cfg := &ForgeConfig{
+				Root:               "original",
+				Token:              "original",
+				LFSEnabled:         false,
+				FollowSymlinks:     false,
+				DefaultMimeType:    "original",
+				ForbiddenMimeTypes: []string{"original"},
+			}
+
+			mergeForgeConfig(ctx, cfg)
+
+			expectedConfig := &ForgeConfig{
+				Root:               "changed",
+				Token:              "changed",
+				LFSEnabled:         true,
+				FollowSymlinks:     true,
+				DefaultMimeType:    "changed",
+				ForbiddenMimeTypes: fixArrayFromCtx(ctx, "forbidden-mime-types", []string{"changed"}),
+			}
+
+			assert.Equal(t, expectedConfig, cfg)
+
+			return nil
+		},
+		[]string{
+			"--forge-root", "changed",
+			"--forge-api-token", "changed",
+			"--enable-lfs-support",
+			"--enable-symlink-support",
+			"--default-mime-type", "changed",
+			"--forbidden-mime-types", "changed",
+		},
+	)
+}
+
+func TestMergeForgeConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgExists(t *testing.T) {
+	type testValuePair struct {
+		args     []string
+		callback func(*ForgeConfig)
+	}
+	testValuePairs := []testValuePair{
+		{args: []string{"--forge-root", "changed"}, callback: func(gc *ForgeConfig) { gc.Root = "changed" }},
+		{args: []string{"--forge-api-token", "changed"}, callback: func(gc *ForgeConfig) { gc.Token = "changed" }},
+		{args: []string{"--enable-lfs-support"}, callback: func(gc *ForgeConfig) { gc.LFSEnabled = true }},
+		{args: []string{"--enable-symlink-support"}, callback: func(gc *ForgeConfig) { gc.FollowSymlinks = true }},
+		{args: []string{"--default-mime-type", "changed"}, callback: func(gc *ForgeConfig) { gc.DefaultMimeType = "changed" }},
+		{args: []string{"--forbidden-mime-types", "changed"}, callback: func(gc *ForgeConfig) { gc.ForbiddenMimeTypes = []string{"changed"} }},
+	}
+
+	for _, pair := range testValuePairs {
+		runApp(
+			t,
+			func(ctx *cli.Context) error {
+				cfg := ForgeConfig{
+					Root:               "original",
+					Token:              "original",
+					LFSEnabled:         false,
+					FollowSymlinks:     false,
+					DefaultMimeType:    "original",
+					ForbiddenMimeTypes: []string{"original"},
+				}
+
+				expectedConfig := cfg
+				pair.callback(&expectedConfig)
+
+				mergeForgeConfig(ctx, &cfg)
+
+				expectedConfig.ForbiddenMimeTypes = fixArrayFromCtx(ctx, "forbidden-mime-types", expectedConfig.ForbiddenMimeTypes)
+
+				assert.Equal(t, expectedConfig, cfg)
+
+				return nil
+			},
+			pair.args,
+		)
+	}
+}
+
+func TestMergeForgeConfigShouldReplaceValuesGivenGiteaOptionsExist(t *testing.T) {
+	runApp(
+		t,
+		func(ctx *cli.Context) error {
+			cfg := &ForgeConfig{
+				Root:  "original",
+				Token: "original",
+			}
+
+			mergeForgeConfig(ctx, cfg)
+
+			expectedConfig := &ForgeConfig{
+				Root:  "changed",
+				Token: "changed",
+			}
+
+			assert.Equal(t, expectedConfig, cfg)
+
+			return nil
+		},
+		[]string{
+			"--gitea-root", "changed",
+			"--gitea-api-token", "changed",
+		},
+	)
+}
+
+func TestMergeDatabaseConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T) {
+	runApp(
+		t,
+		func(ctx *cli.Context) error {
+			cfg := &DatabaseConfig{
+				Type: "original",
+				Conn: "original",
+			}
+
+			mergeDatabaseConfig(ctx, cfg)
+
+			expectedConfig := &DatabaseConfig{
+				Type: "changed",
+				Conn: "changed",
+			}
+
+			assert.Equal(t, expectedConfig, cfg)
+
+			return nil
+		},
+		[]string{
+			"--db-type", "changed",
+			"--db-conn", "changed",
+		},
+	)
+}
+
+func TestMergeDatabaseConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgExists(t *testing.T) {
+	type testValuePair struct {
+		args     []string
+		callback func(*DatabaseConfig)
+	}
+	testValuePairs := []testValuePair{
+		{args: []string{"--db-type", "changed"}, callback: func(gc *DatabaseConfig) { gc.Type = "changed" }},
+		{args: []string{"--db-conn", "changed"}, callback: func(gc *DatabaseConfig) { gc.Conn = "changed" }},
+	}
+
+	for _, pair := range testValuePairs {
+		runApp(
+			t,
+			func(ctx *cli.Context) error {
+				cfg := DatabaseConfig{
+					Type: "original",
+					Conn: "original",
+				}
+
+				expectedConfig := cfg
+				pair.callback(&expectedConfig)
+
+				mergeDatabaseConfig(ctx, &cfg)
+
+				assert.Equal(t, expectedConfig, cfg)
+
+				return nil
+			},
+			pair.args,
+		)
+	}
+}
+
+func TestMergeACMEConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T) {
+	runApp(
+		t,
+		func(ctx *cli.Context) error {
+			cfg := &ACMEConfig{
+				Email:             "original",
+				APIEndpoint:       "original",
+				AcceptTerms:       false,
+				UseRateLimits:     false,
+				EAB_HMAC:          "original",
+				EAB_KID:           "original",
+				DNSProvider:       "original",
+				NoDNS01:           false,
+				AccountConfigFile: "original",
+			}
+
+			mergeACMEConfig(ctx, cfg)
+
+			expectedConfig := &ACMEConfig{
+				Email:             "changed",
+				APIEndpoint:       "changed",
+				AcceptTerms:       true,
+				UseRateLimits:     true,
+				EAB_HMAC:          "changed",
+				EAB_KID:           "changed",
+				DNSProvider:       "changed",
+				NoDNS01:           true,
+				AccountConfigFile: "changed",
+			}
+
+			assert.Equal(t, expectedConfig, cfg)
+
+			return nil
+		},
+		[]string{
+			"--acme-email", "changed",
+			"--acme-api-endpoint", "changed",
+			"--acme-accept-terms",
+			"--acme-use-rate-limits",
+			"--acme-eab-hmac", "changed",
+			"--acme-eab-kid", "changed",
+			"--dns-provider", "changed",
+			"--no-dns-01",
+			"--acme-account-config", "changed",
+		},
+	)
+}
+
+func TestMergeACMEConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgExists(t *testing.T) {
+	type testValuePair struct {
+		args     []string
+		callback func(*ACMEConfig)
+	}
+	testValuePairs := []testValuePair{
+		{args: []string{"--acme-email", "changed"}, callback: func(gc *ACMEConfig) { gc.Email = "changed" }},
+		{args: []string{"--acme-api-endpoint", "changed"}, callback: func(gc *ACMEConfig) { gc.APIEndpoint = "changed" }},
+		{args: []string{"--acme-accept-terms"}, callback: func(gc *ACMEConfig) { gc.AcceptTerms = true }},
+		{args: []string{"--acme-use-rate-limits"}, callback: func(gc *ACMEConfig) { gc.UseRateLimits = true }},
+		{args: []string{"--acme-eab-hmac", "changed"}, callback: func(gc *ACMEConfig) { gc.EAB_HMAC = "changed" }},
+		{args: []string{"--acme-eab-kid", "changed"}, callback: func(gc *ACMEConfig) { gc.EAB_KID = "changed" }},
+		{args: []string{"--dns-provider", "changed"}, callback: func(gc *ACMEConfig) { gc.DNSProvider = "changed" }},
+		{args: []string{"--no-dns-01"}, callback: func(gc *ACMEConfig) { gc.NoDNS01 = true }},
+		{args: []string{"--acme-account-config", "changed"}, callback: func(gc *ACMEConfig) { gc.AccountConfigFile = "changed" }},
+	}
+
+	for _, pair := range testValuePairs {
+		runApp(
+			t,
+			func(ctx *cli.Context) error {
+				cfg := ACMEConfig{
+					Email:             "original",
+					APIEndpoint:       "original",
+					AcceptTerms:       false,
+					UseRateLimits:     false,
+					EAB_HMAC:          "original",
+					EAB_KID:           "original",
+					DNSProvider:       "original",
+					AccountConfigFile: "original",
+				}
+
+				expectedConfig := cfg
+				pair.callback(&expectedConfig)
+
+				mergeACMEConfig(ctx, &cfg)
+
+				assert.Equal(t, expectedConfig, cfg)
+
+				return nil
+			},
+			pair.args,
+		)
+	}
+}

example_config.toml

@@ -0,0 +1,32 @@
+logLevel = 'debug'
+
+[server]
+host = '[::]'
+port = 443
+httpPort = 80
+httpServerEnabled = true
+mainDomain = 'codeberg.page'
+rawDomain = 'raw.codeberg.page'
+pagesBranches = ["pages"]
+allowedCorsDomains = []
+blacklistedPaths = []
+
+[forge]
+root = 'https://codeberg.org'
+token = 'ASDF1234'
+lfsEnabled = true
+followSymlinks = true
+
+[database]
+type = 'sqlite'
+conn = 'certs.sqlite'
+
+[ACME]
+email = 'noreply@example.email'
+apiEndpoint = 'https://acme-v02.api.letsencrypt.org/directory'
+acceptTerms = false
+useRateLimits = false
+eab_hmac = ''
+eab_kid = ''
+dnsProvider = ''
+accountConfigFile = 'acme-account.json'

examples/haproxy-sni/README.md

@@ -1,8 +1,9 @@
 # HAProxy with SNI & Host-based rules
 
-This is a proof of concept, enabling HAProxy to use *either* SNI to redirect to backends with their own HTTPS certificates (which are then fully exposed to the client; HAProxy only proxies on a TCP level in that case), *as well as* to terminate HTTPS and use the Host header to redirect to backends that use HTTP (or a new HTTPS connection).
+This is a proof of concept, enabling HAProxy to use _either_ SNI to redirect to backends with their own HTTPS certificates (which are then fully exposed to the client; HAProxy only proxies on a TCP level in that case), _as well as_ to terminate HTTPS and use the Host header to redirect to backends that use HTTP (or a new HTTPS connection).
 
 ## How it works
+
 1. The `http_redirect_frontend` is only there to listen on port 80 and redirect every request to HTTPS.
 2. The `https_sni_frontend` listens on port 443 and chooses a backend based on the SNI hostname of the TLS connection.
 3. The `https_termination_backend` passes all requests to a unix socket (using the plain TCP data).
@@ -11,6 +12,7 @@ This is a proof of concept, enabling HAProxy to use *either* SNI to redirect to
 In the example (see [haproxy.cfg](haproxy.cfg)), the `pages_backend` is listening via HTTPS and is providing its own HTTPS certificates, while the `gitea_backend` only provides HTTP.
 
 ## How to test
+
 ```bash
 docker-compose up &
 ./test.sh

examples/haproxy-sni/docker-compose.yml

@@ -0,0 +1,21 @@
+version: '3'
+services:
+  haproxy:
+    image: haproxy
+    ports: ['443:443']
+    volumes:
+      - ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
+      - ./dhparam.pem:/etc/ssl/dhparam.pem:ro
+      - ./haproxy-certificates:/etc/ssl/private/haproxy:ro
+    cap_add:
+      - NET_ADMIN
+  gitea:
+    image: caddy
+    volumes:
+      - ./gitea-www:/srv:ro
+      - ./gitea.Caddyfile:/etc/caddy/Caddyfile:ro
+  pages:
+    image: caddy
+    volumes:
+      - ./pages-www:/srv:ro
+      - ./pages.Caddyfile:/etc/caddy/Caddyfile:ro

flake.lock

@@ -0,0 +1,71 @@
+{
+  "nodes": {
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "type": "github"
+      },
+      "original": {
+        "id": "flake-utils",
+        "type": "indirect"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 0,
+        "narHash": "sha256-WFZDy4bG2RkkCQloIEG8BXEvzyKklFVJbAismOJsIp4=",
+        "path": "/nix/store/c77dsgfxjywplw8bk8s8jlkdsr7a1bi9-source",
+        "type": "path"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "type": "indirect"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs",
+        "systems": "systems_2"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_2": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "id": "systems",
+        "type": "indirect"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,27 @@
+{
+  outputs = {
+    self,
+    nixpkgs,
+    flake-utils,
+    systems,
+  }:
+    flake-utils.lib.eachSystem (import systems)
+    (system: let
+      pkgs = import nixpkgs {
+        inherit system;
+      };
+    in {
+      devShells.default = pkgs.mkShell {
+        buildInputs = with pkgs; [
+          glibc.static
+          go
+          gofumpt
+          golangci-lint
+          gopls
+          gotools
+          go-tools
+          sqlite-interactive
+        ];
+      };
+    });
+}

go.mod

@@ -1,135 +1,237 @@
 module codeberg.org/codeberg/pages
 
-go 1.20
+go 1.24.0
 
 require (
-	code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa
+	code.gitea.io/sdk/gitea v0.20.0
 	github.com/OrlovEvgeny/go-mcache v0.0.0-20200121124330-1a8195b34f3a
-	github.com/akrylysov/pogreb v0.10.1
-	github.com/go-acme/lego/v4 v4.5.3
-	github.com/go-sql-driver/mysql v1.6.0
-	github.com/joho/godotenv v1.4.0
-	github.com/lib/pq v1.10.7
-	github.com/mattn/go-sqlite3 v1.14.16
+	github.com/creasty/defaults v1.8.0
+	github.com/go-acme/lego/v4 v4.21.0
+	github.com/go-sql-driver/mysql v1.8.1
+	github.com/hashicorp/go-uuid v1.0.3
+	github.com/hashicorp/golang-lru/v2 v2.0.7
+	github.com/joho/godotenv v1.5.1
+	github.com/lib/pq v1.10.9
+	github.com/mattn/go-sqlite3 v1.14.24
+	github.com/microcosm-cc/bluemonday v1.0.27
+	github.com/pelletier/go-toml/v2 v2.2.3
+	github.com/pires/go-proxyproto v0.8.0
 	github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad
-	github.com/rs/zerolog v1.27.0
-	github.com/stretchr/testify v1.7.0
-	github.com/urfave/cli/v2 v2.3.0
-	xorm.io/xorm v1.3.2
+	github.com/rs/zerolog v1.33.0
+	github.com/stretchr/testify v1.10.0
+	github.com/urfave/cli/v2 v2.27.5
+	golang.org/x/exp v0.0.0-20250305212735-054e65f0b394
+	xorm.io/xorm v1.3.9
 )
 
 require (
-	cloud.google.com/go v0.54.0 // indirect
-	github.com/Azure/azure-sdk-for-go v32.4.0+incompatible // indirect
+	cloud.google.com/go/auth v0.14.0 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.7 // indirect
+	cloud.google.com/go/compute/metadata v0.6.0 // indirect
+	filippo.io/edwards25519 v1.1.0 // indirect
+	github.com/42wim/httpsig v1.2.2 // indirect
+	github.com/AdamSLevy/jsonrpc2/v14 v14.1.0 // indirect
+	github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dns/armdns v1.2.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.3.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.9.0 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
-	github.com/Azure/go-autorest/autorest v0.11.19 // indirect
-	github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect
-	github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 // indirect
-	github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 // indirect
+	github.com/Azure/go-autorest/autorest v0.11.29 // indirect
+	github.com/Azure/go-autorest/autorest/adal v0.9.24 // indirect
+	github.com/Azure/go-autorest/autorest/azure/auth v0.5.13 // indirect
+	github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect
 	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
 	github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
-	github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 // indirect
 	github.com/OpenDNS/vegadns2client v0.0.0-20180418235048-a3fa4a771d87 // indirect
-	github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1 // indirect
-	github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183 // indirect
-	github.com/aws/aws-sdk-go v1.39.0 // indirect
-	github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
-	github.com/cenkalti/backoff/v4 v4.1.1 // indirect
-	github.com/cloudflare/cloudflare-go v0.20.0 // indirect
+	github.com/akamai/AkamaiOPEN-edgegrid-golang v1.2.2 // indirect
+	github.com/aliyun/alibaba-cloud-sdk-go v1.63.83 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.33.0 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.29.0 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.53 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.28 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/lightsail v1.42.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/route53 v1.48.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.24.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.33.8 // indirect
+	github.com/aws/smithy-go v1.22.1 // indirect
+	github.com/aymerick/douceur v0.2.0 // indirect
+	github.com/benbjohnson/clock v1.3.5 // indirect
+	github.com/boombuler/barcode v1.0.2 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+	github.com/civo/civogo v0.3.92 // indirect
+	github.com/cloudflare/cloudflare-go v0.114.0 // indirect
 	github.com/cpu/goacmedns v0.1.1 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/davidmz/go-pageant v1.0.2 // indirect
-	github.com/deepmap/oapi-codegen v1.6.1 // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
-	github.com/dnsimple/dnsimple-go v0.70.1 // indirect
-	github.com/exoscale/egoscale v0.67.0 // indirect
+	github.com/dnsimple/dnsimple-go v1.7.0 // indirect
+	github.com/exoscale/egoscale/v3 v3.1.8 // indirect
 	github.com/fatih/structs v1.1.0 // indirect
-	github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect
-	github.com/go-errors/errors v1.0.1 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.8.0 // indirect
+	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.8 // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/go-errors/errors v1.5.1 // indirect
 	github.com/go-fed/httpsig v1.1.0 // indirect
-	github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48 // indirect
-	github.com/goccy/go-json v0.8.1 // indirect
-	github.com/gofrs/uuid v4.0.0+incompatible // indirect
-	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/go-jose/go-jose/v4 v4.0.4 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.24.0 // indirect
+	github.com/go-resty/resty/v2 v2.16.3 // indirect
+	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
+	github.com/goccy/go-json v0.10.4 // indirect
+	github.com/gofrs/flock v0.12.1 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.5.1 // indirect
+	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
-	github.com/google/uuid v1.3.0 // indirect
-	github.com/googleapis/gax-go/v2 v2.0.5 // indirect
-	github.com/gophercloud/gophercloud v0.16.0 // indirect
-	github.com/gophercloud/utils v0.0.0-20210216074907-f6de111f2eae // indirect
-	github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
-	github.com/hashicorp/go-retryablehttp v0.7.0 // indirect
-	github.com/hashicorp/go-version v1.6.0 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/s2a-go v0.1.9 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
+	github.com/googleapis/gax-go/v2 v2.14.1 // indirect
+	github.com/gophercloud/gophercloud v1.14.1 // indirect
+	github.com/gophercloud/utils v0.0.0-20231010081019-80377eca5d56 // indirect
+	github.com/gorilla/css v1.0.1 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
+	github.com/hashicorp/go-version v1.7.0 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/huaweicloud/huaweicloud-sdk-go-v3 v0.1.132 // indirect
 	github.com/iij/doapi v0.0.0-20190504054126-0bbf12d6d7df // indirect
 	github.com/infobloxopen/infoblox-go-client v1.1.1 // indirect
-	github.com/jarcoal/httpmock v1.0.6 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 // indirect
-	github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b // indirect
-	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
+	github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/labbsr0x/bindman-dns-webhook v1.0.2 // indirect
 	github.com/labbsr0x/goh v1.0.1 // indirect
-	github.com/linode/linodego v0.31.1 // indirect
-	github.com/liquidweb/go-lwApi v0.0.5 // indirect
-	github.com/liquidweb/liquidweb-cli v0.6.9 // indirect
-	github.com/liquidweb/liquidweb-go v1.6.3 // indirect
-	github.com/mattn/go-colorable v0.1.12 // indirect
-	github.com/mattn/go-isatty v0.0.14 // indirect
-	github.com/miekg/dns v1.1.43 // indirect
+	github.com/leodido/go-urn v1.4.0 // indirect
+	github.com/linode/linodego v1.46.0 // indirect
+	github.com/liquidweb/liquidweb-cli v0.7.0 // indirect
+	github.com/liquidweb/liquidweb-go v1.6.4 // indirect
+	github.com/magiconair/properties v1.8.9 // indirect
+	github.com/mattn/go-colorable v0.1.14 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/miekg/dns v1.1.62 // indirect
+	github.com/mimuret/golang-iij-dpf v0.9.1 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/mitchellh/mapstructure v1.4.1 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/namedotcom/go v0.0.0-20180403034216-08470befbe04 // indirect
-	github.com/nrdcg/auroradns v1.0.1 // indirect
-	github.com/nrdcg/desec v0.6.0 // indirect
+	github.com/nrdcg/auroradns v1.1.0 // indirect
+	github.com/nrdcg/bunny-go v0.0.0-20240207213615-dde5bf4577a3 // indirect
+	github.com/nrdcg/desec v0.10.0 // indirect
 	github.com/nrdcg/dnspod-go v0.4.0 // indirect
-	github.com/nrdcg/freemyip v0.2.0 // indirect
-	github.com/nrdcg/goinwx v0.8.1 // indirect
+	github.com/nrdcg/freemyip v0.3.0 // indirect
+	github.com/nrdcg/goinwx v0.10.0 // indirect
+	github.com/nrdcg/mailinabox v0.2.0 // indirect
 	github.com/nrdcg/namesilo v0.2.1 // indirect
-	github.com/nrdcg/porkbun v0.1.1 // indirect
-	github.com/oracle/oci-go-sdk v24.3.0+incompatible // indirect
-	github.com/ovh/go-ovh v1.1.0 // indirect
+	github.com/nrdcg/nodion v0.1.0 // indirect
+	github.com/nrdcg/porkbun v0.4.0 // indirect
+	github.com/nzdjb/go-metaname v1.0.0 // indirect
+	github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect
+	github.com/oracle/oci-go-sdk/v65 v65.81.2 // indirect
+	github.com/ovh/go-ovh v1.6.0 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
+	github.com/peterhellberg/link v1.2.0 // indirect
+	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/pquerna/otp v1.3.0 // indirect
-	github.com/russross/blackfriday/v2 v2.0.1 // indirect
-	github.com/sacloud/libsacloud v1.36.2 // indirect
-	github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210127161313-bd30bebeac4f // indirect
-	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
-	github.com/sirupsen/logrus v1.4.2 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/pquerna/otp v1.4.0 // indirect
+	github.com/regfish/regfish-dnsapi-go v0.1.1 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/sacloud/api-client-go v0.2.10 // indirect
+	github.com/sacloud/go-http v0.1.9 // indirect
+	github.com/sacloud/iaas-api-go v1.14.0 // indirect
+	github.com/sacloud/packages-go v0.0.11 // indirect
+	github.com/sagikazarmark/locafero v0.7.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30 // indirect
+	github.com/selectel/domains-go v1.1.0 // indirect
+	github.com/selectel/go-selvpcclient/v3 v3.2.1 // indirect
+	github.com/shopspring/decimal v1.4.0 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9 // indirect
-	github.com/softlayer/softlayer-go v1.0.3 // indirect
+	github.com/softlayer/softlayer-go v1.1.7 // indirect
 	github.com/softlayer/xmlrpc v0.0.0-20200409220501-5f089df7cb7e // indirect
-	github.com/spf13/cast v1.3.1 // indirect
-	github.com/stretchr/objx v0.3.0 // indirect
+	github.com/sony/gobreaker v1.0.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.12.0 // indirect
+	github.com/spf13/cast v1.7.1 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/spf13/viper v1.19.0 // indirect
+	github.com/stretchr/objx v0.5.2 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/syndtr/goleveldb v1.0.0 // indirect
-	github.com/transip/gotransip/v6 v6.6.1 // indirect
-	github.com/vinyldns/go-vinyldns v0.0.0-20200917153823-148a5f6b8f14 // indirect
-	github.com/vultr/govultr/v2 v2.7.1 // indirect
-	go.opencensus.io v0.22.3 // indirect
-	go.uber.org/ratelimit v0.0.0-20180316092928-c15da0234277 // indirect
-	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect
-	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
-	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
-	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
-	golang.org/x/text v0.3.6 // indirect
-	golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 // indirect
-	google.golang.org/api v0.20.0 // indirect
-	google.golang.org/appengine v1.6.5 // indirect
-	google.golang.org/genproto v0.0.0-20200305110556-506484158171 // indirect
-	google.golang.org/grpc v1.27.1 // indirect
-	google.golang.org/protobuf v1.26.0 // indirect
-	gopkg.in/ini.v1 v1.62.0 // indirect
-	gopkg.in/ns1/ns1-go.v2 v2.6.2 // indirect
-	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
+	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.1084 // indirect
+	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dnspod v1.0.1084 // indirect
+	github.com/tjfoc/gmsm v1.4.1 // indirect
+	github.com/transip/gotransip/v6 v6.26.0 // indirect
+	github.com/ultradns/ultradns-go-sdk v1.8.0-20241010134910-243eeec // indirect
+	github.com/vinyldns/go-vinyldns v0.9.16 // indirect
+	github.com/volcengine/volc-sdk-golang v1.0.193 // indirect
+	github.com/vultr/govultr/v3 v3.14.1 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
+	github.com/yandex-cloud/go-genproto v0.0.0-20241220122821-aeb3b05efd1c // indirect
+	github.com/yandex-cloud/go-sdk v0.0.0-20241220131134-2393e243c134 // indirect
+	go.mongodb.org/mongo-driver v1.17.2 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect
+	go.opentelemetry.io/otel v1.34.0 // indirect
+	go.opentelemetry.io/otel/metric v1.34.0 // indirect
+	go.opentelemetry.io/otel/trace v1.34.0 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/ratelimit v0.3.1 // indirect
+	golang.org/x/crypto v0.36.0 // indirect
+	golang.org/x/mod v0.24.0 // indirect
+	golang.org/x/net v0.37.0 // indirect
+	golang.org/x/oauth2 v0.25.0 // indirect
+	golang.org/x/sync v0.12.0 // indirect
+	golang.org/x/sys v0.31.0 // indirect
+	golang.org/x/text v0.23.0 // indirect
+	golang.org/x/time v0.9.0 // indirect
+	golang.org/x/tools v0.31.0 // indirect
+	google.golang.org/api v0.217.0 // indirect
+	google.golang.org/genproto v0.0.0-20250115164207-1a7da9e5054f // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect
+	google.golang.org/grpc v1.69.4 // indirect
+	google.golang.org/protobuf v1.36.3 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/ns1/ns1-go.v2 v2.13.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	xorm.io/builder v0.3.12 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/api v0.32.1 // indirect
+	k8s.io/apimachinery v0.32.1 // indirect
+	k8s.io/klog/v2 v2.130.1 // indirect
+	k8s.io/utils v0.0.0-20241210054802-24370beab758 // indirect
+	sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.5.0 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
+	xorm.io/builder v0.3.13 // indirect
 )

haproxy-sni/docker-compose.yml

@@ -1,22 +0,0 @@
-version: "3"
-services:
-  haproxy:
-    image: haproxy
-    ports: ["443:443"]
-    volumes:
-    - ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
-    - ./dhparam.pem:/etc/ssl/dhparam.pem:ro
-    - ./haproxy-certificates:/etc/ssl/private/haproxy:ro
-    cap_add:
-    - NET_ADMIN
-  gitea:
-    image: caddy
-    volumes:
-    - ./gitea-www:/srv:ro
-    - ./gitea.Caddyfile:/etc/caddy/Caddyfile:ro
-  pages:
-    image: caddy
-    volumes:
-    - ./pages-www:/srv:ro
-    - ./pages.Caddyfile:/etc/caddy/Caddyfile:ro
-

html/404.html

@@ -1,37 +0,0 @@
-<!doctype html>
-<html class="codeberg-design">
-<head>
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width">
-    <title>%status%</title>
-
-    <link rel="stylesheet" href="https://design.codeberg.org/design-kit/codeberg.css" />
-    <link href="https://fonts.codeberg.org/dist/inter/Inter%20Web/inter.css" rel="stylesheet" />
-    <link href="https://fonts.codeberg.org/dist/fontawesome5/css/all.min.css" rel="stylesheet" />
-
-    <style>
-        body {
-            margin: 0; padding: 1rem; box-sizing: border-box;
-            width: 100%; min-height: 100vh;
-            display: flex;
-            flex-direction: column;
-            align-items: center;
-            justify-content: center;
-        }
-    </style>
-</head>
-<body>
-    <i class="fa fa-search text-primary" style="font-size: 96px;"></i>
-    <h1 class="mb-0 text-primary">
-        Page not found!
-    </h1>
-    <h5 class="text-center" style="max-width: 25em;">
-        Sorry, but this page couldn't be found or is inaccessible (%status%).<br/>
-        We hope this isn't a problem on our end ;) - Make sure to check the <a href="https://docs.codeberg.org/codeberg-pages/troubleshooting/" target="_blank">troubleshooting section in the Docs</a>!
-    </h5>
-    <small class="text-muted">
-        <img src="https://design.codeberg.org/logo-kit/icon.svg" class="align-top">
-        Static pages made easy - <a href="https://codeberg.page">Codeberg Pages</a>
-    </small>
-</body>
-</html>

html/error.go

@@ -1,50 +0,0 @@
-package html
-
-import (
-	"html/template"
-	"net/http"
-	"strconv"
-	"strings"
-
-	"codeberg.org/codeberg/pages/server/context"
-)
-
-// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body,
-// with "%status%" and %message% replaced with the provided statusCode and msg
-func ReturnErrorPage(ctx *context.Context, msg string, statusCode int) {
-	ctx.RespWriter.Header().Set("Content-Type", "text/html; charset=utf-8")
-	ctx.RespWriter.WriteHeader(statusCode)
-
-	msg = generateResponse(msg, statusCode)
-
-	_, _ = ctx.RespWriter.Write([]byte(msg))
-}
-
-// TODO: use template engine
-func generateResponse(msg string, statusCode int) string {
-	if msg == "" {
-		msg = strings.ReplaceAll(NotFoundPage,
-			"%status%",
-			strconv.Itoa(statusCode)+" "+errorMessage(statusCode))
-	} else {
-		msg = strings.ReplaceAll(
-			strings.ReplaceAll(ErrorPage, "%message%", template.HTMLEscapeString(msg)),
-			"%status%",
-			http.StatusText(statusCode))
-	}
-
-	return msg
-}
-
-func errorMessage(statusCode int) string {
-	message := http.StatusText(statusCode)
-
-	switch statusCode {
-	case http.StatusMisdirectedRequest:
-		message += " - domain not specified in <code>.domains</code> file"
-	case http.StatusFailedDependency:
-		message += " - target repo/branch doesn't exist or is private"
-	}
-
-	return message
-}

html/error.html

@@ -1,38 +0,0 @@
-<!doctype html>
-<html class="codeberg-design">
-<head>
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width">
-    <title>%status%</title>
-
-    <link rel="stylesheet" href="https://design.codeberg.org/design-kit/codeberg.css" />
-    <link href="https://fonts.codeberg.org/dist/inter/Inter%20Web/inter.css" rel="stylesheet" />
-    <link href="https://fonts.codeberg.org/dist/fontawesome5/css/all.min.css" rel="stylesheet" />
-
-    <style>
-        body {
-            margin: 0; padding: 1rem; box-sizing: border-box;
-            width: 100%; min-height: 100vh;
-            display: flex;
-            flex-direction: column;
-            align-items: center;
-            justify-content: center;
-        }
-    </style>
-</head>
-<body>
-    <i class="fa fa-search text-primary" style="font-size: 96px;"></i>
-    <h1 class="mb-0 text-primary">
-        %status%!
-    </h1>
-    <h5 class="text-center" style="max-width: 25em;">
-        Sorry, but this page couldn't be served.<br/>
-        We got an <b>"%message%"</b><br/>
-        We hope this isn't a problem on our end ;) - Make sure to check the <a href="https://docs.codeberg.org/codeberg-pages/troubleshooting/" target="_blank">troubleshooting section in the Docs</a>!
-    </h5>
-    <small class="text-muted">
-        <img src="https://design.codeberg.org/logo-kit/icon.svg" class="align-top">
-        Static pages made easy - <a href="https://codeberg.page">Codeberg Pages</a>
-    </small>
-</body>
-</html>

html/error_test.go

@@ -1,38 +0,0 @@
-package html
-
-import (
-	"net/http"
-	"strings"
-	"testing"
-)
-
-func TestValidMessage(t *testing.T) {
-	testString := "requested blacklisted path"
-	statusCode := http.StatusForbidden
-
-	expected := strings.ReplaceAll(
-		strings.ReplaceAll(ErrorPage, "%message%", testString),
-		"%status%",
-		http.StatusText(statusCode))
-	actual := generateResponse(testString, statusCode)
-
-	if expected != actual {
-		t.Errorf("generated response did not match: expected: '%s', got: '%s'", expected, actual)
-	}
-}
-
-func TestMessageWithHtml(t *testing.T) {
-	testString := `abc<img src=1 onerror=alert("xss");`
-	escapedString := "abc&lt;img src=1 onerror=alert(&#34;xss&#34;);"
-	statusCode := http.StatusNotFound
-
-	expected := strings.ReplaceAll(
-		strings.ReplaceAll(ErrorPage, "%message%", escapedString),
-		"%status%",
-		http.StatusText(statusCode))
-	actual := generateResponse(testString, statusCode)
-
-	if expected != actual {
-		t.Errorf("generated response did not match: expected: '%s', got: '%s'", expected, actual)
-	}
-}

html/html.go

@@ -1,9 +1,71 @@
 package html
 
-import _ "embed"
+import (
+	_ "embed"
+	"net/http"
+	"os"
+	"path"
+	"text/template" // do not use html/template here, we sanitize the message before passing it to the template
 
-//go:embed 404.html
-var NotFoundPage string
+	"codeberg.org/codeberg/pages/server/context"
+	"github.com/microcosm-cc/bluemonday"
+	"github.com/rs/zerolog/log"
+)
 
-//go:embed error.html
-var ErrorPage string
+//go:embed templates/error.html
+var errorPage string
+
+var (
+	errorTemplate = template.Must(template.New("error").Parse(loadCustomTemplateOrDefault()))
+	sanitizer     = createBlueMondayPolicy()
+)
+
+type TemplateContext struct {
+	StatusCode int
+	StatusText string
+	Message    string
+}
+
+// ReturnErrorPage sets the response status code and writes the error page to the response body.
+// The error page contains a sanitized version of the message and the statusCode both in text and numeric form.
+//
+// Currently, only the following html tags are supported: <code>
+func ReturnErrorPage(ctx *context.Context, msg string, statusCode int) {
+	ctx.RespWriter.Header().Set("Content-Type", "text/html; charset=utf-8")
+	ctx.RespWriter.WriteHeader(statusCode)
+
+	templateContext := TemplateContext{
+		StatusCode: statusCode,
+		StatusText: http.StatusText(statusCode),
+		Message:    sanitizer.Sanitize(msg),
+	}
+
+	err := errorTemplate.Execute(ctx.RespWriter, templateContext)
+	if err != nil {
+		log.Err(err).Str("message", msg).Int("status", statusCode).Msg("could not write response")
+	}
+}
+
+func createBlueMondayPolicy() *bluemonday.Policy {
+	p := bluemonday.NewPolicy()
+
+	p.AllowElements("code")
+
+	return p
+}
+
+func loadCustomTemplateOrDefault() string {
+	contents, err := os.ReadFile("custom/error.html")
+	if err != nil {
+		if !os.IsNotExist(err) {
+			wd, wdErr := os.Getwd()
+			if wdErr != nil {
+				log.Err(err).Msg("could not load custom error page 'custom/error.html'")
+			} else {
+				log.Err(err).Msgf("could not load custom error page '%v'", path.Join(wd, "custom/error.html"))
+			}
+		}
+		return errorPage
+	}
+	return string(contents)
+}

html/html_test.go

@@ -0,0 +1,54 @@
+package html
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSanitizerSimpleString(t *testing.T) {
+	str := "simple text message without any html elements"
+
+	assert.Equal(t, str, sanitizer.Sanitize(str))
+}
+
+func TestSanitizerStringWithCodeTag(t *testing.T) {
+	str := "simple text message with <code>html</code> tag"
+
+	assert.Equal(t, str, sanitizer.Sanitize(str))
+}
+
+func TestSanitizerStringWithCodeTagWithAttribute(t *testing.T) {
+	str := "simple text message with <code id=\"code\">html</code> tag"
+	expected := "simple text message with <code>html</code> tag"
+
+	assert.Equal(t, expected, sanitizer.Sanitize(str))
+}
+
+func TestSanitizerStringWithATag(t *testing.T) {
+	str := "simple text message with <a>a link to another page</a>"
+	expected := "simple text message with a link to another page"
+
+	assert.Equal(t, expected, sanitizer.Sanitize(str))
+}
+
+func TestSanitizerStringWithATagAndHref(t *testing.T) {
+	str := "simple text message with <a href=\"http://evil.site\">a link to another page</a>"
+	expected := "simple text message with a link to another page"
+
+	assert.Equal(t, expected, sanitizer.Sanitize(str))
+}
+
+func TestSanitizerStringWithImgTag(t *testing.T) {
+	str := "simple text message with a <img alt=\"not found\" src=\"http://evil.site\">"
+	expected := "simple text message with a "
+
+	assert.Equal(t, expected, sanitizer.Sanitize(str))
+}
+
+func TestSanitizerStringWithImgTagAndOnerrorAttribute(t *testing.T) {
+	str := "simple text message with a <img alt=\"not found\" src=\"http://evil.site\" onerror=\"alert(secret)\">"
+	expected := "simple text message with a "
+
+	assert.Equal(t, expected, sanitizer.Sanitize(str))
+}

html/templates/error.html

@@ -0,0 +1,58 @@
+<!doctype html>
+<html class="codeberg-design">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width" />
+    <title>{{.StatusText}}</title>
+
+    <link rel="stylesheet" href="https://design.codeberg.org/design-kit/codeberg.css" />
+    <link rel="stylesheet" href="https://fonts.codeberg.org/dist/inter/Inter%20Web/inter.css" />
+
+    <style>
+      body {
+        margin: 0;
+        padding: 1rem;
+        box-sizing: border-box;
+        width: 100%;
+        min-height: 100vh;
+        display: flex;
+        flex-direction: column;
+        align-items: center;
+        justify-content: center;
+      }
+      code {
+        border-radius: 0.25rem;
+        padding: 0.25rem;
+        background-color: silver;
+      }
+    </style>
+  </head>
+  <body>
+    <svg xmlns="http://www.w3.org/2000/svg" height="10em" viewBox="0 0 24 24" fill="var(--blue-color)">
+      <path
+        d="M 9 2 C 5.1458514 2 2 5.1458514 2 9 C 2 12.854149 5.1458514 16 9 16 C 10.747998 16 12.345009 15.348024 13.574219 14.28125 L 14 14.707031 L 14 16 L 19.585938 21.585938 C 20.137937 22.137937 21.033938 22.137938 21.585938 21.585938 C 22.137938 21.033938 22.137938 20.137938 21.585938 19.585938 L 16 14 L 14.707031 14 L 14.28125 13.574219 C 15.348024 12.345009 16 10.747998 16 9 C 16 5.1458514 12.854149 2 9 2 z M 9 4 C 11.773268 4 14 6.2267316 14 9 C 14 11.773268 11.773268 14 9 14 C 6.2267316 14 4 11.773268 4 9 C 4 6.2267316 6.2267316 4 9 4 z"
+      />
+    </svg>
+    <h1 class="mb-0 text-primary">{{.StatusText}} (Error {{.StatusCode}})!</h1>
+    <h5 class="text-center" style="max-width: 25em">
+      <p>Sorry, but this page couldn't be served:</p>
+      <p><b>"{{.Message}}"</b></p>
+      <p>
+        The page you tried to reach is hosted on Codeberg Pages, which might currently be experiencing technical
+        difficulties. If that is the case, it could take a little while until this page is available again.
+      </p>
+      <p>
+        Otherwise, this page might also be unavailable due to a configuration error. If you are the owner of this
+        website, please make sure to check the
+        <a href="https://docs.codeberg.org/codeberg-pages/troubleshooting/" target="_blank"
+          >troubleshooting section in the Docs</a
+        >!
+      </p>
+    </h5>
+    <small class="text-muted">
+      <img src="https://design.codeberg.org/logo-kit/icon.svg" class="align-top" />
+      Static pages made easy -
+      <a href="https://codeberg.page">Codeberg Pages</a>
+    </small>
+  </body>
+</html>

integration/get_test.go

@@ -64,7 +64,7 @@ func TestGetContent(t *testing.T) {
 	assert.True(t, getSize(resp.Body) > 100)
 	assert.Len(t, resp.Header.Get("ETag"), 44)
 
-	// TODO: test get of non cachable content (content size > fileCacheSizeLimit)
+	// TODO: test get of non cacheable content (content size > fileCacheSizeLimit)
 }
 
 func TestCustomDomain(t *testing.T) {
@@ -109,6 +109,34 @@ func TestCustomDomainRedirects(t *testing.T) {
 	assert.EqualValues(t, "https://mock-pages.codeberg-test.org/README.md", resp.Header.Get("Location"))
 }
 
+func TestRawCustomDomain(t *testing.T) {
+	log.Println("=== TestRawCustomDomain ===")
+	// test raw domain response for custom domain branch
+	resp, err := getTestHTTPSClient().Get("https://raw.localhost.mock.directory:4430/cb_pages_tests/raw-test/example") // need cb_pages_tests fork
+	assert.NoError(t, err)
+	if !assert.NotNil(t, resp) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, http.StatusOK, resp.StatusCode)
+	assert.EqualValues(t, "text/plain; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "76", resp.Header.Get("Content-Length"))
+	assert.EqualValues(t, 76, getSize(resp.Body))
+}
+
+func TestRawIndex(t *testing.T) {
+	log.Println("=== TestRawIndex ===")
+	// test raw domain response for index.html
+	resp, err := getTestHTTPSClient().Get("https://raw.localhost.mock.directory:4430/cb_pages_tests/raw-test/@branch-test/index.html") // need cb_pages_tests fork
+	assert.NoError(t, err)
+	if !assert.NotNil(t, resp) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, http.StatusOK, resp.StatusCode)
+	assert.EqualValues(t, "text/plain; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "597", resp.Header.Get("Content-Length"))
+	assert.EqualValues(t, 597, getSize(resp.Body))
+}
+
 func TestGetNotFound(t *testing.T) {
 	log.Println("=== TestGetNotFound ===")
 	// test custom not found pages
@@ -123,9 +151,50 @@ func TestGetNotFound(t *testing.T) {
 	assert.EqualValues(t, 37, getSize(resp.Body))
 }
 
+func TestRedirect(t *testing.T) {
+	log.Println("=== TestRedirect ===")
+	// test redirects
+	resp, err := getTestHTTPSClient().Get("https://cb_pages_tests.localhost.mock.directory:4430/some_redirects/redirect")
+	assert.NoError(t, err)
+	if !assert.NotNil(t, resp) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, http.StatusMovedPermanently, resp.StatusCode)
+	assert.EqualValues(t, "https://example.com/", resp.Header.Get("Location"))
+}
+
+func TestSPARedirect(t *testing.T) {
+	log.Println("=== TestSPARedirect ===")
+	// test SPA redirects
+	url := "https://cb_pages_tests.localhost.mock.directory:4430/some_redirects/app/aqdjw"
+	resp, err := getTestHTTPSClient().Get(url)
+	assert.NoError(t, err)
+	if !assert.NotNil(t, resp) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, http.StatusOK, resp.StatusCode)
+	assert.EqualValues(t, url, resp.Request.URL.String())
+	assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "258", resp.Header.Get("Content-Length"))
+	assert.EqualValues(t, 258, getSize(resp.Body))
+}
+
+func TestSplatRedirect(t *testing.T) {
+	log.Println("=== TestSplatRedirect ===")
+	// test splat redirects
+	resp, err := getTestHTTPSClient().Get("https://cb_pages_tests.localhost.mock.directory:4430/some_redirects/articles/qfopefe")
+	assert.NoError(t, err)
+	if !assert.NotNil(t, resp) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, http.StatusMovedPermanently, resp.StatusCode)
+	assert.EqualValues(t, "/posts/qfopefe", resp.Header.Get("Location"))
+}
+
 func TestFollowSymlink(t *testing.T) {
 	log.Printf("=== TestFollowSymlink ===\n")
 
+	// file symlink
 	resp, err := getTestHTTPSClient().Get("https://cb_pages_tests.localhost.mock.directory:4430/tests_for_pages-server/@main/link")
 	assert.NoError(t, err)
 	if !assert.NotNil(t, resp) {
@@ -137,6 +206,16 @@ func TestFollowSymlink(t *testing.T) {
 	body := getBytes(resp.Body)
 	assert.EqualValues(t, 4, len(body))
 	assert.EqualValues(t, "abc\n", string(body))
+
+	// relative file links (../index.html file in this case)
+	resp, err = getTestHTTPSClient().Get("https://cb_pages_tests.localhost.mock.directory:4430/tests_for_pages-server/@main/dir_aim/some/")
+	assert.NoError(t, err)
+	if !assert.NotNil(t, resp) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, http.StatusOK, resp.StatusCode)
+	assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "an index\n", string(getBytes(resp.Body)))
 }
 
 func TestLFSSupport(t *testing.T) {
@@ -165,6 +244,18 @@ func TestGetOptions(t *testing.T) {
 	assert.EqualValues(t, "GET, HEAD, OPTIONS", resp.Header.Get("Allow"))
 }
 
+func TestHttpRedirect(t *testing.T) {
+	log.Println("=== TestHttpRedirect ===")
+	resp, err := getTestHTTPSClient().Get("http://mock-pages.codeberg-test.org:8880/README.md")
+	assert.NoError(t, err)
+	if !assert.NotNil(t, resp) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, http.StatusMovedPermanently, resp.StatusCode)
+	assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "https://mock-pages.codeberg-test.org:4430/README.md", resp.Header.Get("Location"))
+}
+
 func getTestHTTPSClient() *http.Client {
 	cookieJar, _ := cookiejar.New(nil)
 	return &http.Client{

integration/main_test.go

@@ -10,9 +10,10 @@ import (
 	"testing"
 	"time"
 
-	"codeberg.org/codeberg/pages/cmd"
-
 	"github.com/urfave/cli/v2"
+
+	cmd "codeberg.org/codeberg/pages/cli"
+	"codeberg.org/codeberg/pages/server"
 )
 
 func TestMain(m *testing.M) {
@@ -23,7 +24,7 @@ func TestMain(m *testing.M) {
 	}
 	defer func() {
 		serverCancel()
-		log.Println("=== TestMain: Server STOPED ===")
+		log.Println("=== TestMain: Server STOPPED ===")
 	}()
 
 	time.Sleep(10 * time.Second)
@@ -32,19 +33,24 @@ func TestMain(m *testing.M) {
 }
 
 func startServer(ctx context.Context) error {
-	args := []string{
-		"--verbose",
-		"--acme-accept-terms", "true",
-	}
+	args := []string{"integration"}
 	setEnvIfNotSet("ACME_API", "https://acme.mock.directory")
 	setEnvIfNotSet("PAGES_DOMAIN", "localhost.mock.directory")
 	setEnvIfNotSet("RAW_DOMAIN", "raw.localhost.mock.directory")
+	setEnvIfNotSet("PAGES_BRANCHES", "pages,main,master")
 	setEnvIfNotSet("PORT", "4430")
+	setEnvIfNotSet("HTTP_PORT", "8880")
+	setEnvIfNotSet("ENABLE_HTTP_SERVER", "true")
 	setEnvIfNotSet("DB_TYPE", "sqlite3")
+	setEnvIfNotSet("GITEA_ROOT", "https://codeberg.org")
+	setEnvIfNotSet("LOG_LEVEL", "trace")
+	setEnvIfNotSet("ENABLE_LFS_SUPPORT", "true")
+	setEnvIfNotSet("ENABLE_SYMLINK_SUPPORT", "true")
+	setEnvIfNotSet("ACME_ACCOUNT_CONFIG", "integration/acme-account.json")
 
 	app := cli.NewApp()
 	app.Name = "pages-server"
-	app.Action = cmd.Serve
+	app.Action = server.Serve
 	app.Flags = cmd.ServerFlags
 
 	go func() {

main.go

@@ -1,29 +1,21 @@
 package main
 
 import (
-	"fmt"
 	"os"
 
 	_ "github.com/joho/godotenv/autoload"
-	"github.com/urfave/cli/v2"
+	"github.com/rs/zerolog/log"
 
-	"codeberg.org/codeberg/pages/cmd"
-	"codeberg.org/codeberg/pages/server/version"
+	"codeberg.org/codeberg/pages/cli"
+	"codeberg.org/codeberg/pages/server"
 )
 
 func main() {
-	app := cli.NewApp()
-	app.Name = "pages-server"
-	app.Version = version.Version
-	app.Usage = "pages server"
-	app.Action = cmd.Serve
-	app.Flags = cmd.ServerFlags
-	app.Commands = []*cli.Command{
-		cmd.Certs,
-	}
+	app := cli.CreatePagesApp()
+	app.Action = server.Serve
 
 	if err := app.Run(os.Args); err != nil {
-		_, _ = fmt.Fprintln(os.Stderr, err)
+		log.Error().Err(err).Msg("A fatal error occurred")
 		os.Exit(1)
 	}
 }

renovate.json

@@ -0,0 +1,27 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:recommended",
+    ":maintainLockFilesWeekly",
+    ":enablePreCommit",
+    "schedule:automergeDaily",
+    "schedule:weekends"
+  ],
+  "automergeType": "branch",
+  "automergeMajor": false,
+  "automerge": true,
+  "prConcurrentLimit": 5,
+  "labels": ["dependencies"],
+  "packageRules": [
+    {
+      "matchManagers": ["gomod", "dockerfile"]
+    },
+    {
+      "groupName": "golang deps non-major",
+      "matchManagers": ["gomod"],
+      "matchUpdateTypes": ["minor", "patch"],
+      "extends": ["schedule:daily"]
+    }
+  ],
+  "postUpdateOptions": ["gomodTidy", "gomodUpdateImportPaths"]
+}

server/acme/client.go

@@ -0,0 +1,26 @@
+package acme
+
+import (
+	"errors"
+	"fmt"
+
+	"codeberg.org/codeberg/pages/config"
+	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/certificates"
+)
+
+var ErrAcmeMissConfig = errors.New("ACME client has wrong config")
+
+func CreateAcmeClient(cfg config.ACMEConfig, enableHTTPServer bool, challengeCache cache.ICache) (*certificates.AcmeClient, error) {
+	// check config
+	if (!cfg.AcceptTerms || (cfg.DNSProvider == "" && !cfg.NoDNS01)) && cfg.APIEndpoint != "https://acme.mock.directory" {
+		return nil, fmt.Errorf("%w: you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER or $NO_DNS_01, unless $ACME_API is set to https://acme.mock.directory", ErrAcmeMissConfig)
+	}
+	if cfg.EAB_HMAC != "" && cfg.EAB_KID == "" {
+		return nil, fmt.Errorf("%w: ACME_EAB_HMAC also needs ACME_EAB_KID to be set", ErrAcmeMissConfig)
+	} else if cfg.EAB_HMAC == "" && cfg.EAB_KID != "" {
+		return nil, fmt.Errorf("%w: ACME_EAB_KID also needs ACME_EAB_HMAC to be set", ErrAcmeMissConfig)
+	}
+
+	return certificates.NewAcmeClient(cfg, enableHTTPServer, challengeCache)
+}

server/cache/interface.go

@@ -2,7 +2,8 @@ package cache
 
 import "time"
 
-type SetGetKey interface {
+// ICache is an interface that defines how the pages server interacts with the cache.
+type ICache interface {
 	Set(key string, value interface{}, ttl time.Duration) error
 	Get(key string) (interface{}, bool)
 	Remove(key string)

server/cache/memory.go

@@ -2,6 +2,6 @@ package cache
 
 import "github.com/OrlovEvgeny/go-mcache"
 
-func NewKeyValueCache() SetGetKey {
+func NewInMemoryCache() ICache {
 	return mcache.New()
 }

server/certificates/acme_client.go

@@ -0,0 +1,93 @@
+package certificates
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/go-acme/lego/v4/lego"
+	"github.com/go-acme/lego/v4/providers/dns"
+	"github.com/reugn/equalizer"
+	"github.com/rs/zerolog/log"
+
+	"codeberg.org/codeberg/pages/config"
+	"codeberg.org/codeberg/pages/server/cache"
+)
+
+type AcmeClient struct {
+	legoClient              *lego.Client
+	dnsChallengerLegoClient *lego.Client
+
+	obtainLocks sync.Map
+
+	acmeUseRateLimits bool
+
+	// limiter
+	acmeClientOrderLimit              *equalizer.TokenBucket
+	acmeClientRequestLimit            *equalizer.TokenBucket
+	acmeClientFailLimit               *equalizer.TokenBucket
+	acmeClientCertificateLimitPerUser map[string]*equalizer.TokenBucket
+}
+
+func NewAcmeClient(cfg config.ACMEConfig, enableHTTPServer bool, challengeCache cache.ICache) (*AcmeClient, error) {
+	acmeConfig, err := setupAcmeConfig(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	acmeClient, err := lego.NewClient(acmeConfig)
+	if err != nil {
+		log.Fatal().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
+	} else {
+		err = acmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
+		if err != nil {
+			log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider")
+		}
+		if enableHTTPServer {
+			err = acmeClient.Challenge.SetHTTP01Provider(AcmeHTTPChallengeProvider{challengeCache})
+			if err != nil {
+				log.Error().Err(err).Msg("Can't create HTTP-01 provider")
+			}
+		}
+	}
+
+	mainDomainAcmeClient, err := lego.NewClient(acmeConfig)
+	if err != nil {
+		log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
+	} else {
+		if cfg.DNSProvider == "" {
+			// using mock wildcard certs
+			mainDomainAcmeClient = nil
+		} else {
+			// use DNS-Challenge https://go-acme.github.io/lego/dns/
+			provider, err := dns.NewDNSChallengeProviderByName(cfg.DNSProvider)
+			if err != nil {
+				return nil, fmt.Errorf("can not create DNS Challenge provider: %w", err)
+			}
+			if err := mainDomainAcmeClient.Challenge.SetDNS01Provider(provider); err != nil {
+				return nil, fmt.Errorf("can not create DNS-01 provider: %w", err)
+			}
+		}
+	}
+
+	return &AcmeClient{
+		legoClient:              acmeClient,
+		dnsChallengerLegoClient: mainDomainAcmeClient,
+
+		acmeUseRateLimits: cfg.UseRateLimits,
+
+		obtainLocks: sync.Map{},
+
+		// limiter
+
+		// rate limit is 300 / 3 hours, we want 200 / 2 hours but to refill more often, so that's 25 new domains every 15 minutes
+		// TODO: when this is used a lot, we probably have to think of a somewhat better solution?
+		acmeClientOrderLimit: equalizer.NewTokenBucket(25, 15*time.Minute),
+		// rate limit is 20 / second, we want 5 / second (especially as one cert takes at least two requests)
+		acmeClientRequestLimit: equalizer.NewTokenBucket(5, 1*time.Second),
+		// rate limit is 5 / hour https://letsencrypt.org/docs/failed-validation-limit/
+		acmeClientFailLimit: equalizer.NewTokenBucket(5, 1*time.Hour),
+		// checkUserLimit() use this to rate also per user
+		acmeClientCertificateLimitPerUser: map[string]*equalizer.TokenBucket{},
+	}, nil
+}

server/certificates/acme_config.go

@@ -0,0 +1,110 @@
+package certificates
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"codeberg.org/codeberg/pages/config"
+	"github.com/go-acme/lego/v4/certcrypto"
+	"github.com/go-acme/lego/v4/lego"
+	"github.com/go-acme/lego/v4/registration"
+	"github.com/rs/zerolog/log"
+)
+
+const challengePath = "/.well-known/acme-challenge/"
+
+func setupAcmeConfig(cfg config.ACMEConfig) (*lego.Config, error) {
+	var myAcmeAccount AcmeAccount
+	var myAcmeConfig *lego.Config
+
+	if cfg.AccountConfigFile == "" {
+		return nil, fmt.Errorf("invalid acme config file: '%s'", cfg.AccountConfigFile)
+	}
+
+	if account, err := os.ReadFile(cfg.AccountConfigFile); err == nil {
+		log.Info().Msgf("found existing acme account config file '%s'", cfg.AccountConfigFile)
+		if err := json.Unmarshal(account, &myAcmeAccount); err != nil {
+			return nil, err
+		}
+
+		myAcmeAccount.Key, err = certcrypto.ParsePEMPrivateKey([]byte(myAcmeAccount.KeyPEM))
+		if err != nil {
+			return nil, err
+		}
+
+		myAcmeConfig = lego.NewConfig(&myAcmeAccount)
+		myAcmeConfig.CADirURL = cfg.APIEndpoint
+		myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
+
+		// Validate Config
+		_, err := lego.NewClient(myAcmeConfig)
+		if err != nil {
+			log.Info().Err(err).Msg("config validation failed, you might just delete the config file and let it recreate")
+			return nil, fmt.Errorf("acme config validation failed: %w", err)
+		}
+
+		return myAcmeConfig, nil
+	} else if !os.IsNotExist(err) {
+		return nil, err
+	}
+
+	log.Info().Msgf("no existing acme account config found, try to create a new one")
+
+	privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return nil, err
+	}
+	myAcmeAccount = AcmeAccount{
+		Email:  cfg.Email,
+		Key:    privateKey,
+		KeyPEM: string(certcrypto.PEMEncode(privateKey)),
+	}
+	myAcmeConfig = lego.NewConfig(&myAcmeAccount)
+	myAcmeConfig.CADirURL = cfg.APIEndpoint
+	myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
+	tempClient, err := lego.NewClient(myAcmeConfig)
+	if err != nil {
+		log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
+	} else {
+		// accept terms & log in to EAB
+		if cfg.EAB_KID == "" || cfg.EAB_HMAC == "" {
+			reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: cfg.AcceptTerms})
+			if err != nil {
+				log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
+			} else {
+				myAcmeAccount.Registration = reg
+			}
+		} else {
+			reg, err := tempClient.Registration.RegisterWithExternalAccountBinding(registration.RegisterEABOptions{
+				TermsOfServiceAgreed: cfg.AcceptTerms,
+				Kid:                  cfg.EAB_KID,
+				HmacEncoded:          cfg.EAB_HMAC,
+			})
+			if err != nil {
+				log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
+			} else {
+				myAcmeAccount.Registration = reg
+			}
+		}
+
+		if myAcmeAccount.Registration != nil {
+			acmeAccountJSON, err := json.Marshal(myAcmeAccount)
+			if err != nil {
+				log.Error().Err(err).Msg("json.Marshalfailed, waiting for manual restart to avoid rate limits")
+				select {}
+			}
+			log.Info().Msgf("new acme account created. write to config file '%s'", cfg.AccountConfigFile)
+			err = os.WriteFile(cfg.AccountConfigFile, acmeAccountJSON, 0o600)
+			if err != nil {
+				log.Error().Err(err).Msg("os.WriteFile failed, waiting for manual restart to avoid rate limits")
+				select {}
+			}
+		}
+	}
+
+	return myAcmeConfig, nil
+}

server/certificates/cached_challengers.go

@@ -0,0 +1,83 @@
+package certificates
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/go-acme/lego/v4/challenge"
+	"github.com/rs/zerolog/log"
+
+	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/context"
+)
+
+type AcmeTLSChallengeProvider struct {
+	challengeCache cache.ICache
+}
+
+// make sure AcmeTLSChallengeProvider match Provider interface
+var _ challenge.Provider = AcmeTLSChallengeProvider{}
+
+func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error {
+	return a.challengeCache.Set(domain, keyAuth, 1*time.Hour)
+}
+
+func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error {
+	a.challengeCache.Remove(domain)
+	return nil
+}
+
+type AcmeHTTPChallengeProvider struct {
+	challengeCache cache.ICache
+}
+
+// make sure AcmeHTTPChallengeProvider match Provider interface
+var _ challenge.Provider = AcmeHTTPChallengeProvider{}
+
+func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error {
+	return a.challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour)
+}
+
+func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
+	a.challengeCache.Remove(domain + "/" + token)
+	return nil
+}
+
+func SetupHTTPACMEChallengeServer(challengeCache cache.ICache, sslPort uint) http.HandlerFunc {
+	// handle custom-ssl-ports to be added on https redirects
+	portPart := ""
+	if sslPort != 443 {
+		portPart = fmt.Sprintf(":%d", sslPort)
+	}
+
+	return func(w http.ResponseWriter, req *http.Request) {
+		ctx := context.New(w, req)
+		domain := ctx.TrimHostPort()
+
+		// it's an acme request
+		if strings.HasPrefix(ctx.Path(), challengePath) {
+			challenge, ok := challengeCache.Get(domain + "/" + strings.TrimPrefix(ctx.Path(), challengePath))
+			if !ok || challenge == nil {
+				log.Info().Msgf("HTTP-ACME challenge for '%s' failed: token not found", domain)
+				ctx.String("no challenge for this token", http.StatusNotFound)
+			}
+			log.Info().Msgf("HTTP-ACME challenge for '%s' succeeded", domain)
+			ctx.String(challenge.(string))
+			return
+		}
+
+		// it's a normal http request that needs to be redirected
+		u, err := url.Parse(fmt.Sprintf("https://%s%s%s", domain, portPart, ctx.Path()))
+		if err != nil {
+			log.Error().Err(err).Msg("could not craft http to https redirect")
+			ctx.String("", http.StatusInternalServerError)
+		}
+
+		newURL := u.String()
+		log.Debug().Msgf("redirect http to https: %s", newURL)
+		ctx.Redirect(newURL, http.StatusMovedPermanently)
+	}
+}

server/certificates/certificates.go

@@ -2,64 +2,71 @@ package certificates
 
 import (
 	"context"
-	"crypto/ecdsa"
-	"crypto/elliptic"
-	"crypto/rand"
 	"crypto/tls"
 	"crypto/x509"
-	"encoding/json"
 	"errors"
 	"fmt"
-	"os"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/go-acme/lego/v4/certcrypto"
 	"github.com/go-acme/lego/v4/certificate"
-	"github.com/go-acme/lego/v4/challenge"
 	"github.com/go-acme/lego/v4/challenge/tlsalpn01"
 	"github.com/go-acme/lego/v4/lego"
-	"github.com/go-acme/lego/v4/providers/dns"
-	"github.com/go-acme/lego/v4/registration"
+	"github.com/hashicorp/golang-lru/v2/expirable"
 	"github.com/reugn/equalizer"
+	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 
 	"codeberg.org/codeberg/pages/server/cache"
+	psContext "codeberg.org/codeberg/pages/server/context"
 	"codeberg.org/codeberg/pages/server/database"
 	dnsutils "codeberg.org/codeberg/pages/server/dns"
 	"codeberg.org/codeberg/pages/server/gitea"
 	"codeberg.org/codeberg/pages/server/upstream"
 )
 
+var ErrUserRateLimitExceeded = errors.New("rate limit exceeded: 10 certificates per user per 24 hours")
+
 // TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates.
 func TLSConfig(mainDomainSuffix string,
 	giteaClient *gitea.Client,
-	dnsProvider string,
-	acmeUseRateLimits bool,
-	keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey,
+	acmeClient *AcmeClient,
+	firstDefaultBranch string,
+	challengeCache, canonicalDomainCache cache.ICache,
 	certDB database.CertDB,
+	noDNS01 bool,
+	rawDomain string,
 ) *tls.Config {
+	// every cert is at most 24h in the cache and 7 days before expiry the cert is renewed
+	keyCache := expirable.NewLRU[string, *tls.Certificate](32, nil, 24*time.Hour)
+
 	return &tls.Config{
 		// check DNS name & get certificate from Let's Encrypt
 		GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
-			sni := strings.ToLower(strings.TrimSpace(info.ServerName))
-			if len(sni) < 1 {
-				return nil, errors.New("missing sni")
+			ctx := psContext.New(nil, nil)
+			log := log.With().Str("ReqId", ctx.ReqId).Logger()
+
+			domain := strings.ToLower(strings.TrimSpace(info.ServerName))
+			log.Debug().Str("domain", domain).Msg("start: get tls certificate")
+			if len(domain) < 1 {
+				return nil, errors.New("missing domain info via SNI (RFC 4366, Section 3.1)")
 			}
 
+			// https request init is actually a acme challenge
 			if info.SupportedProtos != nil {
 				for _, proto := range info.SupportedProtos {
 					if proto != tlsalpn01.ACMETLS1Protocol {
 						continue
 					}
+					log.Info().Msgf("Detect ACME-TLS1 challenge for '%s'", domain)
 
-					challenge, ok := challengeCache.Get(sni)
+					challenge, ok := challengeCache.Get(domain)
 					if !ok {
 						return nil, errors.New("no challenge for this domain")
 					}
-					cert, err := tlsalpn01.ChallengeCert(sni, challenge.(string))
+					cert, err := tlsalpn01.ChallengeCert(domain, challenge.(string))
 					if err != nil {
 						return nil, err
 					}
@@ -69,22 +76,37 @@ func TLSConfig(mainDomainSuffix string,
 
 			targetOwner := ""
 			mayObtainCert := true
-			if strings.HasSuffix(sni, mainDomainSuffix) || strings.EqualFold(sni, mainDomainSuffix[1:]) {
-				// deliver default certificate for the main domain (*.codeberg.page)
-				sni = mainDomainSuffix
+
+			if strings.HasSuffix(domain, mainDomainSuffix) || strings.EqualFold(domain, mainDomainSuffix[1:]) {
+				if noDNS01 {
+					// Limit the domains allowed to request a certificate to pages-server domains
+					// and domains for an existing user of org
+					if !strings.EqualFold(domain, mainDomainSuffix[1:]) && !strings.EqualFold(domain, rawDomain) {
+						targetOwner := strings.TrimSuffix(domain, mainDomainSuffix)
+						owner_exist, err := giteaClient.GiteaCheckIfOwnerExists(targetOwner)
+						mayObtainCert = owner_exist
+						if err != nil {
+							log.Error().Err(err).Msgf("Failed to check '%s' existence on the forge: %s", targetOwner, err)
+							mayObtainCert = false
+						}
+					}
+				} else {
+					// deliver default certificate for the main domain (*.codeberg.page)
+					domain = mainDomainSuffix
+				}
 			} else {
 				var targetRepo, targetBranch string
-				targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(sni, mainDomainSuffix, dnsLookupCache)
+				targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch)
 				if targetOwner == "" {
 					// DNS not set up, return main certificate to redirect to the docs
-					sni = mainDomainSuffix
+					domain = mainDomainSuffix
 				} else {
 					targetOpt := &upstream.Options{
 						TargetOwner:  targetOwner,
 						TargetRepo:   targetRepo,
 						TargetBranch: targetBranch,
 					}
-					_, valid := targetOpt.CheckCanonicalDomain(giteaClient, sni, mainDomainSuffix, canonicalDomainCache)
+					_, valid := targetOpt.CheckCanonicalDomain(ctx, giteaClient, domain, mainDomainSuffix, canonicalDomainCache)
 					if !valid {
 						// We shouldn't obtain a certificate when we cannot check if the
 						// repository has specified this domain in the `.domains` file.
@@ -93,32 +115,35 @@ func TLSConfig(mainDomainSuffix string,
 				}
 			}
 
-			if tlsCertificate, ok := keyCache.Get(sni); ok {
+			if tlsCertificate, ok := keyCache.Get(domain); ok {
 				// we can use an existing certificate object
-				return tlsCertificate.(*tls.Certificate), nil
+				return tlsCertificate, nil
 			}
 
 			var tlsCertificate *tls.Certificate
 			var err error
-			if tlsCertificate, err = retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB); err != nil {
-				// request a new certificate
-				if strings.EqualFold(sni, mainDomainSuffix) {
+			if tlsCertificate, err = acmeClient.retrieveCertFromDB(log, domain, mainDomainSuffix, false, certDB); err != nil {
+				if !errors.Is(err, database.ErrNotFound) {
+					return nil, err
+				}
+				// we could not find a cert in db, request a new certificate
+
+				// first check if we are allowed to obtain a cert for this domain
+				if strings.EqualFold(domain, mainDomainSuffix) {
 					return nil, errors.New("won't request certificate for main domain, something really bad has happened")
 				}
-
 				if !mayObtainCert {
-					return nil, fmt.Errorf("won't request certificate for %q", sni)
+					return nil, fmt.Errorf("won't request certificate for %q", domain)
 				}
 
-				tlsCertificate, err = obtainCert(acmeClient, []string{sni}, nil, targetOwner, dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
+				tlsCertificate, err = acmeClient.obtainCert(log, acmeClient.legoClient, []string{domain}, nil, targetOwner, false, mainDomainSuffix, certDB)
 				if err != nil {
 					return nil, err
 				}
 			}
 
-			if err := keyCache.Set(sni, tlsCertificate, 15*time.Minute); err != nil {
-				return nil, err
-			}
+			keyCache.Add(domain, tlsCertificate)
+
 			return tlsCertificate, nil
 		},
 		NextProtos: []string{
@@ -141,67 +166,20 @@ func TLSConfig(mainDomainSuffix string,
 	}
 }
 
-func checkUserLimit(user string) error {
-	userLimit, ok := acmeClientCertificateLimitPerUser[user]
+func (c *AcmeClient) checkUserLimit(user string) error {
+	userLimit, ok := c.acmeClientCertificateLimitPerUser[user]
 	if !ok {
-		// Each Codeberg user can only add 10 new domains per day.
+		// Each user can only add 10 new domains per day.
 		userLimit = equalizer.NewTokenBucket(10, time.Hour*24)
-		acmeClientCertificateLimitPerUser[user] = userLimit
+		c.acmeClientCertificateLimitPerUser[user] = userLimit
 	}
 	if !userLimit.Ask() {
-		return errors.New("rate limit exceeded: 10 certificates per user per 24 hours")
+		return fmt.Errorf("user '%s' error: %w", user, ErrUserRateLimitExceeded)
 	}
 	return nil
 }
 
-var (
-	acmeClient, mainDomainAcmeClient  *lego.Client
-	acmeClientCertificateLimitPerUser = map[string]*equalizer.TokenBucket{}
-)
-
-// rate limit is 300 / 3 hours, we want 200 / 2 hours but to refill more often, so that's 25 new domains every 15 minutes
-// TODO: when this is used a lot, we probably have to think of a somewhat better solution?
-var acmeClientOrderLimit = equalizer.NewTokenBucket(25, 15*time.Minute)
-
-// rate limit is 20 / second, we want 5 / second (especially as one cert takes at least two requests)
-var acmeClientRequestLimit = equalizer.NewTokenBucket(5, 1*time.Second)
-
-// rate limit is 5 / hour https://letsencrypt.org/docs/failed-validation-limit/
-var acmeClientFailLimit = equalizer.NewTokenBucket(5, 1*time.Hour)
-
-type AcmeTLSChallengeProvider struct {
-	challengeCache cache.SetGetKey
-}
-
-// make sure AcmeTLSChallengeProvider match Provider interface
-var _ challenge.Provider = AcmeTLSChallengeProvider{}
-
-func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error {
-	return a.challengeCache.Set(domain, keyAuth, 1*time.Hour)
-}
-
-func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error {
-	a.challengeCache.Remove(domain)
-	return nil
-}
-
-type AcmeHTTPChallengeProvider struct {
-	challengeCache cache.SetGetKey
-}
-
-// make sure AcmeHTTPChallengeProvider match Provider interface
-var _ challenge.Provider = AcmeHTTPChallengeProvider{}
-
-func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error {
-	return a.challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour)
-}
-
-func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
-	a.challengeCache.Remove(domain + "/" + token)
-	return nil
-}
-
-func retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (*tls.Certificate, error) {
+func (c *AcmeClient) retrieveCertFromDB(log zerolog.Logger, sni, mainDomainSuffix string, useDnsProvider bool, certDB database.CertDB) (*tls.Certificate, error) {
 	// parse certificate from database
 	res, err := certDB.Get(sni)
 	if err != nil {
@@ -217,15 +195,14 @@ func retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider string, acmeUseRateLi
 
 	// TODO: document & put into own function
 	if !strings.EqualFold(sni, mainDomainSuffix) {
-		tlsCertificate.Leaf, err = x509.ParseCertificate(tlsCertificate.Certificate[0])
+		tlsCertificate.Leaf, err = leaf(&tlsCertificate)
 		if err != nil {
-			return nil, fmt.Errorf("error parsin leaf tlsCert: %w", err)
+			return nil, err
 		}
-
 		// renew certificates 7 days before they expire
 		if tlsCertificate.Leaf.NotAfter.Before(time.Now().Add(7 * 24 * time.Hour)) {
 			// TODO: use ValidTill of custom cert struct
-			if res.CSR != nil && len(res.CSR) > 0 {
+			if len(res.CSR) > 0 {
 				// CSR stores the time when the renewal shall be tried again
 				nextTryUnix, err := strconv.ParseInt(string(res.CSR), 10, 64)
 				if err == nil && time.Now().Before(time.Unix(nextTryUnix, 0)) {
@@ -235,7 +212,7 @@ func retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider string, acmeUseRateLi
 			// TODO: make a queue ?
 			go (func() {
 				res.CSR = nil // acme client doesn't like CSR to be set
-				if _, err := obtainCert(acmeClient, []string{sni}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB); err != nil {
+				if _, err := c.obtainCert(log, c.legoClient, []string{sni}, res, "", useDnsProvider, mainDomainSuffix, certDB); err != nil {
 					log.Error().Msgf("Couldn't renew certificate for %s: %v", sni, err)
 				}
 			})()
@@ -245,60 +222,59 @@ func retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider string, acmeUseRateLi
 	return &tlsCertificate, nil
 }
 
-var obtainLocks = sync.Map{}
-
-func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user, dnsProvider, mainDomainSuffix string, acmeUseRateLimits bool, keyDatabase database.CertDB) (*tls.Certificate, error) {
+func (c *AcmeClient) obtainCert(log zerolog.Logger, acmeClient *lego.Client, domains []string, renew *certificate.Resource, user string, useDnsProvider bool, mainDomainSuffix string, keyDatabase database.CertDB) (*tls.Certificate, error) {
 	name := strings.TrimPrefix(domains[0], "*")
-	if dnsProvider == "" && len(domains[0]) > 0 && domains[0][0] == '*' {
-		domains = domains[1:]
-	}
 
 	// lock to avoid simultaneous requests
-	_, working := obtainLocks.LoadOrStore(name, struct{}{})
+	_, working := c.obtainLocks.LoadOrStore(name, struct{}{})
 	if working {
 		for working {
 			time.Sleep(100 * time.Millisecond)
-			_, working = obtainLocks.Load(name)
+			_, working = c.obtainLocks.Load(name)
 		}
-		cert, err := retrieveCertFromDB(name, mainDomainSuffix, dnsProvider, acmeUseRateLimits, keyDatabase)
+		cert, err := c.retrieveCertFromDB(log, name, mainDomainSuffix, useDnsProvider, keyDatabase)
 		if err != nil {
 			return nil, fmt.Errorf("certificate failed in synchronous request: %w", err)
 		}
 		return cert, nil
 	}
-	defer obtainLocks.Delete(name)
+	defer c.obtainLocks.Delete(name)
 
 	if acmeClient == nil {
-		return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!", mainDomainSuffix, keyDatabase)
+		if useDnsProvider {
+			return mockCert(domains[0], "DNS ACME client is not defined", mainDomainSuffix, keyDatabase)
+		} else {
+			return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!", mainDomainSuffix, keyDatabase)
+		}
 	}
 
 	// request actual cert
 	var res *certificate.Resource
 	var err error
 	if renew != nil && renew.CertURL != "" {
-		if acmeUseRateLimits {
-			acmeClientRequestLimit.Take()
+		if c.acmeUseRateLimits {
+			c.acmeClientRequestLimit.Take()
 		}
 		log.Debug().Msgf("Renewing certificate for: %v", domains)
 		res, err = acmeClient.Certificate.Renew(*renew, true, false, "")
 		if err != nil {
 			log.Error().Err(err).Msgf("Couldn't renew certificate for %v, trying to request a new one", domains)
-			if acmeUseRateLimits {
-				acmeClientFailLimit.Take()
+			if c.acmeUseRateLimits {
+				c.acmeClientFailLimit.Take()
 			}
 			res = nil
 		}
 	}
 	if res == nil {
 		if user != "" {
-			if err := checkUserLimit(user); err != nil {
+			if err := c.checkUserLimit(user); err != nil {
 				return nil, err
 			}
 		}
 
-		if acmeUseRateLimits {
-			acmeClientOrderLimit.Take()
-			acmeClientRequestLimit.Take()
+		if c.acmeUseRateLimits {
+			c.acmeClientOrderLimit.Take()
+			c.acmeClientRequestLimit.Take()
 		}
 		log.Debug().Msgf("Re-requesting new certificate for %v", domains)
 		res, err = acmeClient.Certificate.Obtain(certificate.ObtainRequest{
@@ -306,8 +282,8 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re
 			Bundle:     true,
 			MustStaple: false,
 		})
-		if acmeUseRateLimits && err != nil {
-			acmeClientFailLimit.Take()
+		if c.acmeUseRateLimits && err != nil {
+			c.acmeClientFailLimit.Take()
 		}
 	}
 	if err != nil {
@@ -323,6 +299,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re
 			}
 			leaf, err := leaf(&tlsCertificate)
 			if err == nil && leaf.NotAfter.After(time.Now()) {
+				tlsCertificate.Leaf = leaf
 				// avoid sending a mock cert instead of a still valid cert, instead abuse CSR field to store time to try again at
 				renew.CSR = []byte(strconv.FormatInt(time.Now().Add(6*time.Hour).Unix(), 10))
 				if err := keyDatabase.Put(name, renew); err != nil {
@@ -349,136 +326,15 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re
 	return &tlsCertificate, nil
 }
 
-func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcceptTerms bool) (*lego.Config, error) {
-	// TODO: make it a config flag
-	const configFile = "acme-account.json"
-	var myAcmeAccount AcmeAccount
-	var myAcmeConfig *lego.Config
-
-	if account, err := os.ReadFile(configFile); err == nil {
-		if err := json.Unmarshal(account, &myAcmeAccount); err != nil {
-			return nil, err
-		}
-		myAcmeAccount.Key, err = certcrypto.ParsePEMPrivateKey([]byte(myAcmeAccount.KeyPEM))
-		if err != nil {
-			return nil, err
-		}
-		myAcmeConfig = lego.NewConfig(&myAcmeAccount)
-		myAcmeConfig.CADirURL = acmeAPI
-		myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
-
-		// Validate Config
-		_, err := lego.NewClient(myAcmeConfig)
-		if err != nil {
-			// TODO: should we fail hard instead?
-			log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
-		}
-		return myAcmeConfig, nil
-	} else if !os.IsNotExist(err) {
-		return nil, err
-	}
-
-	privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
-	if err != nil {
-		return nil, err
-	}
-	myAcmeAccount = AcmeAccount{
-		Email:  acmeMail,
-		Key:    privateKey,
-		KeyPEM: string(certcrypto.PEMEncode(privateKey)),
-	}
-	myAcmeConfig = lego.NewConfig(&myAcmeAccount)
-	myAcmeConfig.CADirURL = acmeAPI
-	myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
-	tempClient, err := lego.NewClient(myAcmeConfig)
-	if err != nil {
-		log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
-	} else {
-		// accept terms & log in to EAB
-		if acmeEabKID == "" || acmeEabHmac == "" {
-			reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: acmeAcceptTerms})
-			if err != nil {
-				log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
-			} else {
-				myAcmeAccount.Registration = reg
-			}
-		} else {
-			reg, err := tempClient.Registration.RegisterWithExternalAccountBinding(registration.RegisterEABOptions{
-				TermsOfServiceAgreed: acmeAcceptTerms,
-				Kid:                  acmeEabKID,
-				HmacEncoded:          acmeEabHmac,
-			})
-			if err != nil {
-				log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
-			} else {
-				myAcmeAccount.Registration = reg
-			}
-		}
-
-		if myAcmeAccount.Registration != nil {
-			acmeAccountJSON, err := json.Marshal(myAcmeAccount)
-			if err != nil {
-				log.Error().Err(err).Msg("json.Marshalfailed, waiting for manual restart to avoid rate limits")
-				select {}
-			}
-			err = os.WriteFile(configFile, acmeAccountJSON, 0o600)
-			if err != nil {
-				log.Error().Err(err).Msg("os.WriteFile failed, waiting for manual restart to avoid rate limits")
-				select {}
-			}
-		}
-	}
-
-	return myAcmeConfig, nil
-}
-
-func SetupCertificates(mainDomainSuffix, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) error {
+func SetupMainDomainCertificates(log zerolog.Logger, mainDomainSuffix string, acmeClient *AcmeClient, certDB database.CertDB) error {
 	// getting main cert before ACME account so that we can fail here without hitting rate limits
 	mainCertBytes, err := certDB.Get(mainDomainSuffix)
 	if err != nil && !errors.Is(err, database.ErrNotFound) {
 		return fmt.Errorf("cert database is not working: %w", err)
 	}
 
-	acmeClient, err = lego.NewClient(acmeConfig)
-	if err != nil {
-		log.Fatal().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
-	} else {
-		err = acmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
-		if err != nil {
-			log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider")
-		}
-		if enableHTTPServer {
-			err = acmeClient.Challenge.SetHTTP01Provider(AcmeHTTPChallengeProvider{challengeCache})
-			if err != nil {
-				log.Error().Err(err).Msg("Can't create HTTP-01 provider")
-			}
-		}
-	}
-
-	mainDomainAcmeClient, err = lego.NewClient(acmeConfig)
-	if err != nil {
-		log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
-	} else {
-		if dnsProvider == "" {
-			// using mock server, don't use wildcard certs
-			err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
-			if err != nil {
-				log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider")
-			}
-		} else {
-			provider, err := dns.NewDNSChallengeProviderByName(dnsProvider)
-			if err != nil {
-				log.Error().Err(err).Msg("Can't create DNS Challenge provider")
-			}
-			err = mainDomainAcmeClient.Challenge.SetDNS01Provider(provider)
-			if err != nil {
-				log.Error().Err(err).Msg("Can't create DNS-01 provider")
-			}
-		}
-	}
-
 	if mainCertBytes == nil {
-		_, err = obtainCert(mainDomainAcmeClient, []string{"*" + mainDomainSuffix, mainDomainSuffix[1:]}, nil, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
+		_, err = acmeClient.obtainCert(log, acmeClient.dnsChallengerLegoClient, []string{"*" + mainDomainSuffix, mainDomainSuffix[1:]}, nil, "", true, mainDomainSuffix, certDB)
 		if err != nil {
 			log.Error().Err(err).Msg("Couldn't renew main domain certificate, continuing with mock certs only")
 		}
@@ -487,7 +343,7 @@ func SetupCertificates(mainDomainSuffix, dnsProvider string, acmeConfig *lego.Co
 	return nil
 }
 
-func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffix, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) {
+func MaintainCertDB(log zerolog.Logger, ctx context.Context, interval time.Duration, acmeClient *AcmeClient, mainDomainSuffix string, certDB database.CertDB) {
 	for {
 		// delete expired certs that will be invalid until next clean up
 		threshold := time.Now().Add(interval)
@@ -510,14 +366,6 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
 				}
 			}
 			log.Debug().Msgf("Removed %d expired certificates from the database", expiredCertCount)
-
-			// compact the database
-			msg, err := certDB.Compact()
-			if err != nil {
-				log.Error().Err(err).Msg("Compacting key database failed")
-			} else {
-				log.Debug().Msgf("Compacted key database: %s", msg)
-			}
 		}
 
 		// update main cert
@@ -533,7 +381,7 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
 			} else if tlsCertificates[0].NotAfter.Before(time.Now().Add(30 * 24 * time.Hour)) {
 				// renew main certificate 30 days before it expires
 				go (func() {
-					_, err = obtainCert(mainDomainAcmeClient, []string{"*" + mainDomainSuffix, mainDomainSuffix[1:]}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
+					_, err = acmeClient.obtainCert(log, acmeClient.dnsChallengerLegoClient, []string{"*" + mainDomainSuffix, mainDomainSuffix[1:]}, res, "", true, mainDomainSuffix, certDB)
 					if err != nil {
 						log.Error().Err(err).Msg("Couldn't renew certificate for main domain")
 					}
@@ -549,11 +397,20 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
 	}
 }
 
-// leaf returns the parsed leaf certificate, either from c.leaf or by parsing
+// leaf returns the parsed leaf certificate, either from c.Leaf or by parsing
 // the corresponding c.Certificate[0].
+// After successfully parsing the cert c.Leaf gets set to the parsed cert.
 func leaf(c *tls.Certificate) (*x509.Certificate, error) {
 	if c.Leaf != nil {
 		return c.Leaf, nil
 	}
-	return x509.ParseCertificate(c.Certificate[0])
+
+	leaf, err := x509.ParseCertificate(c.Certificate[0])
+	if err != nil {
+		return nil, fmt.Errorf("tlsCert - failed to parse leaf: %w", err)
+	}
+
+	c.Leaf = leaf
+
+	return leaf, err
 }

server/certificates/mock_test.go

@@ -3,13 +3,16 @@ package certificates
 import (
 	"testing"
 
-	"codeberg.org/codeberg/pages/server/database"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+
+	"codeberg.org/codeberg/pages/server/database"
 )
 
 func TestMockCert(t *testing.T) {
-	db, err := database.NewTmpDB()
-	assert.NoError(t, err)
+	db := database.NewMockCertDB(t)
+	db.Mock.On("Put", mock.Anything, mock.Anything).Return(nil)
+
 	cert, err := mockCert("example.com", "some error msg", "codeberg.page", db)
 	assert.NoError(t, err)
 	if assert.NotEmpty(t, cert) {

server/context/context.go

@@ -5,19 +5,29 @@ import (
 	"net/http"
 
 	"codeberg.org/codeberg/pages/server/utils"
+	"github.com/hashicorp/go-uuid"
+	"github.com/rs/zerolog/log"
 )
 
 type Context struct {
 	RespWriter http.ResponseWriter
 	Req        *http.Request
 	StatusCode int
+	ReqId      string
 }
 
 func New(w http.ResponseWriter, r *http.Request) *Context {
+	req_uuid, err := uuid.GenerateUUID()
+	if err != nil {
+		log.Error().Err(err).Msg("Failed to generate request id, assigning error value")
+		req_uuid = "ERROR"
+	}
+
 	return &Context{
 		RespWriter: w,
 		Req:        r,
 		StatusCode: http.StatusOK,
+		ReqId:      req_uuid,
 	}
 }
 
@@ -48,11 +58,9 @@ func (c *Context) Redirect(uri string, statusCode int) {
 	http.Redirect(c.RespWriter, c.Req, uri, statusCode)
 }
 
-// Path returns requested path.
-//
-// The returned bytes are valid until your request handler returns.
+// Path returns the cleaned requested path.
 func (c *Context) Path() string {
-	return c.Req.URL.Path
+	return utils.CleanPath(c.Req.URL.Path)
 }
 
 func (c *Context) Host() string {

server/database/interface.go

@@ -8,14 +8,15 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
+//go:generate go install github.com/vektra/mockery/v2@latest
+//go:generate mockery --name CertDB --output . --filename mock.go --inpackage --case underscore
+
 type CertDB interface {
 	Close() error
 	Put(name string, cert *certificate.Resource) error
 	Get(name string) (*certificate.Resource, error)
 	Delete(key string) error
 	Items(page, pageSize int) ([]*Cert, error)
-	// Compact deprecated // TODO: remove in next version
-	Compact() (string, error)
 }
 
 type Cert struct {

server/database/mock.go

@@ -1,54 +1,122 @@
+// Code generated by mockery v2.20.0. DO NOT EDIT.
+
 package database
 
 import (
-	"fmt"
-	"time"
-
-	"github.com/OrlovEvgeny/go-mcache"
-	"github.com/go-acme/lego/v4/certificate"
+	certificate "github.com/go-acme/lego/v4/certificate"
+	mock "github.com/stretchr/testify/mock"
 )
 
-var _ CertDB = tmpDB{}
-
-type tmpDB struct {
-	intern *mcache.CacheDriver
-	ttl    time.Duration
+// MockCertDB is an autogenerated mock type for the CertDB type
+type MockCertDB struct {
+	mock.Mock
 }
 
-func (p tmpDB) Close() error {
-	_ = p.intern.Close()
-	return nil
-}
+// Close provides a mock function with given fields:
+func (_m *MockCertDB) Close() error {
+	ret := _m.Called()
 
-func (p tmpDB) Put(name string, cert *certificate.Resource) error {
-	return p.intern.Set(name, cert, p.ttl)
-}
-
-func (p tmpDB) Get(name string) (*certificate.Resource, error) {
-	cert, has := p.intern.Get(name)
-	if !has {
-		return nil, fmt.Errorf("cert for %q not found", name)
+	var r0 error
+	if rf, ok := ret.Get(0).(func() error); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Error(0)
 	}
-	return cert.(*certificate.Resource), nil
+
+	return r0
 }
 
-func (p tmpDB) Delete(key string) error {
-	p.intern.Remove(key)
-	return nil
+// Delete provides a mock function with given fields: key
+func (_m *MockCertDB) Delete(key string) error {
+	ret := _m.Called(key)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string) error); ok {
+		r0 = rf(key)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
 }
 
-func (p tmpDB) Compact() (string, error) {
-	p.intern.Truncate()
-	return "Truncate done", nil
+// Get provides a mock function with given fields: name
+func (_m *MockCertDB) Get(name string) (*certificate.Resource, error) {
+	ret := _m.Called(name)
+
+	var r0 *certificate.Resource
+	var r1 error
+	if rf, ok := ret.Get(0).(func(string) (*certificate.Resource, error)); ok {
+		return rf(name)
+	}
+	if rf, ok := ret.Get(0).(func(string) *certificate.Resource); ok {
+		r0 = rf(name)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*certificate.Resource)
+		}
+	}
+
+	if rf, ok := ret.Get(1).(func(string) error); ok {
+		r1 = rf(name)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
 }
 
-func (p tmpDB) Items(page, pageSize int) ([]*Cert, error) {
-	return nil, fmt.Errorf("items not implemented for tmpDB")
+// Items provides a mock function with given fields: page, pageSize
+func (_m *MockCertDB) Items(page int, pageSize int) ([]*Cert, error) {
+	ret := _m.Called(page, pageSize)
+
+	var r0 []*Cert
+	var r1 error
+	if rf, ok := ret.Get(0).(func(int, int) ([]*Cert, error)); ok {
+		return rf(page, pageSize)
+	}
+	if rf, ok := ret.Get(0).(func(int, int) []*Cert); ok {
+		r0 = rf(page, pageSize)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]*Cert)
+		}
+	}
+
+	if rf, ok := ret.Get(1).(func(int, int) error); ok {
+		r1 = rf(page, pageSize)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
 }
 
-func NewTmpDB() (CertDB, error) {
-	return &tmpDB{
-		intern: mcache.New(),
-		ttl:    time.Minute,
-	}, nil
+// Put provides a mock function with given fields: name, cert
+func (_m *MockCertDB) Put(name string, cert *certificate.Resource) error {
+	ret := _m.Called(name, cert)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string, *certificate.Resource) error); ok {
+		r0 = rf(name, cert)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+type mockConstructorTestingTNewMockCertDB interface {
+	mock.TestingT
+	Cleanup(func())
+}
+
+// NewMockCertDB creates a new instance of MockCertDB. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
+func NewMockCertDB(t mockConstructorTestingTNewMockCertDB) *MockCertDB {
+	mock := &MockCertDB{}
+	mock.Mock.Test(t)
+
+	t.Cleanup(func() { mock.AssertExpectations(t) })
+
+	return mock
 }

server/database/pogreb.go

@@ -1,134 +0,0 @@
-package database
-
-import (
-	"bytes"
-	"context"
-	"encoding/gob"
-	"errors"
-	"fmt"
-	"time"
-
-	"github.com/akrylysov/pogreb"
-	"github.com/akrylysov/pogreb/fs"
-	"github.com/go-acme/lego/v4/certificate"
-	"github.com/rs/zerolog/log"
-)
-
-var _ CertDB = aDB{}
-
-type aDB struct {
-	ctx          context.Context
-	cancel       context.CancelFunc
-	intern       *pogreb.DB
-	syncInterval time.Duration
-}
-
-func (p aDB) Close() error {
-	p.cancel()
-	return p.intern.Sync()
-}
-
-func (p aDB) Put(name string, cert *certificate.Resource) error {
-	var resGob bytes.Buffer
-	if err := gob.NewEncoder(&resGob).Encode(cert); err != nil {
-		return err
-	}
-	return p.intern.Put([]byte(name), resGob.Bytes())
-}
-
-func (p aDB) Get(name string) (*certificate.Resource, error) {
-	cert := &certificate.Resource{}
-	resBytes, err := p.intern.Get([]byte(name))
-	if err != nil {
-		return nil, err
-	}
-	if resBytes == nil {
-		return nil, nil
-	}
-	if err := gob.NewDecoder(bytes.NewBuffer(resBytes)).Decode(cert); err != nil {
-		return nil, err
-	}
-	return cert, nil
-}
-
-func (p aDB) Delete(key string) error {
-	return p.intern.Delete([]byte(key))
-}
-
-func (p aDB) Compact() (string, error) {
-	result, err := p.intern.Compact()
-	if err != nil {
-		return "", err
-	}
-	return fmt.Sprintf("%+v", result), nil
-}
-
-func (p aDB) Items(_, _ int) ([]*Cert, error) {
-	items := make([]*Cert, 0, p.intern.Count())
-	iterator := p.intern.Items()
-	for {
-		key, resBytes, err := iterator.Next()
-		if err != nil {
-			if errors.Is(err, pogreb.ErrIterationDone) {
-				break
-			}
-			return nil, err
-		}
-
-		res := &certificate.Resource{}
-		if err := gob.NewDecoder(bytes.NewBuffer(resBytes)).Decode(res); err != nil {
-			return nil, err
-		}
-
-		cert, err := toCert(string(key), res)
-		if err != nil {
-			return nil, err
-		}
-
-		items = append(items, cert)
-	}
-
-	return items, nil
-}
-
-var _ CertDB = &aDB{}
-
-func (p aDB) sync() {
-	for {
-		err := p.intern.Sync()
-		if err != nil {
-			log.Error().Err(err).Msg("Syncing cert database failed")
-		}
-		select {
-		case <-p.ctx.Done():
-			return
-		case <-time.After(p.syncInterval):
-		}
-	}
-}
-
-func NewPogreb(path string) (CertDB, error) {
-	if path == "" {
-		return nil, fmt.Errorf("path not set")
-	}
-	db, err := pogreb.Open(path, &pogreb.Options{
-		BackgroundSyncInterval:       30 * time.Second,
-		BackgroundCompactionInterval: 6 * time.Hour,
-		FileSystem:                   fs.OSMMap,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	ctx, cancel := context.WithCancel(context.Background())
-	result := &aDB{
-		ctx:          ctx,
-		cancel:       cancel,
-		intern:       db,
-		syncInterval: 5 * time.Minute,
-	}
-
-	go result.sync()
-
-	return result, nil
-}

server/database/xorm.go

@@ -37,7 +37,7 @@ func NewXormDB(dbType, dbConn string) (CertDB, error) {
 	}
 
 	if err := e.Sync2(new(Cert)); err != nil {
-		return nil, fmt.Errorf("cound not sync db model :%w", err)
+		return nil, fmt.Errorf("could not sync db model :%w", err)
 	}
 
 	return &xDB{
@@ -52,7 +52,6 @@ func (x xDB) Close() error {
 func (x xDB) Put(domain string, cert *certificate.Resource) error {
 	log.Trace().Str("domain", cert.Domain).Msg("inserting cert to db")
 
-	domain = integrationTestReplacements(domain)
 	c, err := toCert(domain, cert)
 	if err != nil {
 		return err
@@ -64,7 +63,7 @@ func (x xDB) Put(domain string, cert *certificate.Resource) error {
 	}
 	defer sess.Close()
 
-	if exist, _ := sess.ID(c.Domain).Exist(); exist {
+	if exist, _ := sess.ID(c.Domain).Exist(new(Cert)); exist {
 		if _, err := sess.ID(c.Domain).Update(c); err != nil {
 			return err
 		}
@@ -82,7 +81,6 @@ func (x xDB) Get(domain string) (*certificate.Resource, error) {
 	if domain[:1] == "." {
 		domain = "*" + domain
 	}
-	domain = integrationTestReplacements(domain)
 
 	cert := new(Cert)
 	log.Trace().Str("domain", domain).Msg("get cert from db")
@@ -99,18 +97,12 @@ func (x xDB) Delete(domain string) error {
 	if domain[:1] == "." {
 		domain = "*" + domain
 	}
-	domain = integrationTestReplacements(domain)
 
 	log.Trace().Str("domain", domain).Msg("delete cert from db")
 	_, err := x.engine.ID(domain).Delete(new(Cert))
 	return err
 }
 
-func (x xDB) Compact() (string, error) {
-	// not needed
-	return "", nil
-}
-
 // Items return al certs from db, if pageSize is 0 it does not use limit
 func (x xDB) Items(page, pageSize int) ([]*Cert, error) {
 	// paginated return
@@ -144,13 +136,3 @@ func supportedDriver(driver string) bool {
 		return false
 	}
 }
-
-// integrationTestReplacements is needed because integration tests use a single domain cert,
-// while production use a wildcard cert
-// TODO: find a better way to handle this
-func integrationTestReplacements(domainKey string) string {
-	if domainKey == "*.localhost.mock.directory" {
-		return "localhost.mock.directory"
-	}
-	return domainKey
-}

server/database/xorm_test.go

@@ -37,7 +37,7 @@ func TestSanitizeWildcardCerts(t *testing.T) {
 	}))
 
 	// update existing cert
-	assert.Error(t, certDB.Put(".wildcard.de", &certificate.Resource{
+	assert.NoError(t, certDB.Put(".wildcard.de", &certificate.Resource{
 		Domain:      "*.wildcard.de",
 		Certificate: localhost_mock_directory_certificate,
 	}))

server/dns/dns.go

@@ -5,20 +5,26 @@ import (
 	"strings"
 	"time"
 
-	"codeberg.org/codeberg/pages/server/cache"
+	"github.com/hashicorp/golang-lru/v2/expirable"
 )
 
-// lookupCacheTimeout specifies the timeout for the DNS lookup cache.
-var lookupCacheTimeout = 15 * time.Minute
+const (
+	lookupCacheValidity = 30 * time.Second
+	defaultPagesRepo    = "pages"
+)
+
+// TODO(#316): refactor to not use global variables
+var lookupCache *expirable.LRU[string, string] = expirable.NewLRU[string, string](4096, nil, lookupCacheValidity)
 
 // GetTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix.
 // If everything is fine, it returns the target data.
-func GetTargetFromDNS(domain, mainDomainSuffix string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) {
+func GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch string) (targetOwner, targetRepo, targetBranch string) {
 	// Get CNAME or TXT
 	var cname string
 	var err error
-	if cachedName, ok := dnsLookupCache.Get(domain); ok {
-		cname = cachedName.(string)
+
+	if entry, ok := lookupCache.Get(domain); ok {
+		cname = entry
 	} else {
 		cname, err = net.LookupCNAME(domain)
 		cname = strings.TrimSuffix(cname, ".")
@@ -36,7 +42,7 @@ func GetTargetFromDNS(domain, mainDomainSuffix string, dnsLookupCache cache.SetG
 				}
 			}
 		}
-		_ = dnsLookupCache.Set(domain, cname, lookupCacheTimeout)
+		_ = lookupCache.Add(domain, cname)
 	}
 	if cname == "" {
 		return
@@ -50,10 +56,10 @@ func GetTargetFromDNS(domain, mainDomainSuffix string, dnsLookupCache cache.SetG
 		targetBranch = cnameParts[len(cnameParts)-3]
 	}
 	if targetRepo == "" {
-		targetRepo = "pages"
+		targetRepo = defaultPagesRepo
 	}
-	if targetBranch == "" && targetRepo != "pages" {
-		targetBranch = "pages"
+	if targetBranch == "" && targetRepo != defaultPagesRepo {
+		targetBranch = firstDefaultBranch
 	}
 	// if targetBranch is still empty, the caller must find the default branch
 	return

server/gitea/cache.go

@@ -2,14 +2,17 @@ package gitea
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
 	"time"
 
+	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 
 	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/context"
 )
 
 const (
@@ -26,23 +29,27 @@ const (
 	// TODO: move as option into cache interface
 	fileCacheTimeout = 5 * time.Minute
 
+	// ownerExistenceCacheTimeout specifies the timeout for the existence of a repo/org
+	ownerExistenceCacheTimeout = 5 * time.Minute
+
 	// fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default.
 	fileCacheSizeLimit = int64(1000 * 1000)
 )
 
 type FileResponse struct {
-	Exists    bool
-	IsSymlink bool
-	ETag      string
-	MimeType  string
-	Body      []byte
+	Exists    bool   `json:"exists"`
+	IsSymlink bool   `json:"isSymlink"`
+	ETag      string `json:"eTag"`
+	MimeType  string `json:"mimeType"` // uncompressed MIME type
+	RawMime   string `json:"rawMime"`  // raw MIME type (if compressed, type of compression)
+	Body      []byte `json:"-"`        // saved separately
 }
 
 func (f FileResponse) IsEmpty() bool {
-	return len(f.Body) != 0
+	return len(f.Body) == 0
 }
 
-func (f FileResponse) createHttpResponse(cacheKey string) (header http.Header, statusCode int) {
+func (f FileResponse) createHttpResponse(cacheKey string, decompress bool) (header http.Header, statusCode int) {
 	header = make(http.Header)
 
 	if f.Exists {
@@ -55,7 +62,13 @@ func (f FileResponse) createHttpResponse(cacheKey string) (header http.Header, s
 		header.Set(giteaObjectTypeHeader, objTypeSymlink)
 	}
 	header.Set(ETagHeader, f.ETag)
-	header.Set(ContentTypeHeader, f.MimeType)
+
+	if decompress {
+		header.Set(ContentTypeHeader, f.MimeType)
+	} else {
+		header.Set(ContentTypeHeader, f.RawMime)
+	}
+
 	header.Set(ContentLengthHeader, fmt.Sprintf("%d", len(f.Body)))
 	header.Set(PagesCacheIndicatorHeader, "true")
 
@@ -64,42 +77,67 @@ func (f FileResponse) createHttpResponse(cacheKey string) (header http.Header, s
 }
 
 type BranchTimestamp struct {
-	Branch    string
-	Timestamp time.Time
-	notFound  bool
+	NotFound  bool      `json:"notFound"`
+	Branch    string    `json:"branch,omitempty"`
+	Timestamp time.Time `json:"timestamp,omitempty"`
 }
 
 type writeCacheReader struct {
 	originalReader io.ReadCloser
 	buffer         *bytes.Buffer
-	rileResponse   *FileResponse
+	fileResponse   *FileResponse
 	cacheKey       string
-	cache          cache.SetGetKey
+	cache          cache.ICache
 	hasError       bool
+	doNotCache     bool
+	complete       bool
+	log            zerolog.Logger
 }
 
 func (t *writeCacheReader) Read(p []byte) (n int, err error) {
+	t.log.Trace().Msgf("[cache] read %q", t.cacheKey)
 	n, err = t.originalReader.Read(p)
+	if err == io.EOF {
+		t.complete = true
+	}
 	if err != nil && err != io.EOF {
-		log.Trace().Err(err).Msgf("[cache] original reader for %q has returned an error", t.cacheKey)
+		t.log.Trace().Err(err).Msgf("[cache] original reader for %q has returned an error", t.cacheKey)
 		t.hasError = true
 	} else if n > 0 {
-		_, _ = t.buffer.Write(p[:n])
+		if t.buffer.Len()+n > int(fileCacheSizeLimit) {
+			t.doNotCache = true
+			t.buffer.Reset()
+		} else {
+			_, _ = t.buffer.Write(p[:n])
+		}
 	}
 	return
 }
 
 func (t *writeCacheReader) Close() error {
-	if !t.hasError {
-		fc := *t.rileResponse
-		fc.Body = t.buffer.Bytes()
-		_ = t.cache.Set(t.cacheKey, fc, fileCacheTimeout)
+	doWrite := !t.hasError && !t.doNotCache && t.complete
+	fc := *t.fileResponse
+	fc.Body = t.buffer.Bytes()
+	if doWrite {
+		jsonToCache, err := json.Marshal(fc)
+		if err != nil {
+			t.log.Trace().Err(err).Msgf("[cache] marshaling json for %q has returned an error", t.cacheKey+"|Metadata")
+		}
+		err = t.cache.Set(t.cacheKey+"|Metadata", jsonToCache, fileCacheTimeout)
+		if err != nil {
+			t.log.Trace().Err(err).Msgf("[cache] writer for %q has returned an error", t.cacheKey+"|Metadata")
+		}
+		err = t.cache.Set(t.cacheKey+"|Body", fc.Body, fileCacheTimeout)
+		if err != nil {
+			t.log.Trace().Err(err).Msgf("[cache] writer for %q has returned an error", t.cacheKey+"|Body")
+		}
 	}
-	log.Trace().Msgf("cacheReader for %q saved=%t closed", t.cacheKey, !t.hasError)
+	t.log.Trace().Msgf("cacheReader for %q saved=%t closed", t.cacheKey, doWrite)
 	return t.originalReader.Close()
 }
 
-func (f FileResponse) CreateCacheReader(r io.ReadCloser, cache cache.SetGetKey, cacheKey string) io.ReadCloser {
+func (f FileResponse) CreateCacheReader(ctx *context.Context, r io.ReadCloser, cache cache.ICache, cacheKey string) io.ReadCloser {
+	log := log.With().Str("ReqId", ctx.ReqId).Logger()
 	if r == nil || cache == nil || cacheKey == "" {
 		log.Error().Msg("could not create CacheReader")
 		return nil
@@ -108,8 +146,9 @@ func (f FileResponse) CreateCacheReader(r io.ReadCloser, cache cache.SetGetKey,
 	return &writeCacheReader{
 		originalReader: r,
 		buffer:         bytes.NewBuffer(make([]byte, 0)),
-		rileResponse:   &f,
+		fileResponse:   &f,
 		cache:          cache,
 		cacheKey:       cacheKey,
+		log:            log,
 	}
 }

server/gitea/client.go

@@ -2,6 +2,7 @@ package gitea
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -16,16 +17,20 @@ import (
 	"code.gitea.io/sdk/gitea"
 	"github.com/rs/zerolog/log"
 
+	"codeberg.org/codeberg/pages/config"
 	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/context"
+	"codeberg.org/codeberg/pages/server/version"
 )
 
 var ErrorNotFound = errors.New("not found")
 
 const (
-	// cache key prefixe
+	// cache key prefixes
 	branchTimestampCacheKeyPrefix = "branchTime"
 	defaultBranchCacheKeyPrefix   = "defaultBranch"
 	rawContentCacheKeyPrefix      = "rawContent"
+	ownerExistenceKeyPrefix       = "ownerExist"
 
 	// pages server
 	PagesCacheIndicatorHeader = "X-Pages-Cache"
@@ -36,14 +41,16 @@ const (
 	objTypeSymlink        = "symlink"
 
 	// std
-	ETagHeader          = "ETag"
-	ContentTypeHeader   = "Content-Type"
-	ContentLengthHeader = "Content-Length"
+	ETagHeader            = "ETag"
+	ContentTypeHeader     = "Content-Type"
+	ContentLengthHeader   = "Content-Length"
+	ContentEncodingHeader = "Content-Encoding"
 )
 
 type Client struct {
 	sdkClient     *gitea.Client
-	responseCache cache.SetGetKey
+	sdkFileClient *gitea.Client
+	responseCache cache.ICache
 
 	giteaRoot string
 
@@ -54,37 +61,50 @@ type Client struct {
 	defaultMimeType    string
 }
 
-func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, followSymlinks, supportLFS bool) (*Client, error) {
-	rootURL, err := url.Parse(giteaRoot)
+func NewClient(cfg config.ForgeConfig, respCache cache.ICache) (*Client, error) {
+	// url.Parse returns valid on almost anything...
+	rootURL, err := url.ParseRequestURI(cfg.Root)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("invalid forgejo/gitea root url: %w", err)
 	}
-	giteaRoot = strings.Trim(rootURL.String(), "/")
+	giteaRoot := strings.TrimSuffix(rootURL.String(), "/")
 
-	stdClient := http.Client{Timeout: 10 * time.Second}
-
-	// TODO: pass down
-	var (
-		forbiddenMimeTypes map[string]bool
-		defaultMimeType    string
-	)
-
-	if forbiddenMimeTypes == nil {
-		forbiddenMimeTypes = make(map[string]bool)
+	forbiddenMimeTypes := make(map[string]bool, len(cfg.ForbiddenMimeTypes))
+	for _, mimeType := range cfg.ForbiddenMimeTypes {
+		forbiddenMimeTypes[mimeType] = true
 	}
+
+	defaultMimeType := cfg.DefaultMimeType
 	if defaultMimeType == "" {
 		defaultMimeType = "application/octet-stream"
 	}
 
-	sdk, err := gitea.NewClient(giteaRoot, gitea.SetHTTPClient(&stdClient), gitea.SetToken(giteaAPIToken))
+	sdkClient, err := gitea.NewClient(
+		giteaRoot,
+		gitea.SetHTTPClient(&http.Client{Timeout: 10 * time.Second}),
+		gitea.SetToken(cfg.Token),
+		gitea.SetUserAgent("pages-server/"+version.Version),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	sdkFileClient, err := gitea.NewClient(
+		giteaRoot,
+		gitea.SetHTTPClient(&http.Client{Timeout: 1 * time.Hour}),
+		gitea.SetToken(cfg.Token),
+		gitea.SetUserAgent("pages-server/"+version.Version),
+	)
+
 	return &Client{
-		sdkClient:     sdk,
+		sdkClient:     sdkClient,
+		sdkFileClient: sdkFileClient,
 		responseCache: respCache,
 
 		giteaRoot: giteaRoot,
 
-		followSymlinks: followSymlinks,
-		supportLFS:     supportLFS,
+		followSymlinks: cfg.FollowSymlinks,
+		supportLFS:     cfg.LFSEnabled,
 
 		forbiddenMimeTypes: forbiddenMimeTypes,
 		defaultMimeType:    defaultMimeType,
@@ -95,8 +115,8 @@ func (client *Client) ContentWebLink(targetOwner, targetRepo, branch, resource s
 	return path.Join(client.giteaRoot, targetOwner, targetRepo, "src/branch", branch, resource)
 }
 
-func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource string) ([]byte, error) {
-	reader, _, _, err := client.ServeRawContent(targetOwner, targetRepo, ref, resource)
+func (client *Client) GiteaRawContent(ctx *context.Context, targetOwner, targetRepo, ref, resource string) ([]byte, error) {
+	reader, _, _, err := client.ServeRawContent(ctx, targetOwner, targetRepo, ref, resource, false)
 	if err != nil {
 		return nil, err
 	}
@@ -104,31 +124,47 @@ func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource str
 	return io.ReadAll(reader)
 }
 
-func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource string) (io.ReadCloser, http.Header, int, error) {
+func (client *Client) ServeRawContent(ctx *context.Context, targetOwner, targetRepo, ref, resource string, decompress bool) (io.ReadCloser, http.Header, int, error) {
 	cacheKey := fmt.Sprintf("%s/%s/%s|%s|%s", rawContentCacheKeyPrefix, targetOwner, targetRepo, ref, resource)
-	log := log.With().Str("cache_key", cacheKey).Logger()
-
+	log := log.With().Str("ReqId", ctx.ReqId).Str("cache_key", cacheKey).Logger()
+	log.Trace().Msg("try file in cache")
 	// handle if cache entry exist
-	if cache, ok := client.responseCache.Get(cacheKey); ok {
-		cache := cache.(FileResponse)
-		cachedHeader, cachedStatusCode := cache.createHttpResponse(cacheKey)
-		// TODO: check against some timestamp missmatch?!?
+	if cacheMetadata, ok := client.responseCache.Get(cacheKey + "|Metadata"); ok {
+		var cache FileResponse
+		err := json.Unmarshal(cacheMetadata.([]byte), &cache)
+		if err != nil {
+			log.Error().Err(err).Msgf("[cache] failed to unmarshal metadata for: %s", cacheKey)
+			return nil, nil, http.StatusNotFound, err
+		}
+
+		if !cache.Exists {
+			return nil, nil, http.StatusNotFound, ErrorNotFound
+		}
+
+		body, ok := client.responseCache.Get(cacheKey + "|Body")
+		if !ok {
+			log.Error().Msgf("[cache] failed to get body for: %s", cacheKey)
+			return nil, nil, http.StatusNotFound, ErrorNotFound
+		}
+		cache.Body = body.([]byte)
+
+		cachedHeader, cachedStatusCode := cache.createHttpResponse(cacheKey, decompress)
 		if cache.Exists {
 			if cache.IsSymlink {
 				linkDest := string(cache.Body)
 				log.Debug().Msgf("[cache] follow symlink from %q to %q", resource, linkDest)
-				return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest)
+				return client.ServeRawContent(ctx, targetOwner, targetRepo, ref, linkDest, decompress)
 			} else {
-				log.Debug().Msg("[cache] return bytes")
+				log.Debug().Msgf("[cache] return %d bytes", len(cache.Body))
 				return io.NopCloser(bytes.NewReader(cache.Body)), cachedHeader, cachedStatusCode, nil
 			}
 		} else {
-			return nil, cachedHeader, cachedStatusCode, ErrorNotFound
+			return nil, nil, http.StatusNotFound, ErrorNotFound
 		}
 	}
-
+	log.Trace().Msg("file not in cache")
 	// not in cache, open reader via gitea api
-	reader, resp, err := client.sdkClient.GetFileReader(targetOwner, targetRepo, ref, resource, client.supportLFS)
+	reader, resp, err := client.sdkFileClient.GetFileReader(targetOwner, targetRepo, ref, resource, client.supportLFS)
 	if resp != nil {
 		switch resp.StatusCode {
 		case http.StatusOK:
@@ -145,42 +181,58 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
 					}
 					linkDest := strings.TrimSpace(string(linkDestBytes))
 
+					// handle relative links
+					// we first remove the link from the path, and make a relative join (resolve parent paths like "/../" too)
+					linkDest = path.Join(path.Dir(resource), linkDest)
+
 					// we store symlink not content to reduce duplicates in cache
-					if err := client.responseCache.Set(cacheKey, FileResponse{
+					fileResponse := FileResponse{
 						Exists:    true,
 						IsSymlink: true,
 						Body:      []byte(linkDest),
 						ETag:      resp.Header.Get(ETagHeader),
-					}, fileCacheTimeout); err != nil {
+					}
+					log.Trace().Msgf("file response has %d bytes", len(fileResponse.Body))
+					jsonToCache, err := json.Marshal(fileResponse)
+					if err != nil {
+						log.Error().Err(err).Msgf("[cache] marshaling json metadata for %q has returned an error", cacheKey)
+					}
+					if err := client.responseCache.Set(cacheKey+"|Metadata", jsonToCache, fileCacheTimeout); err != nil {
+						log.Error().Err(err).Msg("[cache] error on cache write")
+					}
+					if err := client.responseCache.Set(cacheKey+"|Body", fileResponse.Body, fileCacheTimeout); err != nil {
 						log.Error().Err(err).Msg("[cache] error on cache write")
 					}
 
 					log.Debug().Msgf("follow symlink from %q to %q", resource, linkDest)
-					return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest)
+					return client.ServeRawContent(ctx, targetOwner, targetRepo, ref, linkDest, decompress)
 				}
 			}
 
 			// now we are sure it's content so set the MIME type
-			mimeType := client.getMimeTypeByExtension(resource)
+			mimeType, rawType := client.getMimeTypeByExtension(resource)
 			resp.Response.Header.Set(ContentTypeHeader, mimeType)
-
-			if !shouldRespBeSavedToCache(resp.Response) {
-				return reader, resp.Response.Header, resp.StatusCode, err
+			if decompress {
+				resp.Response.Header.Set(ContentTypeHeader, mimeType)
+			} else {
+				resp.Response.Header.Set(ContentTypeHeader, rawType)
 			}
 
-			// now we write to cache and respond at the sime time
+			// now we write to cache and respond at the same time
 			fileResp := FileResponse{
 				Exists:   true,
 				ETag:     resp.Header.Get(ETagHeader),
 				MimeType: mimeType,
+				RawMime:  rawType,
 			}
-			return fileResp.CreateCacheReader(reader, client.responseCache, cacheKey), resp.Response.Header, resp.StatusCode, nil
+			return fileResp.CreateCacheReader(ctx, reader, client.responseCache, cacheKey), resp.Response.Header, resp.StatusCode, nil
 
 		case http.StatusNotFound:
-			if err := client.responseCache.Set(cacheKey, FileResponse{
-				Exists: false,
-				ETag:   resp.Header.Get(ETagHeader),
-			}, fileCacheTimeout); err != nil {
+			jsonToCache, err := json.Marshal(FileResponse{ETag: resp.Header.Get(ETagHeader)})
+			if err != nil {
+				log.Error().Err(err).Msgf("[cache] marshaling json metadata for %q has returned an error", cacheKey)
+			}
+			if err := client.responseCache.Set(cacheKey+"|Metadata", jsonToCache, fileCacheTimeout); err != nil {
 				log.Error().Err(err).Msg("[cache] error on cache write")
 			}
 
@@ -195,21 +247,36 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
 func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchName string) (*BranchTimestamp, error) {
 	cacheKey := fmt.Sprintf("%s/%s/%s/%s", branchTimestampCacheKeyPrefix, repoOwner, repoName, branchName)
 
-	if stamp, ok := client.responseCache.Get(cacheKey); ok && stamp != nil {
-		branchTimeStamp := stamp.(*BranchTimestamp)
-		if branchTimeStamp.notFound {
-			log.Trace().Msgf("[cache] use branch %q not found", branchName)
+	if stampRaw, ok := client.responseCache.Get(cacheKey); ok {
+		var stamp BranchTimestamp
+		err := json.Unmarshal(stampRaw.([]byte), &stamp)
+		if err != nil {
+			log.Error().Err(err).Bytes("stamp", stampRaw.([]byte)).Msgf("[cache] failed to unmarshal timestamp for: %s", cacheKey)
 			return &BranchTimestamp{}, ErrorNotFound
 		}
-		log.Trace().Msgf("[cache] use branch %q exist", branchName)
-		return branchTimeStamp, nil
+
+		if stamp.NotFound {
+			log.Trace().Msgf("[cache] branch %q does not exist", branchName)
+
+			return &BranchTimestamp{}, ErrorNotFound
+		} else {
+			log.Trace().Msgf("[cache] use branch %q exist", branchName)
+			// This comes from the refactoring of the caching library.
+			// The branch as reported by the API was stored in the cache, and I'm not sure if there are
+			// situations where it differs from the name in the request, hence this is left here.
+			return &stamp, nil
+		}
 	}
 
 	branch, resp, err := client.sdkClient.GetRepoBranch(repoOwner, repoName, branchName)
 	if err != nil {
 		if resp != nil && resp.StatusCode == http.StatusNotFound {
 			log.Trace().Msgf("[cache] set cache branch %q not found", branchName)
-			if err := client.responseCache.Set(cacheKey, &BranchTimestamp{Branch: branchName, notFound: true}, branchExistenceCacheTimeout); err != nil {
+			jsonToCache, err := json.Marshal(BranchTimestamp{NotFound: true})
+			if err != nil {
+				log.Error().Err(err).Msgf("[cache] marshaling empty timestamp for '%s' has returned an error", cacheKey)
+			}
+			if err := client.responseCache.Set(cacheKey, jsonToCache, branchExistenceCacheTimeout); err != nil {
 				log.Error().Err(err).Msg("[cache] error on cache write")
 			}
 			return &BranchTimestamp{}, ErrorNotFound
@@ -226,7 +293,11 @@ func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchNam
 	}
 
 	log.Trace().Msgf("set cache branch [%s] exist", branchName)
-	if err := client.responseCache.Set(cacheKey, stamp, branchExistenceCacheTimeout); err != nil {
+	jsonToCache, err := json.Marshal(stamp)
+	if err != nil {
+		log.Error().Err(err).Msgf("[cache] marshaling timestamp for %q has returned an error", cacheKey)
+	}
+	if err := client.responseCache.Set(cacheKey, jsonToCache, branchExistenceCacheTimeout); err != nil {
 		log.Error().Err(err).Msg("[cache] error on cache write")
 	}
 	return stamp, nil
@@ -235,8 +306,8 @@ func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchNam
 func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (string, error) {
 	cacheKey := fmt.Sprintf("%s/%s/%s", defaultBranchCacheKeyPrefix, repoOwner, repoName)
 
-	if branch, ok := client.responseCache.Get(cacheKey); ok && branch != nil {
-		return branch.(string), nil
+	if branch, ok := client.responseCache.Get(cacheKey); ok {
+		return string(branch.([]byte)), nil
 	}
 
 	repo, resp, err := client.sdkClient.GetRepo(repoOwner, repoName)
@@ -248,37 +319,68 @@ func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (str
 	}
 
 	branch := repo.DefaultBranch
-	if err := client.responseCache.Set(cacheKey, branch, defaultBranchCacheTimeout); err != nil {
+	if err := client.responseCache.Set(cacheKey, []byte(branch), defaultBranchCacheTimeout); err != nil {
 		log.Error().Err(err).Msg("[cache] error on cache write")
 	}
 	return branch, nil
 }
 
-func (client *Client) getMimeTypeByExtension(resource string) string {
-	mimeType := mime.TypeByExtension(path.Ext(resource))
+func (client *Client) GiteaCheckIfOwnerExists(owner string) (bool, error) {
+	cacheKey := fmt.Sprintf("%s/%s", ownerExistenceKeyPrefix, owner)
+
+	if existRaw, ok := client.responseCache.Get(cacheKey); ok && existRaw != nil {
+		exist, err := strconv.ParseBool(existRaw.(string))
+		return exist, err
+	}
+
+	_, resp, err := client.sdkClient.GetUserInfo(owner)
+	if resp.StatusCode == http.StatusOK && err == nil {
+		if err := client.responseCache.Set(cacheKey, []byte("true"), ownerExistenceCacheTimeout); err != nil {
+			log.Error().Err(err).Msg("[cache] error on cache write")
+		}
+		return true, nil
+	} else if resp.StatusCode != http.StatusNotFound {
+		return false, err
+	}
+
+	_, resp, err = client.sdkClient.GetOrg(owner)
+	if resp.StatusCode == http.StatusOK && err == nil {
+		if err := client.responseCache.Set(cacheKey, []byte("true"), ownerExistenceCacheTimeout); err != nil {
+			log.Error().Err(err).Msg("[cache] error on cache write")
+		}
+		return true, nil
+	} else if resp.StatusCode != http.StatusNotFound {
+		return false, err
+	}
+	if err := client.responseCache.Set(cacheKey, []byte("false"), ownerExistenceCacheTimeout); err != nil {
+		log.Error().Err(err).Msg("[cache] error on cache write")
+	}
+	return false, nil
+}
+
+func (client *Client) extToMime(ext string) string {
+	mimeType := mime.TypeByExtension(path.Ext(ext))
 	mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
 	if client.forbiddenMimeTypes[mimeTypeSplit[0]] || mimeType == "" {
 		mimeType = client.defaultMimeType
 	}
-	log.Trace().Msgf("probe mime of %q is %q", resource, mimeType)
+	log.Trace().Msgf("probe mime of extension '%q' is '%q'", ext, mimeType)
+
 	return mimeType
 }
 
-func shouldRespBeSavedToCache(resp *http.Response) bool {
-	if resp == nil {
-		return false
+func (client *Client) getMimeTypeByExtension(resource string) (mimeType, rawType string) {
+	rawExt := path.Ext(resource)
+	innerExt := rawExt
+	switch rawExt {
+	case ".gz", ".br", ".zst":
+		innerExt = path.Ext(resource[:len(resource)-len(rawExt)])
 	}
-
-	contentLengthRaw := resp.Header.Get(ContentLengthHeader)
-	if contentLengthRaw == "" {
-		return false
+	rawType = client.extToMime(rawExt)
+	mimeType = rawType
+	if innerExt != rawExt {
+		mimeType = client.extToMime(innerExt)
 	}
-
-	contentLeng, err := strconv.ParseInt(contentLengthRaw, 10, 64)
-	if err != nil {
-		log.Error().Err(err).Msg("could not parse content length")
-	}
-
-	// if content to big or could not be determined we not cache it
-	return contentLeng > 0 && contentLeng < fileCacheSizeLimit
+	log.Trace().Msgf("probe mime of %q is (%q / raw %q)", resource, mimeType, rawType)
+	return mimeType, rawType
 }

server/handler/handler.go

@@ -6,32 +6,31 @@ import (
 
 	"github.com/rs/zerolog/log"
 
+	"codeberg.org/codeberg/pages/config"
 	"codeberg.org/codeberg/pages/html"
 	"codeberg.org/codeberg/pages/server/cache"
 	"codeberg.org/codeberg/pages/server/context"
 	"codeberg.org/codeberg/pages/server/gitea"
-	"codeberg.org/codeberg/pages/server/version"
 )
 
 const (
 	headerAccessControlAllowOrigin  = "Access-Control-Allow-Origin"
 	headerAccessControlAllowMethods = "Access-Control-Allow-Methods"
 	defaultPagesRepo                = "pages"
-	defaultPagesBranch              = "pages"
 )
 
 // Handler handles a single HTTP request to the web server.
-func Handler(mainDomainSuffix, rawDomain string,
+func Handler(
+	cfg config.ServerConfig,
 	giteaClient *gitea.Client,
-	rawInfoPage string,
-	blacklistedPaths, allowedCorsDomains []string,
-	dnsLookupCache, canonicalDomainCache cache.SetGetKey,
+	canonicalDomainCache, redirectsCache cache.ICache,
 ) http.HandlerFunc {
 	return func(w http.ResponseWriter, req *http.Request) {
-		log := log.With().Strs("Handler", []string{req.Host, req.RequestURI}).Logger()
 		ctx := context.New(w, req)
+		log := log.With().Str("ReqId", ctx.ReqId).Strs("Handler", []string{req.Host, req.RequestURI}).Logger()
+		log.Debug().Msg("\n----------------------------------------------------------")
 
-		ctx.RespWriter.Header().Set("Server", "CodebergPages/"+version.Version)
+		ctx.RespWriter.Header().Set("Server", "pages-server")
 
 		// Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin
 		ctx.RespWriter.Header().Set("Referrer-Policy", "strict-origin-when-cross-origin")
@@ -41,8 +40,8 @@ func Handler(mainDomainSuffix, rawDomain string,
 
 		trimmedHost := ctx.TrimHostPort()
 
-		// Add HSTS for RawDomain and MainDomainSuffix
-		if hsts := getHSTSHeader(trimmedHost, mainDomainSuffix, rawDomain); hsts != "" {
+		// Add HSTS for RawDomain and MainDomain
+		if hsts := getHSTSHeader(trimmedHost, cfg.MainDomain, cfg.RawDomain); hsts != "" {
 			ctx.RespWriter.Header().Set("Strict-Transport-Security", hsts)
 		}
 
@@ -64,16 +63,16 @@ func Handler(mainDomainSuffix, rawDomain string,
 		}
 
 		// Block blacklisted paths (like ACME challenges)
-		for _, blacklistedPath := range blacklistedPaths {
+		for _, blacklistedPath := range cfg.BlacklistedPaths {
 			if strings.HasPrefix(ctx.Path(), blacklistedPath) {
-				html.ReturnErrorPage(ctx, "requested blacklisted path", http.StatusForbidden)
+				html.ReturnErrorPage(ctx, "requested path is blacklisted", http.StatusForbidden)
 				return
 			}
 		}
 
 		// Allow CORS for specified domains
 		allowCors := false
-		for _, allowedCorsDomain := range allowedCorsDomains {
+		for _, allowedCorsDomain := range cfg.AllowedCorsDomains {
 			if strings.EqualFold(trimmedHost, allowedCorsDomain) {
 				allowCors = true
 				break
@@ -87,27 +86,29 @@ func Handler(mainDomainSuffix, rawDomain string,
 		// Prepare request information to Gitea
 		pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/")
 
-		if rawDomain != "" && strings.EqualFold(trimmedHost, rawDomain) {
-			log.Debug().Msg("raw domain request detecded")
+		if cfg.RawDomain != "" && strings.EqualFold(trimmedHost, cfg.RawDomain) {
+			log.Debug().Msg("raw domain request detected")
 			handleRaw(log, ctx, giteaClient,
-				mainDomainSuffix, rawInfoPage,
+				cfg.MainDomain,
 				trimmedHost,
 				pathElements,
-				canonicalDomainCache)
-		} else if strings.HasSuffix(trimmedHost, mainDomainSuffix) {
-			log.Debug().Msg("subdomain request detecded")
+				canonicalDomainCache, redirectsCache)
+		} else if strings.HasSuffix(trimmedHost, cfg.MainDomain) {
+			log.Debug().Msg("subdomain request detected")
 			handleSubDomain(log, ctx, giteaClient,
-				mainDomainSuffix,
+				cfg.MainDomain,
+				cfg.PagesBranches,
 				trimmedHost,
 				pathElements,
-				canonicalDomainCache)
+				canonicalDomainCache, redirectsCache)
 		} else {
-			log.Debug().Msg("custom domain request detecded")
+			log.Debug().Msg("custom domain request detected")
 			handleCustomDomain(log, ctx, giteaClient,
-				mainDomainSuffix,
+				cfg.MainDomain,
 				trimmedHost,
 				pathElements,
-				dnsLookupCache, canonicalDomainCache)
+				cfg.PagesBranches[0],
+				canonicalDomainCache, redirectsCache)
 		}
 	}
 }

server/handler/handler_custom_domain.go

@@ -18,10 +18,11 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g
 	mainDomainSuffix string,
 	trimmedHost string,
 	pathElements []string,
-	dnsLookupCache, canonicalDomainCache cache.SetGetKey,
+	firstDefaultBranch string,
+	canonicalDomainCache, redirectsCache cache.ICache,
 ) {
 	// Serve pages from custom domains
-	targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, mainDomainSuffix, dnsLookupCache)
+	targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, mainDomainSuffix, firstDefaultBranch)
 	if targetOwner == "" {
 		html.ReturnErrorPage(ctx,
 			"could not obtain repo owner from custom domain",
@@ -46,13 +47,13 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g
 		TargetBranch:  targetBranch,
 		TargetPath:    path.Join(pathParts...),
 	}, canonicalLink); works {
-		canonicalDomain, valid := targetOpt.CheckCanonicalDomain(giteaClient, trimmedHost, mainDomainSuffix, canonicalDomainCache)
+		canonicalDomain, valid := targetOpt.CheckCanonicalDomain(ctx, giteaClient, trimmedHost, mainDomainSuffix, canonicalDomainCache)
 		if !valid {
 			html.ReturnErrorPage(ctx, "domain not specified in <code>.domains</code> file", http.StatusMisdirectedRequest)
 			return
 		} else if canonicalDomain != trimmedHost {
 			// only redirect if the target is also a codeberg page!
-			targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix, dnsLookupCache)
+			targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix, firstDefaultBranch)
 			if targetOwner != "" {
 				ctx.Redirect("https://"+canonicalDomain+"/"+targetOpt.TargetPath, http.StatusTemporaryRedirect)
 				return
@@ -62,8 +63,8 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g
 			return
 		}
 
-		log.Debug().Msg("tryBranch, now trying upstream 7")
-		tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
+		log.Debug().Str("url", trimmedHost).Msg("tryBranch, now trying upstream")
+		tryUpstream(log, ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
 		return
 	}
 

server/handler/handler_raw_domain.go

@@ -16,17 +16,21 @@ import (
 )
 
 func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client,
-	mainDomainSuffix, rawInfoPage string,
+	mainDomainSuffix string,
 	trimmedHost string,
 	pathElements []string,
-	canonicalDomainCache cache.SetGetKey,
+	canonicalDomainCache, redirectsCache cache.ICache,
 ) {
 	// Serve raw content from RawDomain
 	log.Debug().Msg("raw domain")
 
 	if len(pathElements) < 2 {
-		// https://{RawDomain}/{owner}/{repo}[/@{branch}]/{path} is required
-		ctx.Redirect(rawInfoPage, http.StatusTemporaryRedirect)
+		html.ReturnErrorPage(
+			ctx,
+			"a url in the form of <code>https://{domain}/{owner}/{repo}[/@{branch}]/{path}</code> is required",
+			http.StatusBadRequest,
+		)
+
 		return
 	}
 
@@ -41,7 +45,7 @@ func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Clie
 			TargetPath:   path.Join(pathElements[3:]...),
 		}, true); works {
 			log.Trace().Msg("tryUpstream: serve raw domain with specified branch")
-			tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
+			tryUpstream(log, ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
 			return
 		}
 		log.Debug().Msg("missing branch info")
@@ -58,10 +62,10 @@ func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Clie
 		TargetPath:    path.Join(pathElements[2:]...),
 	}, true); works {
 		log.Trace().Msg("tryUpstream: serve raw domain with default branch")
-		tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
+		tryUpstream(log, ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
 	} else {
 		html.ReturnErrorPage(ctx,
-			fmt.Sprintf("raw domain could not find repo '%s/%s' or repo is empty", targetOpt.TargetOwner, targetOpt.TargetRepo),
+			fmt.Sprintf("raw domain could not find repo <code>%s/%s</code> or repo is empty", targetOpt.TargetOwner, targetOpt.TargetRepo),
 			http.StatusNotFound)
 	}
 }

server/handler/handler_sub_domain.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"github.com/rs/zerolog"
+	"golang.org/x/exp/slices"
 
 	"codeberg.org/codeberg/pages/html"
 	"codeberg.org/codeberg/pages/server/cache"
@@ -17,9 +18,10 @@ import (
 
 func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client,
 	mainDomainSuffix string,
+	defaultPagesBranches []string,
 	trimmedHost string,
 	pathElements []string,
-	canonicalDomainCache cache.SetGetKey,
+	canonicalDomainCache, redirectsCache cache.ICache,
 ) {
 	// Serve pages from subdomains of MainDomainSuffix
 	log.Debug().Msg("main domain suffix")
@@ -51,11 +53,13 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite
 			TargetPath:    path.Join(pathElements[2:]...),
 		}, true); works {
 			log.Trace().Msg("tryUpstream: serve with specified repo and branch")
-			tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
+			tryUpstream(log, ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
 		} else {
-			html.ReturnErrorPage(ctx,
-				fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo),
-				http.StatusFailedDependency)
+			html.ReturnErrorPage(
+				ctx,
+				formatSetBranchNotFoundMessage(pathElements[1][1:], targetOwner, pathElements[0]),
+				http.StatusFailedDependency,
+			)
 		}
 		return
 	}
@@ -63,38 +67,66 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite
 	// Check if the first directory is a branch for the defaultPagesRepo
 	// example.codeberg.page/@main/index.html
 	if strings.HasPrefix(pathElements[0], "@") {
+		targetBranch := pathElements[0][1:]
+
+		// if the default pages branch can be determined exactly, it does not need to be set
+		if len(defaultPagesBranches) == 1 && slices.Contains(defaultPagesBranches, targetBranch) {
+			// example.codeberg.org/@pages/... redirects to example.codeberg.org/...
+			ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), http.StatusTemporaryRedirect)
+			return
+		}
+
 		log.Debug().Msg("main domain preparations, now trying with specified branch")
 		if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
 			TryIndexPages: true,
 			TargetOwner:   targetOwner,
 			TargetRepo:    defaultPagesRepo,
-			TargetBranch:  pathElements[0][1:],
+			TargetBranch:  targetBranch,
 			TargetPath:    path.Join(pathElements[1:]...),
 		}, true); works {
 			log.Trace().Msg("tryUpstream: serve default pages repo with specified branch")
-			tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
+			tryUpstream(log, ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
 		} else {
-			html.ReturnErrorPage(ctx,
-				fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo),
-				http.StatusFailedDependency)
+			html.ReturnErrorPage(
+				ctx,
+				formatSetBranchNotFoundMessage(targetBranch, targetOwner, defaultPagesRepo),
+				http.StatusFailedDependency,
+			)
 		}
 		return
 	}
 
-	// Check if the first directory is a repo with a defaultPagesRepo branch
-	// example.codeberg.page/myrepo/index.html
-	// example.codeberg.page/pages/... is not allowed here.
-	log.Debug().Msg("main domain preparations, now trying with specified repo")
-	if pathElements[0] != defaultPagesRepo {
+	for _, defaultPagesBranch := range defaultPagesBranches {
+		// Check if the first directory is a repo with a default pages branch
+		// example.codeberg.page/myrepo/index.html
+		// example.codeberg.page/{PAGES_BRANCHE}/... is not allowed here.
+		log.Debug().Msg("main domain preparations, now trying with specified repo")
+		if pathElements[0] != defaultPagesBranch {
+			if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
+				TryIndexPages: true,
+				TargetOwner:   targetOwner,
+				TargetRepo:    pathElements[0],
+				TargetBranch:  defaultPagesBranch,
+				TargetPath:    path.Join(pathElements[1:]...),
+			}, false); works {
+				log.Debug().Msg("tryBranch, now trying upstream 5")
+				tryUpstream(log, ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
+				return
+			}
+		}
+
+		// Try to use the defaultPagesRepo on an default pages branch
+		// example.codeberg.page/index.html
+		log.Debug().Msg("main domain preparations, now trying with default repo")
 		if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
 			TryIndexPages: true,
 			TargetOwner:   targetOwner,
-			TargetRepo:    pathElements[0],
+			TargetRepo:    defaultPagesRepo,
 			TargetBranch:  defaultPagesBranch,
-			TargetPath:    path.Join(pathElements[1:]...),
+			TargetPath:    path.Join(pathElements...),
 		}, false); works {
-			log.Debug().Msg("tryBranch, now trying upstream 5")
-			tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
+			log.Debug().Msg("tryBranch, now trying upstream 6")
+			tryUpstream(log, ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
 			return
 		}
 	}
@@ -109,12 +141,16 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite
 		TargetPath:    path.Join(pathElements...),
 	}, false); works {
 		log.Debug().Msg("tryBranch, now trying upstream 6")
-		tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
+		tryUpstream(log, ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
 		return
 	}
 
 	// Couldn't find a valid repo/branch
 	html.ReturnErrorPage(ctx,
-		fmt.Sprintf("could not find a valid repository[%s]", targetRepo),
+		fmt.Sprintf("could not find a valid repository or branch for repository: <code>%s</code>", targetRepo),
 		http.StatusNotFound)
 }
+
+func formatSetBranchNotFoundMessage(branch, owner, repo string) string {
+	return fmt.Sprintf("explicitly set branch <code>%q</code> does not exist at <code>%s/%s</code>", branch, owner, repo)
+}

server/handler/handler_test.go

@@ -1,30 +1,39 @@
 package handler
 
 import (
+	"net/http"
 	"net/http/httptest"
 	"testing"
 	"time"
 
+	"codeberg.org/codeberg/pages/config"
 	"codeberg.org/codeberg/pages/server/cache"
 	"codeberg.org/codeberg/pages/server/gitea"
 	"github.com/rs/zerolog/log"
 )
 
 func TestHandlerPerformance(t *testing.T) {
-	giteaClient, _ := gitea.NewClient("https://codeberg.org", "", cache.NewKeyValueCache(), false, false)
-	testHandler := Handler(
-		"codeberg.page", "raw.codeberg.org",
-		giteaClient,
-		"https://docs.codeberg.org/pages/raw-content/",
-		[]string{"/.well-known/acme-challenge/"},
-		[]string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"},
-		cache.NewKeyValueCache(),
-		cache.NewKeyValueCache(),
-	)
+	cfg := config.ForgeConfig{
+		Root:           "https://codeberg.org",
+		Token:          "",
+		LFSEnabled:     false,
+		FollowSymlinks: false,
+	}
+	giteaClient, _ := gitea.NewClient(cfg, cache.NewInMemoryCache())
+	serverCfg := config.ServerConfig{
+		MainDomain: "codeberg.page",
+		RawDomain:  "raw.codeberg.page",
+		BlacklistedPaths: []string{
+			"/.well-known/acme-challenge/",
+		},
+		AllowedCorsDomains: []string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"},
+		PagesBranches:      []string{"pages"},
+	}
+	testHandler := Handler(serverCfg, giteaClient, cache.NewInMemoryCache(), cache.NewInMemoryCache())
 
 	testCase := func(uri string, status int) {
 		t.Run(uri, func(t *testing.T) {
-			req := httptest.NewRequest("GET", uri, nil)
+			req := httptest.NewRequest("GET", uri, http.NoBody)
 			w := httptest.NewRecorder()
 
 			log.Printf("Start: %v\n", time.Now())

server/handler/try.go

@@ -1,6 +1,7 @@
 package handler
 
 import (
+	"fmt"
 	"net/http"
 	"strings"
 
@@ -14,14 +15,15 @@ import (
 )
 
 // tryUpstream forwards the target request to the Gitea API, and shows an error page on failure.
-func tryUpstream(ctx *context.Context, giteaClient *gitea.Client,
+func tryUpstream(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client,
 	mainDomainSuffix, trimmedHost string,
 	options *upstream.Options,
-	canonicalDomainCache cache.SetGetKey,
+	canonicalDomainCache cache.ICache,
+	redirectsCache cache.ICache,
 ) {
 	// check if a canonical domain exists on a request on MainDomain
-	if strings.HasSuffix(trimmedHost, mainDomainSuffix) {
-		canonicalDomain, _ := options.CheckCanonicalDomain(giteaClient, "", mainDomainSuffix, canonicalDomainCache)
+	if strings.HasSuffix(trimmedHost, mainDomainSuffix) && !options.ServeRaw {
+		canonicalDomain, _ := options.CheckCanonicalDomain(ctx, giteaClient, "", mainDomainSuffix, canonicalDomainCache)
 		if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix) {
 			canonicalPath := ctx.Req.RequestURI
 			if options.TargetRepo != defaultPagesRepo {
@@ -30,7 +32,12 @@ func tryUpstream(ctx *context.Context, giteaClient *gitea.Client,
 					canonicalPath = "/" + path[2]
 				}
 			}
-			ctx.Redirect("https://"+canonicalDomain+canonicalPath, http.StatusTemporaryRedirect)
+
+			redirect_to := "https://" + canonicalDomain + canonicalPath
+
+			log.Debug().Str("to", redirect_to).Msg("redirecting")
+
+			ctx.Redirect(redirect_to, http.StatusTemporaryRedirect)
 			return
 		}
 	}
@@ -39,8 +46,9 @@ func tryUpstream(ctx *context.Context, giteaClient *gitea.Client,
 	options.Host = trimmedHost
 
 	// Try to request the file from the Gitea API
-	if !options.Upstream(ctx, giteaClient) {
-		html.ReturnErrorPage(ctx, "", ctx.StatusCode)
+	log.Debug().Msg("requesting from upstream")
+	if !options.Upstream(ctx, giteaClient, redirectsCache) {
+		html.ReturnErrorPage(ctx, fmt.Sprintf("Forge returned %d %s", ctx.StatusCode, http.StatusText(ctx.StatusCode)), ctx.StatusCode)
 	}
 }
 

server/profiling.go

@@ -0,0 +1,21 @@
+package server
+
+import (
+	"net/http"
+	_ "net/http/pprof"
+
+	"github.com/rs/zerolog/log"
+)
+
+func StartProfilingServer(listeningAddress string) {
+	server := &http.Server{
+		Addr:    listeningAddress,
+		Handler: http.DefaultServeMux,
+	}
+
+	log.Info().Msgf("Starting debug server on %s", listeningAddress)
+
+	go func() {
+		log.Fatal().Err(server.ListenAndServe()).Msg("Failed to start debug server")
+	}()
+}

server/setup.go

@@ -1,27 +0,0 @@
-package server
-
-import (
-	"net/http"
-	"strings"
-
-	"codeberg.org/codeberg/pages/server/cache"
-	"codeberg.org/codeberg/pages/server/context"
-	"codeberg.org/codeberg/pages/server/utils"
-)
-
-func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey) http.HandlerFunc {
-	challengePath := "/.well-known/acme-challenge/"
-
-	return func(w http.ResponseWriter, req *http.Request) {
-		ctx := context.New(w, req)
-		if strings.HasPrefix(ctx.Path(), challengePath) {
-			challenge, ok := challengeCache.Get(utils.TrimHostPort(ctx.Host()) + "/" + strings.TrimPrefix(ctx.Path(), challengePath))
-			if !ok || challenge == nil {
-				ctx.String("no challenge for this token", http.StatusNotFound)
-			}
-			ctx.String(challenge.(string))
-		} else {
-			ctx.Redirect("https://"+ctx.Host()+ctx.Path(), http.StatusMovedPermanently)
-		}
-	}
-}

server/startup.go

@@ -0,0 +1,145 @@
+package server
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/pires/go-proxyproto"
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+	"github.com/urfave/cli/v2"
+
+	cmd "codeberg.org/codeberg/pages/cli"
+	"codeberg.org/codeberg/pages/config"
+	"codeberg.org/codeberg/pages/server/acme"
+	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/certificates"
+	"codeberg.org/codeberg/pages/server/gitea"
+	"codeberg.org/codeberg/pages/server/handler"
+)
+
+// Serve sets up and starts the web server.
+func Serve(ctx *cli.Context) error {
+	// initialize logger with Trace, overridden later with actual level
+	log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Caller().Logger().Level(zerolog.TraceLevel)
+
+	cfg, err := config.ReadConfig(ctx)
+	if err != nil {
+		log.Error().Err(err).Msg("could not read config")
+	}
+
+	config.MergeConfig(ctx, cfg)
+
+	// Initialize the logger.
+	logLevel, err := zerolog.ParseLevel(cfg.LogLevel)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("Setting log level to: %s\n", logLevel)
+	log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Caller().Logger().Level(logLevel)
+
+	listeningSSLAddress := fmt.Sprintf("%s:%d", cfg.Server.Host, cfg.Server.Port)
+	listeningHTTPAddress := fmt.Sprintf("%s:%d", cfg.Server.Host, cfg.Server.HttpPort)
+
+	if cfg.Server.RawDomain != "" {
+		cfg.Server.AllowedCorsDomains = append(cfg.Server.AllowedCorsDomains, cfg.Server.RawDomain)
+	}
+
+	// Make sure MainDomain has a leading dot
+	if !strings.HasPrefix(cfg.Server.MainDomain, ".") {
+		// TODO make this better
+		cfg.Server.MainDomain = "." + cfg.Server.MainDomain
+	}
+
+	if len(cfg.Server.PagesBranches) == 0 {
+		return fmt.Errorf("no default branches set (PAGES_BRANCHES)")
+	}
+
+	// Init ssl cert database
+	certDB, closeFn, err := cmd.OpenCertDB(ctx)
+	if err != nil {
+		return err
+	}
+	defer closeFn()
+
+	challengeCache := cache.NewInMemoryCache()
+	// canonicalDomainCache stores canonical domains
+	canonicalDomainCache := cache.NewInMemoryCache()
+	// redirectsCache stores redirects in _redirects files
+	redirectsCache := cache.NewInMemoryCache()
+	// clientResponseCache stores responses from the Gitea server
+	clientResponseCache := cache.NewInMemoryCache()
+
+	giteaClient, err := gitea.NewClient(cfg.Forge, clientResponseCache)
+	if err != nil {
+		return fmt.Errorf("could not create new gitea client: %v", err)
+	}
+
+	acmeClient, err := acme.CreateAcmeClient(cfg.ACME, cfg.Server.HttpServerEnabled, challengeCache)
+	if err != nil {
+		return err
+	}
+
+	if err := certificates.SetupMainDomainCertificates(log.Logger, cfg.Server.MainDomain, acmeClient, certDB); err != nil {
+		return err
+	}
+
+	// Create listener for SSL connections
+	log.Info().Msgf("Create TCP listener for SSL on %s", listeningSSLAddress)
+	listener, err := net.Listen("tcp", listeningSSLAddress)
+	if err != nil {
+		return fmt.Errorf("couldn't create listener: %v", err)
+	}
+
+	if cfg.Server.UseProxyProtocol {
+		listener = &proxyproto.Listener{Listener: listener}
+	}
+	// Setup listener for SSL connections
+	listener = tls.NewListener(listener, certificates.TLSConfig(
+		cfg.Server.MainDomain,
+		giteaClient,
+		acmeClient,
+		cfg.Server.PagesBranches[0],
+		challengeCache, canonicalDomainCache,
+		certDB,
+		cfg.ACME.NoDNS01,
+		cfg.Server.RawDomain,
+	))
+
+	interval := 12 * time.Hour
+	certMaintainCtx, cancelCertMaintain := context.WithCancel(context.Background())
+	defer cancelCertMaintain()
+	go certificates.MaintainCertDB(log.Logger, certMaintainCtx, interval, acmeClient, cfg.Server.MainDomain, certDB)
+
+	if cfg.Server.HttpServerEnabled {
+		// Create handler for http->https redirect and http acme challenges
+		httpHandler := certificates.SetupHTTPACMEChallengeServer(challengeCache, uint(cfg.Server.Port))
+
+		// Create listener for http and start listening
+		go func() {
+			log.Info().Msgf("Start HTTP server listening on %s", listeningHTTPAddress)
+			err := http.ListenAndServe(listeningHTTPAddress, httpHandler)
+			if err != nil {
+				log.Error().Err(err).Msg("Couldn't start HTTP server")
+			}
+		}()
+	}
+
+	if ctx.IsSet("enable-profiling") {
+		StartProfilingServer(ctx.String("profiling-address"))
+	}
+
+	// Create ssl handler based on settings
+	sslHandler := handler.Handler(cfg.Server, giteaClient, canonicalDomainCache, redirectsCache)
+
+	// Start the ssl listener
+	log.Info().Msgf("Start SSL server using TCP listener on %s", listener.Addr())
+
+	return http.Serve(listener, sslHandler)
+}

server/upstream/domains.go

@@ -8,6 +8,7 @@ import (
 	"github.com/rs/zerolog/log"
 
 	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/context"
 	"codeberg.org/codeberg/pages/server/gitea"
 )
 
@@ -17,7 +18,7 @@ var canonicalDomainCacheTimeout = 15 * time.Minute
 const canonicalDomainConfig = ".domains"
 
 // CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file).
-func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (domain string, valid bool) {
+func (o *Options) CheckCanonicalDomain(ctx *context.Context, giteaClient *gitea.Client, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.ICache) (domain string, valid bool) {
 	// Check if this request is cached.
 	if cachedValue, ok := canonicalDomainCache.Get(o.TargetOwner + "/" + o.TargetRepo + "/" + o.TargetBranch); ok {
 		domains := cachedValue.([]string)
@@ -30,7 +31,7 @@ func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain,
 		return domains[0], valid
 	}
 
-	body, err := giteaClient.GiteaRawContent(o.TargetOwner, o.TargetRepo, o.TargetBranch, canonicalDomainConfig)
+	body, err := giteaClient.GiteaRawContent(ctx, o.TargetOwner, o.TargetRepo, o.TargetBranch, canonicalDomainConfig)
 	if err != nil && !errors.Is(err, gitea.ErrorNotFound) {
 		log.Error().Err(err).Msgf("could not read %s of %s/%s", canonicalDomainConfig, o.TargetOwner, o.TargetRepo)
 	}
@@ -41,7 +42,7 @@ func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain,
 		domain = strings.TrimSpace(domain)
 		domain = strings.TrimPrefix(domain, "http://")
 		domain = strings.TrimPrefix(domain, "https://")
-		if len(domain) > 0 && !strings.HasPrefix(domain, "#") && !strings.ContainsAny(domain, "\t /") && strings.ContainsRune(domain, '.') {
+		if domain != "" && !strings.HasPrefix(domain, "#") && !strings.ContainsAny(domain, "\t /") && strings.ContainsRune(domain, '.') {
 			domains = append(domains, domain)
 		}
 		if domain == actualDomain {

server/upstream/header.go

@@ -24,5 +24,8 @@ func (o *Options) setHeader(ctx *context.Context, header http.Header) {
 	} else {
 		ctx.RespWriter.Header().Set(gitea.ContentTypeHeader, mime)
 	}
-	ctx.RespWriter.Header().Set(headerLastModified, o.BranchTimestamp.In(time.UTC).Format(time.RFC1123))
+	if encoding := header.Get(gitea.ContentEncodingHeader); encoding != "" && encoding != "identity" {
+		ctx.RespWriter.Header().Set(gitea.ContentEncodingHeader, encoding)
+	}
+	ctx.RespWriter.Header().Set(headerLastModified, o.BranchTimestamp.In(time.UTC).Format(http.TimeFormat))
 }

server/upstream/redirects.go

@@ -0,0 +1,108 @@
+package upstream
+
+import (
+	"strconv"
+	"strings"
+	"time"
+
+	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/context"
+	"codeberg.org/codeberg/pages/server/gitea"
+	"github.com/rs/zerolog/log"
+)
+
+type Redirect struct {
+	From       string
+	To         string
+	StatusCode int
+}
+
+// rewriteURL returns the destination URL and true if r matches reqURL.
+func (r *Redirect) rewriteURL(reqURL string) (dstURL string, ok bool) {
+	// check if from url matches request url
+	if strings.TrimSuffix(r.From, "/") == strings.TrimSuffix(reqURL, "/") {
+		return r.To, true
+	}
+	// handle wildcard redirects
+	if strings.HasSuffix(r.From, "/*") {
+		trimmedFromURL := strings.TrimSuffix(r.From, "/*")
+		if reqURL == trimmedFromURL || strings.HasPrefix(reqURL, trimmedFromURL+"/") {
+			if strings.Contains(r.To, ":splat") {
+				matched := strings.TrimPrefix(reqURL, trimmedFromURL)
+				matched = strings.TrimPrefix(matched, "/")
+				return strings.ReplaceAll(r.To, ":splat", matched), true
+			}
+			return r.To, true
+		}
+	}
+	return "", false
+}
+
+// redirectsCacheTimeout specifies the timeout for the redirects cache.
+var redirectsCacheTimeout = 10 * time.Minute
+
+const redirectsConfig = "_redirects"
+
+// getRedirects returns redirects specified in the _redirects file.
+func (o *Options) getRedirects(ctx *context.Context, giteaClient *gitea.Client, redirectsCache cache.ICache) []Redirect {
+	var redirects []Redirect
+	cacheKey := o.TargetOwner + "/" + o.TargetRepo + "/" + o.TargetBranch
+
+	// Check for cached redirects
+	if cachedValue, ok := redirectsCache.Get(cacheKey); ok {
+		redirects = cachedValue.([]Redirect)
+	} else {
+		// Get _redirects file and parse
+		body, err := giteaClient.GiteaRawContent(ctx, o.TargetOwner, o.TargetRepo, o.TargetBranch, redirectsConfig)
+		if err == nil {
+			for _, line := range strings.Split(string(body), "\n") {
+				redirectArr := strings.Fields(line)
+
+				// Ignore comments and invalid lines
+				if strings.HasPrefix(line, "#") || len(redirectArr) < 2 {
+					continue
+				}
+
+				// Get redirect status code
+				statusCode := 301
+				if len(redirectArr) == 3 {
+					statusCode, err = strconv.Atoi(redirectArr[2])
+					if err != nil {
+						log.Info().Err(err).Msgf("could not read %s of %s/%s", redirectsConfig, o.TargetOwner, o.TargetRepo)
+					}
+				}
+
+				redirects = append(redirects, Redirect{
+					From:       redirectArr[0],
+					To:         redirectArr[1],
+					StatusCode: statusCode,
+				})
+			}
+		}
+		_ = redirectsCache.Set(cacheKey, redirects, redirectsCacheTimeout)
+	}
+	return redirects
+}
+
+func (o *Options) matchRedirects(ctx *context.Context, giteaClient *gitea.Client, redirects []Redirect, redirectsCache cache.ICache) (final bool) {
+	reqURL := ctx.Req.RequestURI
+	// remove repo and branch from request url
+	reqURL = strings.TrimPrefix(reqURL, "/"+o.TargetRepo)
+	reqURL = strings.TrimPrefix(reqURL, "/@"+o.TargetBranch)
+
+	for _, redirect := range redirects {
+		if dstURL, ok := redirect.rewriteURL(reqURL); ok {
+			if o.TargetPath == dstURL { // recursion base case, rewrite directly when paths are the same
+				return true
+			} else if redirect.StatusCode == 200 { // do rewrite if status code is 200
+				o.TargetPath = dstURL
+				o.Upstream(ctx, giteaClient, redirectsCache)
+			} else {
+				ctx.Redirect(dstURL, redirect.StatusCode)
+			}
+			return true
+		}
+	}
+
+	return false
+}

server/upstream/redirects_test.go

@@ -0,0 +1,36 @@
+package upstream
+
+import (
+	"testing"
+)
+
+func TestRedirect_rewriteURL(t *testing.T) {
+	for _, tc := range []struct {
+		redirect   Redirect
+		reqURL     string
+		wantDstURL string
+		wantOk     bool
+	}{
+		{Redirect{"/", "/dst", 200}, "/", "/dst", true},
+		{Redirect{"/", "/dst", 200}, "/foo", "", false},
+		{Redirect{"/src", "/dst", 200}, "/src", "/dst", true},
+		{Redirect{"/src", "/dst", 200}, "/foo", "", false},
+		{Redirect{"/src", "/dst", 200}, "/src/foo", "", false},
+		{Redirect{"/*", "/dst", 200}, "/", "/dst", true},
+		{Redirect{"/*", "/dst", 200}, "/src", "/dst", true},
+		{Redirect{"/src/*", "/dst/:splat", 200}, "/src", "/dst/", true},
+		{Redirect{"/src/*", "/dst/:splat", 200}, "/src/", "/dst/", true},
+		{Redirect{"/src/*", "/dst/:splat", 200}, "/src/foo", "/dst/foo", true},
+		{Redirect{"/src/*", "/dst/:splat", 200}, "/src/foo/bar", "/dst/foo/bar", true},
+		{Redirect{"/src/*", "/dst/:splatsuffix", 200}, "/src/foo", "/dst/foosuffix", true},
+		{Redirect{"/src/*", "/dst:splat", 200}, "/src/foo", "/dstfoo", true},
+		{Redirect{"/src/*", "/dst", 200}, "/srcfoo", "", false},
+		// This is the example from FEATURES.md:
+		{Redirect{"/articles/*", "/posts/:splat", 302}, "/articles/2022/10/12/post-1/", "/posts/2022/10/12/post-1/", true},
+	} {
+		if dstURL, ok := tc.redirect.rewriteURL(tc.reqURL); dstURL != tc.wantDstURL || ok != tc.wantOk {
+			t.Errorf("%#v.rewriteURL(%q) = %q, %v; want %q, %v",
+				tc.redirect, tc.reqURL, dstURL, ok, tc.wantDstURL, tc.wantOk)
+		}
+	}
+}

server/upstream/upstream.go

@@ -1,16 +1,20 @@
 package upstream
 
 import (
+	"cmp"
 	"errors"
 	"fmt"
 	"io"
 	"net/http"
+	"slices"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/rs/zerolog/log"
 
 	"codeberg.org/codeberg/pages/html"
+	"codeberg.org/codeberg/pages/server/cache"
 	"codeberg.org/codeberg/pages/server/context"
 	"codeberg.org/codeberg/pages/server/gitea"
 )
@@ -18,6 +22,8 @@ import (
 const (
 	headerLastModified    = "Last-Modified"
 	headerIfModifiedSince = "If-Modified-Since"
+	headerAcceptEncoding  = "Accept-Encoding"
+	headerContentEncoding = "Content-Encoding"
 
 	rawMime = "text/plain; charset=utf-8"
 )
@@ -51,12 +57,80 @@ type Options struct {
 	ServeRaw bool
 }
 
+// allowed encodings
+var allowedEncodings = map[string]string{
+	"gzip":     ".gz",
+	"br":       ".br",
+	"zstd":     ".zst",
+	"identity": "",
+}
+
+// parses Accept-Encoding header into a list of acceptable encodings
+func AcceptEncodings(header string) []string {
+	log.Trace().Msgf("got accept-encoding: %s", header)
+	encodings := []string{}
+	globQuality := 0.0
+	qualities := make(map[string]float64)
+
+	for _, encoding := range strings.Split(header, ",") {
+		name, quality_str, has_quality := strings.Cut(encoding, ";q=")
+		quality := 1.0
+
+		if has_quality {
+			var err error
+			quality, err = strconv.ParseFloat(quality_str, 64)
+			if err != nil || quality < 0 {
+				continue
+			}
+		}
+
+		name = strings.TrimSpace(name)
+
+		if name == "*" {
+			globQuality = quality
+		} else {
+			_, allowed := allowedEncodings[name]
+			if allowed {
+				qualities[name] = quality
+				if quality > 0 {
+					encodings = append(encodings, name)
+				}
+			}
+		}
+	}
+
+	if globQuality > 0 {
+		for encoding := range allowedEncodings {
+			_, exists := qualities[encoding]
+			if !exists {
+				encodings = append(encodings, encoding)
+				qualities[encoding] = globQuality
+			}
+		}
+	} else {
+		_, exists := qualities["identity"]
+		if !exists {
+			encodings = append(encodings, "identity")
+			qualities["identity"] = -1
+		}
+	}
+
+	slices.SortStableFunc(encodings, func(x, y string) int {
+		// sort in reverse order; big quality comes first
+		return cmp.Compare(qualities[y], qualities[x])
+	})
+	log.Trace().Msgf("decided encoding order: %v", encodings)
+	return encodings
+}
+
 // Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
-func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (final bool) {
-	log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger()
+func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redirectsCache cache.ICache) bool {
+	log := log.With().Str("ReqId", ctx.ReqId).Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger()
+
+	log.Debug().Msg("Start")
 
 	if o.TargetOwner == "" || o.TargetRepo == "" {
-		html.ReturnErrorPage(ctx, "either repo owner or name info is missing", http.StatusBadRequest)
+		html.ReturnErrorPage(ctx, "forge client: either repo owner or name info is missing", http.StatusBadRequest)
 		return true
 	}
 
@@ -66,7 +140,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin
 		// handle 404
 		if err != nil && errors.Is(err, gitea.ErrorNotFound) || !branchExist {
 			html.ReturnErrorPage(ctx,
-				fmt.Sprintf("branch %q for '%s/%s' not found", o.TargetBranch, o.TargetOwner, o.TargetRepo),
+				fmt.Sprintf("branch <code>%q</code> for <code>%s/%s</code> not found", o.TargetBranch, o.TargetOwner, o.TargetRepo),
 				http.StatusNotFound)
 			return true
 		}
@@ -74,7 +148,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin
 		// handle unexpected errors
 		if err != nil {
 			html.ReturnErrorPage(ctx,
-				fmt.Sprintf("could not get timestamp of branch %q: %v", o.TargetBranch, err),
+				fmt.Sprintf("could not get timestamp of branch <code>%q</code>: '%v'", o.TargetBranch, err),
 				http.StatusFailedDependency)
 			return true
 		}
@@ -94,48 +168,90 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin
 
 	log.Debug().Msg("Preparing")
 
-	reader, header, statusCode, err := giteaClient.ServeRawContent(o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath)
-	if reader != nil {
-		defer reader.Close()
+	var reader io.ReadCloser
+	var header http.Header
+	var statusCode int
+	var err error
+
+	// pick first non-404 response for encoding, *only* if not root
+	if o.TargetPath == "" || strings.HasSuffix(o.TargetPath, "/") {
+		err = gitea.ErrorNotFound
+	} else {
+		for _, encoding := range AcceptEncodings(ctx.Req.Header.Get(headerAcceptEncoding)) {
+			log.Trace().Msgf("try %s encoding", encoding)
+
+			// add extension for encoding
+			path := o.TargetPath + allowedEncodings[encoding]
+			reader, header, statusCode, err = giteaClient.ServeRawContent(ctx, o.TargetOwner, o.TargetRepo, o.TargetBranch, path, true)
+			if statusCode == http.StatusNotFound {
+				continue
+			}
+			if err != nil {
+				break
+			}
+			log.Debug().Msgf("using %s encoding", encoding)
+			if encoding != "identity" {
+				header.Set(headerContentEncoding, encoding)
+			}
+			break
+		}
+		if reader != nil {
+			defer reader.Close()
+		}
 	}
 
 	log.Debug().Msg("Aquisting")
 
 	// Handle not found error
 	if err != nil && errors.Is(err, gitea.ErrorNotFound) {
+		log.Debug().Msg("Handling not found error")
+		// Get and match redirects
+		redirects := o.getRedirects(ctx, giteaClient, redirectsCache)
+		if o.matchRedirects(ctx, giteaClient, redirects, redirectsCache) {
+			log.Trace().Msg("redirect")
+			return true
+		}
+
 		if o.TryIndexPages {
+			log.Trace().Msg("try index page")
 			// copy the o struct & try if an index page exists
 			optionsForIndexPages := *o
 			optionsForIndexPages.TryIndexPages = false
 			optionsForIndexPages.appendTrailingSlash = true
 			for _, indexPage := range upstreamIndexPages {
 				optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage
-				if optionsForIndexPages.Upstream(ctx, giteaClient) {
+				if optionsForIndexPages.Upstream(ctx, giteaClient, redirectsCache) {
 					return true
 				}
 			}
+			log.Trace().Msg("try html file with path name")
 			// compatibility fix for GitHub Pages (/example → /example.html)
 			optionsForIndexPages.appendTrailingSlash = false
 			optionsForIndexPages.redirectIfExists = strings.TrimSuffix(ctx.Path(), "/") + ".html"
 			optionsForIndexPages.TargetPath = o.TargetPath + ".html"
-			if optionsForIndexPages.Upstream(ctx, giteaClient) {
+			if optionsForIndexPages.Upstream(ctx, giteaClient, redirectsCache) {
 				return true
 			}
 		}
 
+		log.Debug().Msg("not found")
+
 		ctx.StatusCode = http.StatusNotFound
 		if o.TryIndexPages {
+			log.Trace().Msg("try not found page")
 			// copy the o struct & try if a not found page exists
 			optionsForNotFoundPages := *o
 			optionsForNotFoundPages.TryIndexPages = false
 			optionsForNotFoundPages.appendTrailingSlash = false
 			for _, notFoundPage := range upstreamNotFoundPages {
 				optionsForNotFoundPages.TargetPath = "/" + notFoundPage
-				if optionsForNotFoundPages.Upstream(ctx, giteaClient) {
+				if optionsForNotFoundPages.Upstream(ctx, giteaClient, redirectsCache) {
 					return true
 				}
 			}
+			log.Trace().Msg("not found page missing")
 		}
+
 		return false
 	}
 
@@ -145,16 +261,16 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin
 		var msg string
 
 		if err != nil {
-			msg = "gitea client returned unexpected error"
+			msg = "forge client: returned unexpected error"
 			log.Error().Err(err).Msg(msg)
-			msg = fmt.Sprintf("%s: %v", msg, err)
+			msg = fmt.Sprintf("%s: '%v'", msg, err)
 		}
 		if reader == nil {
-			msg = "gitea client returned no reader"
+			msg = "forge client: returned no reader"
 			log.Error().Msg(msg)
 		}
 		if statusCode != http.StatusOK {
-			msg = fmt.Sprintf("Couldn't fetch contents (status code %d)", statusCode)
+			msg = fmt.Sprintf("forge client: couldn't fetch contents: <code>%d - %s</code>", statusCode, http.StatusText(statusCode))
 			log.Error().Msg(msg)
 		}
 
@@ -165,10 +281,12 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin
 	// Append trailing slash if missing (for index files), and redirect to fix filenames in general
 	// o.appendTrailingSlash is only true when looking for index pages
 	if o.appendTrailingSlash && !strings.HasSuffix(ctx.Path(), "/") {
+		log.Trace().Msg("append trailing slash and redirect")
 		ctx.Redirect(ctx.Path()+"/", http.StatusTemporaryRedirect)
 		return true
 	}
-	if strings.HasSuffix(ctx.Path(), "/index.html") {
+	if strings.HasSuffix(ctx.Path(), "/index.html") && !o.ServeRaw {
+		log.Trace().Msg("remove index.html from path and redirect")
 		ctx.Redirect(strings.TrimSuffix(ctx.Path(), "index.html"), http.StatusTemporaryRedirect)
 		return true
 	}

server/utils/utils.go

@@ -1,6 +1,8 @@
 package utils
 
 import (
+	"net/url"
+	"path"
 	"strings"
 )
 
@@ -11,3 +13,15 @@ func TrimHostPort(host string) string {
 	}
 	return host
 }
+
+func CleanPath(uriPath string) string {
+	unescapedPath, _ := url.PathUnescape(uriPath)
+	cleanedPath := path.Join("/", unescapedPath)
+
+	// If the path refers to a directory, add a trailing slash.
+	if !strings.HasSuffix(cleanedPath, "/") && (strings.HasSuffix(unescapedPath, "/") || strings.HasSuffix(unescapedPath, "/.") || strings.HasSuffix(unescapedPath, "/..")) {
+		cleanedPath += "/"
+	}
+
+	return cleanedPath
+}

server/utils/utils_test.go

@@ -11,3 +11,59 @@ func TestTrimHostPort(t *testing.T) {
 	assert.EqualValues(t, "", TrimHostPort(":"))
 	assert.EqualValues(t, "example.com", TrimHostPort("example.com:80"))
 }
+
+// TestCleanPath is mostly copied from fasthttp, to keep the behaviour we had before migrating away from it.
+// Source (MIT licensed): https://github.com/valyala/fasthttp/blob/v1.48.0/uri_test.go#L154
+// Copyright (c) 2015-present Aliaksandr Valialkin, VertaMedia, Kirill Danshin, Erik Dubbelboer, FastHTTP Authors
+func TestCleanPath(t *testing.T) {
+	// double slash
+	testURIPathNormalize(t, "/aa//bb", "/aa/bb")
+
+	// triple slash
+	testURIPathNormalize(t, "/x///y/", "/x/y/")
+
+	// multi slashes
+	testURIPathNormalize(t, "/abc//de///fg////", "/abc/de/fg/")
+
+	// encoded slashes
+	testURIPathNormalize(t, "/xxxx%2fyyy%2f%2F%2F", "/xxxx/yyy/")
+
+	// dotdot
+	testURIPathNormalize(t, "/aaa/..", "/")
+
+	// dotdot with trailing slash
+	testURIPathNormalize(t, "/xxx/yyy/../", "/xxx/")
+
+	// multi dotdots
+	testURIPathNormalize(t, "/aaa/bbb/ccc/../../ddd", "/aaa/ddd")
+
+	// dotdots separated by other data
+	testURIPathNormalize(t, "/a/b/../c/d/../e/..", "/a/c/")
+
+	// too many dotdots
+	testURIPathNormalize(t, "/aaa/../../../../xxx", "/xxx")
+	testURIPathNormalize(t, "/../../../../../..", "/")
+	testURIPathNormalize(t, "/../../../../../../", "/")
+
+	// encoded dotdots
+	testURIPathNormalize(t, "/aaa%2Fbbb%2F%2E.%2Fxxx", "/aaa/xxx")
+
+	// double slash with dotdots
+	testURIPathNormalize(t, "/aaa////..//b", "/b")
+
+	// fake dotdot
+	testURIPathNormalize(t, "/aaa/..bbb/ccc/..", "/aaa/..bbb/")
+
+	// single dot
+	testURIPathNormalize(t, "/a/./b/././c/./d.html", "/a/b/c/d.html")
+	testURIPathNormalize(t, "./foo/", "/foo/")
+	testURIPathNormalize(t, "./../.././../../aaa/bbb/../../../././../", "/")
+	testURIPathNormalize(t, "./a/./.././../b/./foo.html", "/b/foo.html")
+}
+
+func testURIPathNormalize(t *testing.T, requestURI, expectedPath string) {
+	cleanedPath := CleanPath(requestURI)
+	if cleanedPath != expectedPath {
+		t.Fatalf("Unexpected path %q. Expected %q. requestURI=%q", cleanedPath, expectedPath, requestURI)
+	}
+}