Merge branch 'main' into issue115

This commit is contained in:
6543 2022-11-12 21:02:19 +01:00
commit 516cb6677c
No known key found for this signature in database
GPG key ID: C99B82E40B027BAE
36 changed files with 1286 additions and 920 deletions

114
server/handler/handler.go Normal file
View file

@ -0,0 +1,114 @@
package handler
import (
"net/http"
"strings"
"github.com/rs/zerolog/log"
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/context"
"codeberg.org/codeberg/pages/server/gitea"
"codeberg.org/codeberg/pages/server/version"
)
const (
headerAccessControlAllowOrigin = "Access-Control-Allow-Origin"
headerAccessControlAllowMethods = "Access-Control-Allow-Methods"
)
// Handler handles a single HTTP request to the web server.
func Handler(mainDomainSuffix, rawDomain string,
giteaClient *gitea.Client,
rawInfoPage string,
blacklistedPaths, allowedCorsDomains []string,
defaultPagesBranches []string,
dnsLookupCache, canonicalDomainCache cache.SetGetKey,
) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
log := log.With().Strs("Handler", []string{string(req.Host), req.RequestURI}).Logger()
ctx := context.New(w, req)
ctx.RespWriter.Header().Set("Server", "CodebergPages/"+version.Version)
// Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin
ctx.RespWriter.Header().Set("Referrer-Policy", "strict-origin-when-cross-origin")
// Enable browser caching for up to 10 minutes
ctx.RespWriter.Header().Set("Cache-Control", "public, max-age=600")
trimmedHost := ctx.TrimHostPort()
// Add HSTS for RawDomain and MainDomainSuffix
if hsts := getHSTSHeader(trimmedHost, mainDomainSuffix, rawDomain); hsts != "" {
ctx.RespWriter.Header().Set("Strict-Transport-Security", hsts)
}
// Handle all http methods
ctx.RespWriter.Header().Set("Allow", http.MethodGet+", "+http.MethodHead+", "+http.MethodOptions)
switch ctx.Req.Method {
case http.MethodOptions:
// return Allow header
ctx.RespWriter.WriteHeader(http.StatusNoContent)
return
case http.MethodGet,
http.MethodHead:
// end switch case and handle allowed requests
break
default:
// Block all methods not required for static pages
ctx.String("Method not allowed", http.StatusMethodNotAllowed)
return
}
// Block blacklisted paths (like ACME challenges)
for _, blacklistedPath := range blacklistedPaths {
if strings.HasPrefix(ctx.Path(), blacklistedPath) {
html.ReturnErrorPage(ctx, "requested blacklisted path", http.StatusForbidden)
return
}
}
// Allow CORS for specified domains
allowCors := false
for _, allowedCorsDomain := range allowedCorsDomains {
if strings.EqualFold(trimmedHost, allowedCorsDomain) {
allowCors = true
break
}
}
if allowCors {
ctx.RespWriter.Header().Set(headerAccessControlAllowOrigin, "*")
ctx.RespWriter.Header().Set(headerAccessControlAllowMethods, http.MethodGet+", "+http.MethodHead)
}
// Prepare request information to Gitea
pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/")
if rawDomain != "" && strings.EqualFold(trimmedHost, rawDomain) {
log.Debug().Msg("raw domain request detecded")
handleRaw(log, ctx, giteaClient,
mainDomainSuffix, rawInfoPage,
trimmedHost,
pathElements,
canonicalDomainCache)
} else if strings.HasSuffix(trimmedHost, mainDomainSuffix) {
log.Debug().Msg("subdomain request detecded")
handleSubDomain(log, ctx, giteaClient,
mainDomainSuffix,
defaultPagesBranches,
trimmedHost,
pathElements,
canonicalDomainCache)
} else {
log.Debug().Msg("custom domain request detecded")
handleCustomDomain(log, ctx, giteaClient,
mainDomainSuffix,
trimmedHost,
pathElements,
defaultPagesBranches[0],
dnsLookupCache, canonicalDomainCache)
}
}
}

View file

@ -0,0 +1,72 @@
package handler
import (
"net/http"
"path"
"strings"
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/context"
"codeberg.org/codeberg/pages/server/dns"
"codeberg.org/codeberg/pages/server/gitea"
"codeberg.org/codeberg/pages/server/upstream"
"github.com/rs/zerolog"
)
func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client,
mainDomainSuffix string,
trimmedHost string,
pathElements []string,
firstDefaultBranch string,
dnsLookupCache, canonicalDomainCache cache.SetGetKey,
) {
// Serve pages from custom domains
targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, mainDomainSuffix, firstDefaultBranch, dnsLookupCache)
if targetOwner == "" {
html.ReturnErrorPage(ctx,
"could not obtain repo owner from custom domain",
http.StatusFailedDependency)
return
}
pathParts := pathElements
canonicalLink := false
if strings.HasPrefix(pathElements[0], "@") {
targetBranch = pathElements[0][1:]
pathParts = pathElements[1:]
canonicalLink = true
}
// Try to use the given repo on the given branch or the default branch
log.Debug().Msg("custom domain preparations, now trying with details from DNS")
if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
TryIndexPages: true,
TargetOwner: targetOwner,
TargetRepo: targetRepo,
TargetBranch: targetBranch,
TargetPath: path.Join(pathParts...),
}, canonicalLink); works {
canonicalDomain, valid := targetOpt.CheckCanonicalDomain(giteaClient, trimmedHost, mainDomainSuffix, canonicalDomainCache)
if !valid {
html.ReturnErrorPage(ctx, "domain not specified in <code>.domains</code> file", http.StatusMisdirectedRequest)
return
} else if canonicalDomain != trimmedHost {
// only redirect if the target is also a codeberg page!
targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix, firstDefaultBranch, dnsLookupCache)
if targetOwner != "" {
ctx.Redirect("https://"+canonicalDomain+string(targetOpt.TargetPath), http.StatusTemporaryRedirect)
return
}
html.ReturnErrorPage(ctx, "target is no codeberg page", http.StatusFailedDependency)
return
}
log.Debug().Msg("tryBranch, now trying upstream 7")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
return
}
html.ReturnErrorPage(ctx, "could not find target for custom domain", http.StatusFailedDependency)
}

View file

@ -0,0 +1,67 @@
package handler
import (
"fmt"
"net/http"
"path"
"strings"
"github.com/rs/zerolog"
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/context"
"codeberg.org/codeberg/pages/server/gitea"
"codeberg.org/codeberg/pages/server/upstream"
)
func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client,
mainDomainSuffix, rawInfoPage string,
trimmedHost string,
pathElements []string,
canonicalDomainCache cache.SetGetKey,
) {
// Serve raw content from RawDomain
log.Debug().Msg("raw domain")
if len(pathElements) < 2 {
// https://{RawDomain}/{owner}/{repo}[/@{branch}]/{path} is required
ctx.Redirect(rawInfoPage, http.StatusTemporaryRedirect)
return
}
// raw.codeberg.org/example/myrepo/@main/index.html
if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") {
log.Debug().Msg("raw domain preparations, now trying with specified branch")
if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
ServeRaw: true,
TargetOwner: pathElements[0],
TargetRepo: pathElements[1],
TargetBranch: pathElements[2][1:],
TargetPath: path.Join(pathElements[3:]...),
}, true); works {
log.Trace().Msg("tryUpstream: serve raw domain with specified branch")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
return
}
log.Debug().Msg("missing branch info")
html.ReturnErrorPage(ctx, "missing branch info", http.StatusFailedDependency)
return
}
log.Debug().Msg("raw domain preparations, now trying with default branch")
if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
TryIndexPages: false,
ServeRaw: true,
TargetOwner: pathElements[0],
TargetRepo: pathElements[1],
TargetPath: path.Join(pathElements[2:]...),
}, true); works {
log.Trace().Msg("tryUpstream: serve raw domain with default branch")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
} else {
html.ReturnErrorPage(ctx,
fmt.Sprintf("raw domain could not find repo '%s/%s' or repo is empty", targetOpt.TargetOwner, targetOpt.TargetRepo),
http.StatusNotFound)
}
}

View file

@ -0,0 +1,123 @@
package handler
import (
"fmt"
"net/http"
"path"
"strings"
"github.com/rs/zerolog"
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/context"
"codeberg.org/codeberg/pages/server/gitea"
"codeberg.org/codeberg/pages/server/upstream"
)
func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client,
mainDomainSuffix string,
defaultPagesBranches []string,
trimmedHost string,
pathElements []string,
canonicalDomainCache cache.SetGetKey,
) {
// Serve pages from subdomains of MainDomainSuffix
log.Debug().Msg("main domain suffix")
targetOwner := strings.TrimSuffix(trimmedHost, mainDomainSuffix)
targetRepo := pathElements[0]
if targetOwner == "www" {
// www.codeberg.page redirects to codeberg.page // TODO: rm hardcoded - use cname?
ctx.Redirect("https://"+string(mainDomainSuffix[1:])+string(ctx.Path()), http.StatusPermanentRedirect)
return
}
// Check if the first directory is a repo with the second directory as a branch
// example.codeberg.page/myrepo/@main/index.html
if len(pathElements) > 1 && strings.HasPrefix(pathElements[1], "@") {
if targetRepo == "pages" {
// example.codeberg.org/pages/@... redirects to example.codeberg.org/@...
ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), http.StatusTemporaryRedirect)
return
}
log.Debug().Msg("main domain preparations, now trying with specified repo & branch")
if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
TryIndexPages: true,
TargetOwner: targetOwner,
TargetRepo: pathElements[0],
TargetBranch: pathElements[1][1:],
TargetPath: path.Join(pathElements[2:]...),
}, true); works {
log.Trace().Msg("tryUpstream: serve with specified repo and branch")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
} else {
html.ReturnErrorPage(ctx,
fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo),
http.StatusFailedDependency)
}
return
}
// Check if the first directory is a branch for the "pages" repo
// example.codeberg.page/@main/index.html
if strings.HasPrefix(pathElements[0], "@") {
log.Debug().Msg("main domain preparations, now trying with specified branch")
if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
TryIndexPages: true,
TargetOwner: targetOwner,
TargetRepo: "pages",
TargetBranch: pathElements[0][1:],
TargetPath: path.Join(pathElements[1:]...),
}, true); works {
log.Trace().Msg("tryUpstream: serve default pages repo with specified branch")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
} else {
html.ReturnErrorPage(ctx,
fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo),
http.StatusFailedDependency)
}
return
}
for _, branch := range defaultPagesBranches {
// Check if the first directory is a repo with a default pages branch
// example.codeberg.page/myrepo/index.html
// example.codeberg.page/{PAGES_BRANCHE}/... is not allowed here.
log.Debug().Msg("main domain preparations, now trying with specified repo")
if pathElements[0] != branch {
if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
TryIndexPages: true,
TargetOwner: targetOwner,
TargetRepo: pathElements[0],
TargetBranch: branch,
TargetPath: path.Join(pathElements[1:]...),
}, false); works {
log.Debug().Msg("tryBranch, now trying upstream 5")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
return
}
}
// Try to use the "pages" repo on its default branch
// example.codeberg.page/index.html
log.Debug().Msg("main domain preparations, now trying with default repo/branch")
if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
TryIndexPages: true,
TargetOwner: targetOwner,
TargetRepo: branch,
TargetPath: path.Join(pathElements...),
}, false); works {
log.Debug().Msg("tryBranch, now trying upstream 6")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
return
}
}
// Couldn't find a valid repo/branch
html.ReturnErrorPage(ctx,
fmt.Sprintf("couldn't find a valid repo[%s]", targetRepo),
http.StatusFailedDependency)
}

View file

@ -0,0 +1,49 @@
package handler
import (
"net/http/httptest"
"testing"
"time"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/gitea"
"github.com/rs/zerolog/log"
)
func TestHandlerPerformance(t *testing.T) {
giteaClient, _ := gitea.NewClient("https://codeberg.org", "", cache.NewKeyValueCache(), false, false)
testHandler := Handler(
"codeberg.page", "raw.codeberg.org",
giteaClient,
"https://docs.codeberg.org/pages/raw-content/",
[]string{"/.well-known/acme-challenge/"},
[]string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"},
[]string{"pages"},
cache.NewKeyValueCache(),
cache.NewKeyValueCache(),
)
testCase := func(uri string, status int) {
req := httptest.NewRequest("GET", uri, nil)
w := httptest.NewRecorder()
log.Printf("Start: %v\n", time.Now())
start := time.Now()
testHandler(w, req)
end := time.Now()
log.Printf("Done: %v\n", time.Now())
resp := w.Result()
if resp.StatusCode != status {
t.Errorf("request failed with status code %d", resp.StatusCode)
} else {
t.Logf("request took %d milliseconds", end.Sub(start).Milliseconds())
}
}
testCase("https://mondstern.codeberg.page/", 424) // TODO: expect 200
testCase("https://mondstern.codeberg.page/", 424) // TODO: expect 200
testCase("https://example.momar.xyz/", 424) // TODO: expect 200
testCase("https://codeberg.page/", 424) // TODO: expect 200
}

15
server/handler/hsts.go Normal file
View file

@ -0,0 +1,15 @@
package handler
import (
"strings"
)
// getHSTSHeader returns a HSTS header with includeSubdomains & preload for MainDomainSuffix and RawDomain, or an empty
// string for custom domains.
func getHSTSHeader(host, mainDomainSuffix, rawDomain string) string {
if strings.HasSuffix(host, mainDomainSuffix) || strings.EqualFold(host, rawDomain) {
return "max-age=63072000; includeSubdomains; preload"
} else {
return ""
}
}

76
server/handler/try.go Normal file
View file

@ -0,0 +1,76 @@
package handler
import (
"net/http"
"strings"
"github.com/rs/zerolog"
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/context"
"codeberg.org/codeberg/pages/server/gitea"
"codeberg.org/codeberg/pages/server/upstream"
)
// tryUpstream forwards the target request to the Gitea API, and shows an error page on failure.
func tryUpstream(ctx *context.Context, giteaClient *gitea.Client,
mainDomainSuffix, trimmedHost string,
options *upstream.Options,
canonicalDomainCache cache.SetGetKey,
) {
// check if a canonical domain exists on a request on MainDomain
if strings.HasSuffix(trimmedHost, mainDomainSuffix) {
canonicalDomain, _ := options.CheckCanonicalDomain(giteaClient, "", string(mainDomainSuffix), canonicalDomainCache)
if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) {
canonicalPath := ctx.Req.RequestURI
if options.TargetRepo != "pages" {
path := strings.SplitN(canonicalPath, "/", 3)
if len(path) >= 3 {
canonicalPath = "/" + path[2]
}
}
ctx.Redirect("https://"+canonicalDomain+canonicalPath, http.StatusTemporaryRedirect)
return
}
}
// add host for debugging
options.Host = string(trimmedHost)
// Try to request the file from the Gitea API
if !options.Upstream(ctx, giteaClient) {
html.ReturnErrorPage(ctx, "", ctx.StatusCode)
}
}
// tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty,
// it will also disallow search indexing and add a Link header to the canonical URL.
func tryBranch(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client,
targetOptions *upstream.Options, canonicalLink bool,
) (*upstream.Options, bool) {
if targetOptions.TargetOwner == "" || targetOptions.TargetRepo == "" {
log.Debug().Msg("tryBranch: owner or repo is empty")
return nil, false
}
// Replace "~" to "/" so we can access branch that contains slash character
// Branch name cannot contain "~" so doing this is okay
targetOptions.TargetBranch = strings.ReplaceAll(targetOptions.TargetBranch, "~", "/")
// Check if the branch exists, otherwise treat it as a file path
branchExist, _ := targetOptions.GetBranchTimestamp(giteaClient)
if !branchExist {
log.Debug().Msg("tryBranch: branch doesn't exist")
return nil, false
}
if canonicalLink {
// Hide from search machines & add canonical link
ctx.RespWriter.Header().Set("X-Robots-Tag", "noarchive, noindex")
ctx.RespWriter.Header().Set("Link", targetOptions.ContentWebLink(giteaClient)+"; rel=\"canonical\"")
}
log.Debug().Msg("tryBranch: true")
return targetOptions, true
}