pages/pages-server
1
0
Fork 0
mirror of https://codeberg.org/Codeberg/pages-server.git synced 2024-11-18 10:29:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Implement first version without custom domains, TLS and caching

Browse source
This commit is contained in:
Moritz Marquardt 2021-03-16 20:22:06 +01:00
commit 0756993f7e
No known key found for this signature in database
GPG key ID: D5788327BEE388B6
5 changed files with 426 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
.idea/
*.iml

32
404.html Normal file
View file

@ -0,0 +1,32 @@
<!doctype html>
<html class="codeberg-design">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width">
<title>Not found</title>
<link rel="stylesheet" href="https://design.codeberg.org/design-kit/codeberg.css" />
<link href="https://fonts.codeberg.org/dist/inter/Inter%20Web/inter.css" rel="stylesheet" />
<link href="https://fonts.codeberg.org/dist/fontawesome5/css/all.min.css" rel="stylesheet" />
<style>
body {
margin: 0; padding: 1rem; box-sizing: border-box;
width: 100%; min-height: 100vw;
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
}
</style>
</head>
<body>
<i class="fa fa-bug" style="font-size: 96px"></i>
<h1>You found a bug!</h1>
<h3>Sorry, this page doesn't exist or is otherwise inaccessible (code %status)</h3>
<small>
<img src="https://design.codeberg.org/logo-kit/icon.svg">
Website powered by <a href="https://codeberg.page">Codeberg Pages</a>
</small>
</body>
</html>

5
go.mod Normal file
View file

@ -0,0 +1,5 @@
module codeberg.org/codeberg/pages
go 1.16
require github.com/valyala/fasthttp v1.22.0

23
go.sum Normal file
View file

@ -0,0 +1,23 @@
github.com/andybalholm/brotli v1.0.1 h1:KqhlKozYbRtJvsPrrEeXcO+N2l6NYT5A2QAFmSULpEc=
github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
github.com/klauspost/compress v1.11.8 h1:difgzQsp5mdAz9v8lm3P/I+EpDKMU/6uTMw1y1FObuo=
github.com/klauspost/compress v1.11.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.22.0 h1:OpwH5KDOJ9cS2bq8fD+KfT4IrksK0llvkHf4MZx42jQ=
github.com/valyala/fasthttp v1.22.0/go.mod h1:0mw2RjXGOzxf4NL2jni3gUQ7LfjjUSiG5sskOUUSEpU=
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20210226101413-39120d07d75e/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

364
main.go Normal file
View file

@ -0,0 +1,364 @@
// Package main is the new Codeberg Pages server, a solution for serving static pages from Gitea repositories.
//
// Mapping custom domains is not static anymore, but can be done with DNS:
//
// 1) add a "codeberg-pages-domains.txt" text file to your repository, containing the allowed domains
//
// 2) add a CNAME entry to your domain, pointing to "[[{branch}.]{repo}.]{owner}.codeberg.page" (repo defaults to
// "pages", "branch" defaults to the default branch if "repo" is "pages", or to "pages" if "repo" is something else):
// www.example.org. IN CNAME main.pages.example.codeberg.page.
// 3) if a CNAME is set for "www.example.org", you can redirect there from the naked domain by adding an ALIAS record
// for "example.org" (if your provider allows ALIAS or similar records):
// example.org IN ALIAS codeberg.page.
//
// Certificates are generated, updated and cleaned up automatically via Let's Encrypt through a TLS challenge.
package main
import (
"bytes"
"crypto/tls"
"fmt"
"mime"
"net"
"os"
"path"
"strconv"
"strings"
"time"
_ "embed"
"github.com/valyala/fasthttp"
)
// MainDomainSuffix specifies the main domain (starting with a dot) for which subdomains shall be served as static
// pages, or used for comparison in CNAME lookups. Static pages can be accessed through
// https://{owner}.{MainDomain}[/{repo}], with repo defaulting to "pages".
var MainDomainSuffix = []byte(".codeberg.page")
// GiteaRoot specifies the root URL of the Gitea instance, without a trailing slash.
var GiteaRoot = []byte("https://codeberg.org")
//go:embed 404.html
var NotFoundPage []byte
// BrokenDNSPage will be shown (with a redirect) when trying to access a domain for which no DNS CNAME record exists.
var BrokenDNSPage = "https://docs.codeberg.org/codeberg-pages/custom-domains/"
// RawDomain specifies the domain from which raw repository content shall be served in the following format:
// https://{RawDomain}/{owner}/{repo}[/{branch|tag|commit}/{version}]/{filepath...}
// (set to []byte(nil) to disable raw content hosting)
var RawDomain = []byte("raw.codeberg.page")
// RawInfoPage will be shown (with a redirect) when trying to access RawDomain directly (or without owner/repo/path).
var RawInfoPage = "https://docs.codeberg.org/codeberg-pages/raw-content/"
// AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed.
var AllowedCorsDomains = [][]byte{
RawDomain,
[]byte("fonts.codeberg.org"),
[]byte("design.codeberg.org"),
}
// BlacklistedPaths specifies forbidden path prefixes for all Codeberg Pages.
var BlacklistedPaths = [][]byte{
[]byte("/.well-known/acme-challenge/"),
}
// IndexPages lists pages that may be considered as index pages for directories.
var IndexPages = []string{
"index.html",
}
// ReservedUsernames specifies the usernames that are reserved by Gitea and thus may not be used as owner names.
// The contents are taken from https://github.com/go-gitea/gitea/blob/master/models/user.go#L783; reserved names with
// dots are removed as they are forbidden for Codeberg Pages anyways.
var ReservedUsernames = map[string]struct{}{
"admin": {},
"api": {},
"assets": {},
"attachments": {},
"avatars": {},
"captcha": {},
"commits": {},
"debug": {},
"error": {},
"explore": {},
"ghost": {},
"help": {},
"install": {},
"issues": {},
"less": {},
"login": {},
"metrics": {},
"milestones": {},
"new": {},
"notifications": {},
"org": {},
"plugins": {},
"pulls": {},
"raw": {},
"repo": {},
"search": {},
"stars": {},
"template": {},
"user": {},
}
// main sets up and starts the web server.
func main() {
// Make sure MainDomain has a trailing dot, and GiteaRoot has no trailing slash
if !bytes.HasPrefix(MainDomainSuffix, []byte{'.'}) {
MainDomainSuffix = append([]byte{'.'}, MainDomainSuffix...)
}
GiteaRoot = bytes.TrimSuffix(GiteaRoot, []byte{'/'})
// Use HOST and PORT environment variables to determine listening address
address := fmt.Sprintf("%s:%s", envOr("HOST", "[::]"), envOr("PORT", "80"))
fmt.Printf("Listening on http://%s\n", address)
// Enable compression by wrapping the handler() method with the compression function provided by FastHTTP
compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed)
// Setup listener and TLS
listener, err := net.Listen("tcp", address)
if err != nil {
fmt.Printf("Couldn't create listener: %s\n", err)
os.Exit(1)
}
if envOr("LETS_ENCRYPT", "0") == "1" {
tls.NewListener(listener, &tls.Config{
GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
// TODO: check DNS name & get certificate from Let's Encrypt
return nil, fmt.Errorf("NYI")
},
PreferServerCipherSuites: true,
// TODO: optimize cipher suites, minimum TLS version, etc.
})
}
// Start the web server
err = (&fasthttp.Server{
Handler: compressedHandler,
DisablePreParseMultipartForm: false,
MaxRequestBodySize: 0,
NoDefaultServerHeader: true,
ReadTimeout: 10 * time.Second,
}).Serve(listener)
if err != nil {
fmt.Printf("Couldn't start server: %s\n", err)
os.Exit(1)
}
}
// handler handles a single HTTP request to the web server.
func handler(ctx *fasthttp.RequestCtx) {
ctx.Response.Header.Set("Server", "Codeberg Pages")
// Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin
ctx.Response.Header.Set("Referrer-Policy", "strict-origin-when-cross-origin")
// Enable caching, but require revalidation to reduce confusion
ctx.Response.Header.Set("Cache-Control", "must-revalidate")
// Block all methods not required for static pages
if !ctx.IsGet() && !ctx.IsHead() && !ctx.IsOptions() {
ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS")
ctx.Error("Method not allowed", fasthttp.StatusMethodNotAllowed)
return
}
// Block blacklisted paths (like ACME challenges)
for _, blacklistedPath := range BlacklistedPaths {
if bytes.HasPrefix(ctx.Path(), blacklistedPath) {
returnErrorPage(ctx, fasthttp.StatusForbidden)
return
}
}
// Allow CORS for specified domains
if ctx.IsOptions() {
allowCors := false
for _, allowedCorsDomain := range AllowedCorsDomains {
if bytes.Equal(ctx.Request.Host(), allowedCorsDomain) {
allowCors = true
break
}
}
if allowCors {
ctx.Response.Header.Set("Access-Control-Allow-Origin", "*")
ctx.Response.Header.Set("Access-Control-Allow-Methods", "GET, HEAD")
}
ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS")
ctx.Response.Header.SetStatusCode(fasthttp.StatusNoContent)
return
}
// Prepare request information to Gitea
var targetOwner, targetRepo, targetPath string
var targetOptions = upstreamOptions{
ForbiddenMimeTypes: map[string]struct{}{},
TryIndexPages: true,
}
var alsoTryPagesRepo = false // Also try to treat the repo as the first path element & fall back to the "pages" repo
if RawDomain != nil && bytes.Equal(ctx.Request.Host(), RawDomain) {
// Serve raw content from RawDomain
targetOptions.TryIndexPages = false
targetOptions.ForbiddenMimeTypes["text/html"] = struct{}{}
pathElements := strings.SplitN(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/", 3)
if len(pathElements) < 3 {
// https://{RawDomain}/{owner}/{repo}/{path} is required
ctx.Redirect(RawInfoPage, fasthttp.StatusTemporaryRedirect)
return
}
targetOwner = pathElements[0]
targetRepo = pathElements[1]
targetPath = pathElements[2]
} else if bytes.HasSuffix(ctx.Request.Host(), MainDomainSuffix) {
// Serve pages from subdomains of MainDomainSuffix
pathElements := strings.SplitN(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/", 2)
targetOwner = string(bytes.TrimSuffix(ctx.Request.Host(), MainDomainSuffix))
targetRepo = pathElements[0]
targetPath = pathElements[1]
alsoTryPagesRepo = true
} else {
// Serve pages from external domains
targetOwner, targetRepo, targetPath = getTargetFromDNS(ctx.Request.Host())
if targetOwner == "" {
ctx.Redirect(BrokenDNSPage, fasthttp.StatusTemporaryRedirect)
return
}
}
// Check if a username can't exist because it's reserved (we'd risk to hit a Gitea route in that case)
if _, ok := ReservedUsernames[targetOwner]; ok {
returnErrorPage(ctx, fasthttp.StatusForbidden)
return
}
// Pass request to Gitea
url := "/" + targetOwner + "/" + targetRepo + "/raw/" + targetPath
if strings.HasPrefix(targetPath, "blob/") {
returnErrorPage(ctx, fasthttp.StatusForbidden)
return
}
// Try target
if upstream(ctx, url, targetOptions) {
return
}
// Try target with pages repo
if alsoTryPagesRepo {
targetPath = targetRepo + "/" + targetPath
targetRepo = "pages"
url := "/" + targetOwner + "/" + targetRepo + "/raw/" + targetPath
if strings.HasPrefix(targetPath, "blob/") {
returnErrorPage(ctx, fasthttp.StatusForbidden)
return
}
if upstream(ctx, url, targetOptions) {
return
}
}
returnErrorPage(ctx, fasthttp.StatusNotFound)
}
func getTargetFromDNS(host []byte) (targetOwner, targetRepo, targetPath string) {
// TODO: read CNAME record for host and "www.{host}" to get those values
// TODO: check codeberg-pages-domains.txt
return
}
// returnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced
// with the provided status code.
func returnErrorPage(ctx *fasthttp.RequestCtx, code int) {
ctx.Response.SetStatusCode(code)
ctx.Response.SetBody(bytes.ReplaceAll(NotFoundPage, []byte("%status"), []byte(strconv.Itoa(code))))
}
// upstream requests an URL from GiteaRoot and writes it to the request context; if "final" is set, it also returns a
// 404 error if the page couldn't be loaded.
func upstream(ctx *fasthttp.RequestCtx, url string, options upstreamOptions) (success bool) {
// Prepare necessary (temporary) variables with default values
body := make([]byte, 0)
if options.ForbiddenMimeTypes == nil {
options.ForbiddenMimeTypes = map[string]struct{}{}
}
// Make a request to the upstream URL
status, body, err := fasthttp.GetTimeout(body, string(GiteaRoot) + url, 10 * time.Second)
// Handle errors
if err != nil {
// Connection error, probably Gitea or the internet connection is down?
fmt.Printf("Couldn't fetch URL \"%s\": %s", url, err)
ctx.Response.SetStatusCode(fasthttp.StatusBadGateway)
return false
}
if status != 200 {
if options.TryIndexPages {
// copy the options struct & try if an index page exists
optionsForIndexPages := options
optionsForIndexPages.TryIndexPages = false
optionsForIndexPages.AppendTrailingSlash = true
for _, indexPage := range IndexPages {
if upstream(ctx, url + "/" + indexPage, optionsForIndexPages) {
return true
}
}
}
ctx.Response.SetStatusCode(status)
return false
}
// Append trailing slash if missing (for index files)
if options.AppendTrailingSlash && !bytes.HasSuffix(ctx.Request.URI().Path(), []byte{'/'}) {
ctx.Redirect(string(ctx.Request.URI().Path()) + "/", fasthttp.StatusTemporaryRedirect)
return true
}
// Set the MIME type
mimeType := mime.TypeByExtension(path.Ext(url))
mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
if _, ok := options.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" {
if options.DefaultMimeType != "" {
mimeType = options.DefaultMimeType
} else {
mimeType = "application/octet-stream"
}
}
ctx.Response.Header.SetContentType(mimeType)
// TODO: enable Caching - set Date header and respect If-Modified-Since!
// Set the response body
ctx.Response.SetStatusCode(fasthttp.StatusOK)
ctx.Response.SetBody(body)
return true
}
// upstreamOptions provides various options for the upstream request.
type upstreamOptions struct {
DefaultMimeType string
ForbiddenMimeTypes map[string]struct{}
TryIndexPages bool
AppendTrailingSlash bool
}
// envOr reads an environment variable and returns a default value if it's empty.
func envOr(env string, or string) string {
if v := os.Getenv(env); v != "" {
return v
}
return or
}