Use hashicorp's LRU cache for DNS & certificates

DNS caching is also limited to 30 seconds now instead of 5 minutes
This commit is contained in:
Moritz Marquardt 2024-04-16 22:22:09 +02:00
parent 7694deec83
commit 18d09a163c
8 changed files with 49 additions and 36 deletions

1
go.mod
View file

@ -10,6 +10,7 @@ require (
github.com/creasty/defaults v1.7.0 github.com/creasty/defaults v1.7.0
github.com/go-acme/lego/v4 v4.5.3 github.com/go-acme/lego/v4 v4.5.3
github.com/go-sql-driver/mysql v1.6.0 github.com/go-sql-driver/mysql v1.6.0
github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/joho/godotenv v1.4.0 github.com/joho/godotenv v1.4.0
github.com/lib/pq v1.10.7 github.com/lib/pq v1.10.7
github.com/mattn/go-sqlite3 v1.14.16 github.com/mattn/go-sqlite3 v1.14.16

2
go.sum
View file

@ -332,6 +332,8 @@ github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=

View file

@ -6,12 +6,11 @@ import (
"crypto/x509" "crypto/x509"
"errors" "errors"
"fmt" "fmt"
"github.com/hashicorp/golang-lru/v2"
"strconv" "strconv"
"strings" "strings"
"time" "time"
"github.com/OrlovEvgeny/go-mcache"
"github.com/go-acme/lego/v4/certcrypto" "github.com/go-acme/lego/v4/certcrypto"
"github.com/go-acme/lego/v4/certificate" "github.com/go-acme/lego/v4/certificate"
"github.com/go-acme/lego/v4/challenge/tlsalpn01" "github.com/go-acme/lego/v4/challenge/tlsalpn01"
@ -28,12 +27,14 @@ import (
var ErrUserRateLimitExceeded = errors.New("rate limit exceeded: 10 certificates per user per 24 hours") var ErrUserRateLimitExceeded = errors.New("rate limit exceeded: 10 certificates per user per 24 hours")
var keyCache *lru.Cache[string, tls.Certificate]
// TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates. // TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates.
func TLSConfig(mainDomainSuffix string, func TLSConfig(mainDomainSuffix string,
giteaClient *gitea.Client, giteaClient *gitea.Client,
acmeClient *AcmeClient, acmeClient *AcmeClient,
firstDefaultBranch string, firstDefaultBranch string,
keyCache *mcache.CacheDriver, challengeCache cache.ICache, dnsLookupCache *mcache.CacheDriver, canonicalDomainCache cache.ICache, challengeCache cache.ICache, canonicalDomainCache cache.ICache,
certDB database.CertDB, certDB database.CertDB,
noDNS01 bool, noDNS01 bool,
rawDomain string, rawDomain string,
@ -88,7 +89,7 @@ func TLSConfig(mainDomainSuffix string,
} }
} else { } else {
var targetRepo, targetBranch string var targetRepo, targetBranch string
targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch, dnsLookupCache) targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch)
if targetOwner == "" { if targetOwner == "" {
// DNS not set up, return main certificate to redirect to the docs // DNS not set up, return main certificate to redirect to the docs
domain = mainDomainSuffix domain = mainDomainSuffix
@ -107,9 +108,17 @@ func TLSConfig(mainDomainSuffix string,
} }
} }
if keyCache == nil {
var err error
keyCache, err = lru.New[string, tls.Certificate](4096)
if err != nil {
panic(err) // This should only happen if 4096 < 0 at the time of writing, which should be reason enough to panic.
}
}
if tlsCertificate, ok := keyCache.Get(domain); ok { if tlsCertificate, ok := keyCache.Get(domain); ok {
// we can use an existing certificate object // we can use an existing certificate object
return tlsCertificate.(*tls.Certificate), nil return &tlsCertificate, nil
} }
var tlsCertificate *tls.Certificate var tlsCertificate *tls.Certificate
@ -134,9 +143,7 @@ func TLSConfig(mainDomainSuffix string,
} }
} }
if err := keyCache.Set(domain, tlsCertificate, 15*time.Minute); err != nil { keyCache.Add(domain, *tlsCertificate)
return nil, err
}
return tlsCertificate, nil return tlsCertificate, nil
}, },
NextProtos: []string{ NextProtos: []string{

View file

@ -1,26 +1,38 @@
package dns package dns
import ( import (
"github.com/hashicorp/golang-lru/v2"
"net" "net"
"strings" "strings"
"time" "time"
"github.com/OrlovEvgeny/go-mcache"
) )
// lookupCacheTimeout specifies the timeout for the DNS lookup cache. type lookupCacheEntry struct {
var lookupCacheTimeout = 15 * time.Minute cachedName string
timestamp time.Time
}
var lookupCacheValidity = 30 * time.Second
var lookupCache *lru.Cache[string, lookupCacheEntry]
var defaultPagesRepo = "pages" var defaultPagesRepo = "pages"
// GetTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix. // GetTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix.
// If everything is fine, it returns the target data. // If everything is fine, it returns the target data.
func GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch string, dnsLookupCache *mcache.CacheDriver) (targetOwner, targetRepo, targetBranch string) { func GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch string) (targetOwner, targetRepo, targetBranch string) {
// Get CNAME or TXT // Get CNAME or TXT
var cname string var cname string
var err error var err error
if cachedName, ok := dnsLookupCache.Get(domain); ok {
cname = cachedName.(string) if lookupCache == nil {
lookupCache, err = lru.New[string, lookupCacheEntry](4096)
if err != nil {
panic(err) // This should only happen if 4096 < 0 at the time of writing, which should be reason enough to panic.
}
}
if entry, ok := lookupCache.Get(domain); ok && time.Now().Before(entry.timestamp.Add(lookupCacheValidity)) {
cname = entry.cachedName
} else { } else {
cname, err = net.LookupCNAME(domain) cname, err = net.LookupCNAME(domain)
cname = strings.TrimSuffix(cname, ".") cname = strings.TrimSuffix(cname, ".")
@ -38,7 +50,10 @@ func GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch string, dnsLo
} }
} }
} }
_ = dnsLookupCache.Set(domain, cname, lookupCacheTimeout) _ = lookupCache.Add(domain, lookupCacheEntry{
cname,
time.Now(),
})
} }
if cname == "" { if cname == "" {
return return

View file

@ -4,8 +4,6 @@ import (
"net/http" "net/http"
"strings" "strings"
"github.com/OrlovEvgeny/go-mcache"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"codeberg.org/codeberg/pages/config" "codeberg.org/codeberg/pages/config"
@ -25,7 +23,7 @@ const (
func Handler( func Handler(
cfg config.ServerConfig, cfg config.ServerConfig,
giteaClient *gitea.Client, giteaClient *gitea.Client,
dnsLookupCache *mcache.CacheDriver, canonicalDomainCache, redirectsCache cache.ICache, canonicalDomainCache, redirectsCache cache.ICache,
) http.HandlerFunc { ) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) { return func(w http.ResponseWriter, req *http.Request) {
log.Debug().Msg("\n----------------------------------------------------------") log.Debug().Msg("\n----------------------------------------------------------")
@ -110,7 +108,7 @@ func Handler(
trimmedHost, trimmedHost,
pathElements, pathElements,
cfg.PagesBranches[0], cfg.PagesBranches[0],
dnsLookupCache, canonicalDomainCache, redirectsCache) canonicalDomainCache, redirectsCache)
} }
} }
} }

View file

@ -5,8 +5,6 @@ import (
"path" "path"
"strings" "strings"
"github.com/OrlovEvgeny/go-mcache"
"codeberg.org/codeberg/pages/html" "codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/context" "codeberg.org/codeberg/pages/server/context"
@ -21,10 +19,10 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g
trimmedHost string, trimmedHost string,
pathElements []string, pathElements []string,
firstDefaultBranch string, firstDefaultBranch string,
dnsLookupCache *mcache.CacheDriver, canonicalDomainCache, redirectsCache cache.ICache, canonicalDomainCache, redirectsCache cache.ICache,
) { ) {
// Serve pages from custom domains // Serve pages from custom domains
targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, mainDomainSuffix, firstDefaultBranch, dnsLookupCache) targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, mainDomainSuffix, firstDefaultBranch)
if targetOwner == "" { if targetOwner == "" {
html.ReturnErrorPage(ctx, html.ReturnErrorPage(ctx,
"could not obtain repo owner from custom domain", "could not obtain repo owner from custom domain",
@ -55,7 +53,7 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g
return return
} else if canonicalDomain != trimmedHost { } else if canonicalDomain != trimmedHost {
// only redirect if the target is also a codeberg page! // only redirect if the target is also a codeberg page!
targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix, firstDefaultBranch, dnsLookupCache) targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix, firstDefaultBranch)
if targetOwner != "" { if targetOwner != "" {
ctx.Redirect("https://"+canonicalDomain+"/"+targetOpt.TargetPath, http.StatusTemporaryRedirect) ctx.Redirect("https://"+canonicalDomain+"/"+targetOpt.TargetPath, http.StatusTemporaryRedirect)
return return

View file

@ -6,8 +6,6 @@ import (
"testing" "testing"
"time" "time"
"github.com/OrlovEvgeny/go-mcache"
"codeberg.org/codeberg/pages/config" "codeberg.org/codeberg/pages/config"
"codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/gitea" "codeberg.org/codeberg/pages/server/gitea"
@ -31,7 +29,7 @@ func TestHandlerPerformance(t *testing.T) {
AllowedCorsDomains: []string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"}, AllowedCorsDomains: []string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"},
PagesBranches: []string{"pages"}, PagesBranches: []string{"pages"},
} }
testHandler := Handler(serverCfg, giteaClient, mcache.New(), cache.NewInMemoryCache(), cache.NewInMemoryCache()) testHandler := Handler(serverCfg, giteaClient, cache.NewInMemoryCache(), cache.NewInMemoryCache())
testCase := func(uri string, status int) { testCase := func(uri string, status int) {
t.Run(uri, func(t *testing.T) { t.Run(uri, func(t *testing.T) {

View file

@ -11,7 +11,6 @@ import (
"strings" "strings"
"time" "time"
"github.com/OrlovEvgeny/go-mcache"
"github.com/redis/go-redis/v9" "github.com/redis/go-redis/v9"
"github.com/rs/zerolog" "github.com/rs/zerolog"
@ -73,11 +72,6 @@ func Serve(ctx *cli.Context) error {
} }
defer closeFn() defer closeFn()
// keyCache stores the parsed certificate objects (Redis is no advantage here)
keyCache := mcache.New()
// dnsLookupCache stores DNS lookups for custom domains (Redis is no advantage here)
dnsLookupCache := mcache.New()
var redisErr error = nil var redisErr error = nil
createCache := func(name string) cache.ICache { createCache := func(name string) cache.ICache {
if cfg.Cache.RedisURL != "" { if cfg.Cache.RedisURL != "" {
@ -129,7 +123,7 @@ func Serve(ctx *cli.Context) error {
giteaClient, giteaClient,
acmeClient, acmeClient,
cfg.Server.PagesBranches[0], cfg.Server.PagesBranches[0],
keyCache, challengeCache, dnsLookupCache, canonicalDomainCache, challengeCache, canonicalDomainCache,
certDB, certDB,
cfg.ACME.NoDNS01, cfg.ACME.NoDNS01,
cfg.Server.RawDomain, cfg.Server.RawDomain,
@ -155,7 +149,7 @@ func Serve(ctx *cli.Context) error {
} }
// Create ssl handler based on settings // Create ssl handler based on settings
sslHandler := handler.Handler(cfg.Server, giteaClient, dnsLookupCache, canonicalDomainCache, redirectsCache) sslHandler := handler.Handler(cfg.Server, giteaClient, canonicalDomainCache, redirectsCache)
// Start the ssl listener // Start the ssl listener
log.Info().Msgf("Start SSL server using TCP listener on %s", listener.Addr()) log.Info().Msgf("Start SSL server using TCP listener on %s", listener.Addr())