pages-server/server/certificates/certificates.go
Felipe Leopoldo Sologuren Gutiérrez c286b3b1d0 Added TokenBucket to limit the rate of validation failures (#151)
Added new TockenBucket named `acmeClientFailLimit` to avoid being banned because of the [Failed validation limit](https://letsencrypt.org/docs/failed-validation-limit/) of Let's Encrypt.

The behaviour is similar to the other limiters blocking the `obtainCert` func ensuring rate under limit.

Co-authored-by: fsologureng <sologuren@estudiohum.cl>
Co-authored-by: 6543 <6543@obermui.de>
Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/151
Reviewed-by: 6543 <6543@obermui.de>
Co-authored-by: Felipe Leopoldo Sologuren Gutiérrez <fsologureng@noreply.codeberg.org>
Co-committed-by: Felipe Leopoldo Sologuren Gutiérrez <fsologureng@noreply.codeberg.org>
2023-01-04 05:26:14 +00:00

553 lines
18 KiB
Go

package certificates
import (
"bytes"
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"encoding/gob"
"encoding/json"
"errors"
"fmt"
"os"
"strconv"
"strings"
"sync"
"time"
"github.com/go-acme/lego/v4/certcrypto"
"github.com/go-acme/lego/v4/certificate"
"github.com/go-acme/lego/v4/challenge"
"github.com/go-acme/lego/v4/challenge/tlsalpn01"
"github.com/go-acme/lego/v4/lego"
"github.com/go-acme/lego/v4/providers/dns"
"github.com/go-acme/lego/v4/registration"
"github.com/reugn/equalizer"
"github.com/rs/zerolog/log"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/database"
dnsutils "codeberg.org/codeberg/pages/server/dns"
"codeberg.org/codeberg/pages/server/gitea"
"codeberg.org/codeberg/pages/server/upstream"
)
// TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates.
func TLSConfig(mainDomainSuffix string,
giteaClient *gitea.Client,
dnsProvider string,
acmeUseRateLimits bool,
keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey,
certDB database.CertDB,
) *tls.Config {
return &tls.Config{
// check DNS name & get certificate from Let's Encrypt
GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
sni := strings.ToLower(strings.TrimSpace(info.ServerName))
if len(sni) < 1 {
return nil, errors.New("missing sni")
}
if info.SupportedProtos != nil {
for _, proto := range info.SupportedProtos {
if proto != tlsalpn01.ACMETLS1Protocol {
continue
}
challenge, ok := challengeCache.Get(sni)
if !ok {
return nil, errors.New("no challenge for this domain")
}
cert, err := tlsalpn01.ChallengeCert(sni, challenge.(string))
if err != nil {
return nil, err
}
return cert, nil
}
}
targetOwner := ""
if strings.HasSuffix(sni, mainDomainSuffix) || strings.EqualFold(sni, mainDomainSuffix[1:]) {
// deliver default certificate for the main domain (*.codeberg.page)
sni = mainDomainSuffix
} else {
var targetRepo, targetBranch string
targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(sni, mainDomainSuffix, dnsLookupCache)
if targetOwner == "" {
// DNS not set up, return main certificate to redirect to the docs
sni = mainDomainSuffix
} else {
targetOpt := &upstream.Options{
TargetOwner: targetOwner,
TargetRepo: targetRepo,
TargetBranch: targetBranch,
}
_, valid := targetOpt.CheckCanonicalDomain(giteaClient, sni, mainDomainSuffix, canonicalDomainCache)
if !valid {
sni = mainDomainSuffix
}
}
}
if tlsCertificate, ok := keyCache.Get(sni); ok {
// we can use an existing certificate object
return tlsCertificate.(*tls.Certificate), nil
}
var tlsCertificate tls.Certificate
var err error
var ok bool
if tlsCertificate, ok = retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB); !ok {
// request a new certificate
if strings.EqualFold(sni, mainDomainSuffix) {
return nil, errors.New("won't request certificate for main domain, something really bad has happened")
}
tlsCertificate, err = obtainCert(acmeClient, []string{sni}, nil, targetOwner, dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
if err != nil {
return nil, err
}
}
if err := keyCache.Set(sni, &tlsCertificate, 15*time.Minute); err != nil {
return nil, err
}
return &tlsCertificate, nil
},
PreferServerCipherSuites: true,
NextProtos: []string{
"h2",
"http/1.1",
tlsalpn01.ACMETLS1Protocol,
},
// generated 2021-07-13, Mozilla Guideline v5.6, Go 1.14.4, intermediate configuration
// https://ssl-config.mozilla.org/#server=go&version=1.14.4&config=intermediate&guideline=5.6
MinVersion: tls.VersionTLS12,
CipherSuites: []uint16{
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
},
}
}
func checkUserLimit(user string) error {
userLimit, ok := acmeClientCertificateLimitPerUser[user]
if !ok {
// Each Codeberg user can only add 10 new domains per day.
userLimit = equalizer.NewTokenBucket(10, time.Hour*24)
acmeClientCertificateLimitPerUser[user] = userLimit
}
if !userLimit.Ask() {
return errors.New("rate limit exceeded: 10 certificates per user per 24 hours")
}
return nil
}
var (
acmeClient, mainDomainAcmeClient *lego.Client
acmeClientCertificateLimitPerUser = map[string]*equalizer.TokenBucket{}
)
// rate limit is 300 / 3 hours, we want 200 / 2 hours but to refill more often, so that's 25 new domains every 15 minutes
// TODO: when this is used a lot, we probably have to think of a somewhat better solution?
var acmeClientOrderLimit = equalizer.NewTokenBucket(25, 15*time.Minute)
// rate limit is 20 / second, we want 5 / second (especially as one cert takes at least two requests)
var acmeClientRequestLimit = equalizer.NewTokenBucket(5, 1*time.Second)
// rate limit is 5 / hour https://letsencrypt.org/docs/failed-validation-limit/
var acmeClientFailLimit = equalizer.NewTokenBucket(5, 1*time.Hour)
type AcmeTLSChallengeProvider struct {
challengeCache cache.SetGetKey
}
// make sure AcmeTLSChallengeProvider match Provider interface
var _ challenge.Provider = AcmeTLSChallengeProvider{}
func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error {
return a.challengeCache.Set(domain, keyAuth, 1*time.Hour)
}
func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error {
a.challengeCache.Remove(domain)
return nil
}
type AcmeHTTPChallengeProvider struct {
challengeCache cache.SetGetKey
}
// make sure AcmeHTTPChallengeProvider match Provider interface
var _ challenge.Provider = AcmeHTTPChallengeProvider{}
func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error {
return a.challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour)
}
func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
a.challengeCache.Remove(domain + "/" + token)
return nil
}
func retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (tls.Certificate, bool) {
// parse certificate from database
res, err := certDB.Get(sni)
if err != nil {
panic(err) // TODO: no panic
}
if res == nil {
return tls.Certificate{}, false
}
tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey)
if err != nil {
panic(err)
}
// TODO: document & put into own function
if !strings.EqualFold(sni, mainDomainSuffix) {
tlsCertificate.Leaf, err = x509.ParseCertificate(tlsCertificate.Certificate[0])
if err != nil {
panic(err)
}
// renew certificates 7 days before they expire
if tlsCertificate.Leaf.NotAfter.Before(time.Now().Add(7 * 24 * time.Hour)) {
// TODO: add ValidUntil to custom res struct
if res.CSR != nil && len(res.CSR) > 0 {
// CSR stores the time when the renewal shall be tried again
nextTryUnix, err := strconv.ParseInt(string(res.CSR), 10, 64)
if err == nil && time.Now().Before(time.Unix(nextTryUnix, 0)) {
return tlsCertificate, true
}
}
go (func() {
res.CSR = nil // acme client doesn't like CSR to be set
tlsCertificate, err = obtainCert(acmeClient, []string{sni}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
if err != nil {
log.Error().Msgf("Couldn't renew certificate for %s: %v", sni, err)
}
})()
}
}
return tlsCertificate, true
}
var obtainLocks = sync.Map{}
func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user, dnsProvider, mainDomainSuffix string, acmeUseRateLimits bool, keyDatabase database.CertDB) (tls.Certificate, error) {
name := strings.TrimPrefix(domains[0], "*")
if dnsProvider == "" && len(domains[0]) > 0 && domains[0][0] == '*' {
domains = domains[1:]
}
// lock to avoid simultaneous requests
_, working := obtainLocks.LoadOrStore(name, struct{}{})
if working {
for working {
time.Sleep(100 * time.Millisecond)
_, working = obtainLocks.Load(name)
}
cert, ok := retrieveCertFromDB(name, mainDomainSuffix, dnsProvider, acmeUseRateLimits, keyDatabase)
if !ok {
return tls.Certificate{}, errors.New("certificate failed in synchronous request")
}
return cert, nil
}
defer obtainLocks.Delete(name)
if acmeClient == nil {
return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!", mainDomainSuffix, keyDatabase), nil
}
// request actual cert
var res *certificate.Resource
var err error
if renew != nil && renew.CertURL != "" {
if acmeUseRateLimits {
acmeClientRequestLimit.Take()
}
log.Debug().Msgf("Renewing certificate for: %v", domains)
res, err = acmeClient.Certificate.Renew(*renew, true, false, "")
if err != nil {
log.Error().Err(err).Msgf("Couldn't renew certificate for %v, trying to request a new one", domains)
if acmeUseRateLimits {
acmeClientFailLimit.Take()
}
res = nil
}
}
if res == nil {
if user != "" {
if err := checkUserLimit(user); err != nil {
return tls.Certificate{}, err
}
}
if acmeUseRateLimits {
acmeClientOrderLimit.Take()
acmeClientRequestLimit.Take()
}
log.Debug().Msgf("Re-requesting new certificate for %v", domains)
res, err = acmeClient.Certificate.Obtain(certificate.ObtainRequest{
Domains: domains,
Bundle: true,
MustStaple: false,
})
if acmeUseRateLimits && err != nil {
acmeClientFailLimit.Take()
}
}
if err != nil {
log.Error().Err(err).Msgf("Couldn't obtain again a certificate or %v", domains)
if renew != nil && renew.CertURL != "" {
tlsCertificate, err := tls.X509KeyPair(renew.Certificate, renew.PrivateKey)
if err != nil {
return mockCert(domains[0], err.Error(), mainDomainSuffix, keyDatabase), err
}
leaf, err := leaf(&tlsCertificate)
if err == nil && leaf.NotAfter.After(time.Now()) {
// avoid sending a mock cert instead of a still valid cert, instead abuse CSR field to store time to try again at
renew.CSR = []byte(strconv.FormatInt(time.Now().Add(6*time.Hour).Unix(), 10))
if err := keyDatabase.Put(name, renew); err != nil {
return mockCert(domains[0], err.Error(), mainDomainSuffix, keyDatabase), err
}
return tlsCertificate, nil
}
}
return mockCert(domains[0], err.Error(), mainDomainSuffix, keyDatabase), err
}
log.Debug().Msgf("Obtained certificate for %v", domains)
if err := keyDatabase.Put(name, res); err != nil {
return tls.Certificate{}, err
}
tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey)
if err != nil {
return tls.Certificate{}, err
}
return tlsCertificate, nil
}
func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcceptTerms bool) (*lego.Config, error) {
const configFile = "acme-account.json"
var myAcmeAccount AcmeAccount
var myAcmeConfig *lego.Config
if account, err := os.ReadFile(configFile); err == nil {
if err := json.Unmarshal(account, &myAcmeAccount); err != nil {
return nil, err
}
myAcmeAccount.Key, err = certcrypto.ParsePEMPrivateKey([]byte(myAcmeAccount.KeyPEM))
if err != nil {
return nil, err
}
myAcmeConfig = lego.NewConfig(&myAcmeAccount)
myAcmeConfig.CADirURL = acmeAPI
myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
// Validate Config
_, err := lego.NewClient(myAcmeConfig)
if err != nil {
// TODO: should we fail hard instead?
log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
}
return myAcmeConfig, nil
} else if !os.IsNotExist(err) {
return nil, err
}
privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
return nil, err
}
myAcmeAccount = AcmeAccount{
Email: acmeMail,
Key: privateKey,
KeyPEM: string(certcrypto.PEMEncode(privateKey)),
}
myAcmeConfig = lego.NewConfig(&myAcmeAccount)
myAcmeConfig.CADirURL = acmeAPI
myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
tempClient, err := lego.NewClient(myAcmeConfig)
if err != nil {
log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
} else {
// accept terms & log in to EAB
if acmeEabKID == "" || acmeEabHmac == "" {
reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: acmeAcceptTerms})
if err != nil {
log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
} else {
myAcmeAccount.Registration = reg
}
} else {
reg, err := tempClient.Registration.RegisterWithExternalAccountBinding(registration.RegisterEABOptions{
TermsOfServiceAgreed: acmeAcceptTerms,
Kid: acmeEabKID,
HmacEncoded: acmeEabHmac,
})
if err != nil {
log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
} else {
myAcmeAccount.Registration = reg
}
}
if myAcmeAccount.Registration != nil {
acmeAccountJSON, err := json.Marshal(myAcmeAccount)
if err != nil {
log.Error().Err(err).Msg("json.Marshalfailed, waiting for manual restart to avoid rate limits")
select {}
}
err = os.WriteFile(configFile, acmeAccountJSON, 0o600)
if err != nil {
log.Error().Err(err).Msg("os.WriteFile failed, waiting for manual restart to avoid rate limits")
select {}
}
}
}
return myAcmeConfig, nil
}
func SetupCertificates(mainDomainSuffix, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) error {
// getting main cert before ACME account so that we can fail here without hitting rate limits
mainCertBytes, err := certDB.Get(mainDomainSuffix)
if err != nil {
return fmt.Errorf("cert database is not working")
}
acmeClient, err = lego.NewClient(acmeConfig)
if err != nil {
log.Fatal().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
} else {
err = acmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
if err != nil {
log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider")
}
if enableHTTPServer {
err = acmeClient.Challenge.SetHTTP01Provider(AcmeHTTPChallengeProvider{challengeCache})
if err != nil {
log.Error().Err(err).Msg("Can't create HTTP-01 provider")
}
}
}
mainDomainAcmeClient, err = lego.NewClient(acmeConfig)
if err != nil {
log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
} else {
if dnsProvider == "" {
// using mock server, don't use wildcard certs
err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
if err != nil {
log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider")
}
} else {
provider, err := dns.NewDNSChallengeProviderByName(dnsProvider)
if err != nil {
log.Error().Err(err).Msg("Can't create DNS Challenge provider")
}
err = mainDomainAcmeClient.Challenge.SetDNS01Provider(provider)
if err != nil {
log.Error().Err(err).Msg("Can't create DNS-01 provider")
}
}
}
if mainCertBytes == nil {
_, err = obtainCert(mainDomainAcmeClient, []string{"*" + mainDomainSuffix, mainDomainSuffix[1:]}, nil, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
if err != nil {
log.Error().Err(err).Msg("Couldn't renew main domain certificate, continuing with mock certs only")
}
}
return nil
}
func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffix, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) {
for {
// clean up expired certs
now := time.Now()
expiredCertCount := 0
keyDatabaseIterator := certDB.Items()
key, resBytes, err := keyDatabaseIterator.Next()
for err == nil {
if !strings.EqualFold(string(key), mainDomainSuffix) {
resGob := bytes.NewBuffer(resBytes)
resDec := gob.NewDecoder(resGob)
res := &certificate.Resource{}
err = resDec.Decode(res)
if err != nil {
panic(err)
}
tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
if err != nil || tlsCertificates[0].NotAfter.Before(now) {
err := certDB.Delete(string(key))
if err != nil {
log.Error().Err(err).Msgf("Deleting expired certificate for %q failed", string(key))
} else {
expiredCertCount++
}
}
}
key, resBytes, err = keyDatabaseIterator.Next()
}
log.Debug().Msgf("Removed %d expired certificates from the database", expiredCertCount)
// compact the database
msg, err := certDB.Compact()
if err != nil {
log.Error().Err(err).Msg("Compacting key database failed")
} else {
log.Debug().Msgf("Compacted key database: %s", msg)
}
// update main cert
res, err := certDB.Get(mainDomainSuffix)
if err != nil {
log.Error().Msgf("Couldn't get cert for domain %q", mainDomainSuffix)
} else if res == nil {
log.Error().Msgf("Couldn't renew certificate for main domain %q expected main domain cert to exist, but it's missing - seems like the database is corrupted", mainDomainSuffix)
} else {
tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
// renew main certificate 30 days before it expires
if tlsCertificates[0].NotAfter.Before(time.Now().Add(30 * 24 * time.Hour)) {
go (func() {
_, err = obtainCert(mainDomainAcmeClient, []string{"*" + mainDomainSuffix, mainDomainSuffix[1:]}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
if err != nil {
log.Error().Err(err).Msg("Couldn't renew certificate for main domain")
}
})()
}
}
select {
case <-ctx.Done():
return
case <-time.After(interval):
}
}
}
// leaf returns the parsed leaf certificate, either from c.leaf or by parsing
// the corresponding c.Certificate[0].
func leaf(c *tls.Certificate) (*x509.Certificate, error) {
if c.Leaf != nil {
return c.Leaf, nil
}
return x509.ParseCertificate(c.Certificate[0])
}