Merge branch 'main' into move-more-logic-into-client

This commit is contained in:
6543 2022-08-12 05:09:34 +02:00
commit bddb4b9c5f
No known key found for this signature in database
GPG key ID: C99B82E40B027BAE
12 changed files with 107 additions and 92 deletions

View file

@ -228,7 +228,7 @@ func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUs
res.CSR = nil // acme client doesn't like CSR to be set
tlsCertificate, err = obtainCert(acmeClient, []string{string(sni)}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
if err != nil {
log.Printf("Couldn't renew certificate for %s: %s", sni, err)
log.Error().Msgf("Couldn't renew certificate for %s: %v", string(sni), err)
}
})()
}
@ -271,10 +271,10 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re
if acmeUseRateLimits {
acmeClientRequestLimit.Take()
}
log.Printf("Renewing certificate for %v", domains)
log.Debug().Msgf("Renewing certificate for: %v", domains)
res, err = acmeClient.Certificate.Renew(*renew, true, false, "")
if err != nil {
log.Printf("Couldn't renew certificate for %v, trying to request a new one: %s", domains, err)
log.Error().Err(err).Msgf("Couldn't renew certificate for %v, trying to request a new one", domains)
res = nil
}
}
@ -289,7 +289,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re
acmeClientOrderLimit.Take()
acmeClientRequestLimit.Take()
}
log.Printf("Requesting new certificate for %v", domains)
log.Debug().Msgf("Re-requesting new certificate for %v", domains)
res, err = acmeClient.Certificate.Obtain(certificate.ObtainRequest{
Domains: domains,
Bundle: true,
@ -297,7 +297,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re
})
}
if err != nil {
log.Printf("Couldn't obtain certificate for %v: %s", domains, err)
log.Error().Err(err).Msgf("Couldn't obtain again a certificate or %v", domains)
if renew != nil && renew.CertURL != "" {
tlsCertificate, err := tls.X509KeyPair(renew.Certificate, renew.PrivateKey)
if err == nil && tlsCertificate.Leaf.NotAfter.After(time.Now()) {
@ -311,7 +311,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re
}
return mockCert(domains[0], err.Error(), string(mainDomainSuffix), keyDatabase), err
}
log.Printf("Obtained certificate for %v", domains)
log.Debug().Msgf("Obtained certificate for %v", domains)
if err := keyDatabase.Put(name, res); err != nil {
return tls.Certificate{}, err
@ -344,7 +344,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
_, err := lego.NewClient(myAcmeConfig)
if err != nil {
// TODO: should we fail hard instead?
log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
}
return myAcmeConfig, nil
} else if !os.IsNotExist(err) {
@ -365,13 +365,13 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
tempClient, err := lego.NewClient(myAcmeConfig)
if err != nil {
log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
} else {
// accept terms & log in to EAB
if acmeEabKID == "" || acmeEabHmac == "" {
reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: acmeAcceptTerms})
if err != nil {
log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err)
log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
} else {
myAcmeAccount.Registration = reg
}
@ -382,7 +382,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
HmacEncoded: acmeEabHmac,
})
if err != nil {
log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err)
log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
} else {
myAcmeAccount.Registration = reg
}
@ -391,12 +391,12 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
if myAcmeAccount.Registration != nil {
acmeAccountJSON, err := json.Marshal(myAcmeAccount)
if err != nil {
log.Printf("[FAIL] Error during json.Marshal(myAcmeAccount), waiting for manual restart to avoid rate limits: %s", err)
log.Error().Err(err).Msg("json.Marshalfailed, waiting for manual restart to avoid rate limits")
select {}
}
err = os.WriteFile(configFile, acmeAccountJSON, 0o600)
if err != nil {
log.Printf("[FAIL] Error during os.WriteFile(\"acme-account.json\"), waiting for manual restart to avoid rate limits: %s", err)
log.Error().Err(err).Msg("os.WriteFile failed, waiting for manual restart to avoid rate limits")
select {}
}
}
@ -414,38 +414,38 @@ func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *
acmeClient, err = lego.NewClient(acmeConfig)
if err != nil {
log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
} else {
err = acmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
if err != nil {
log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err)
log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider")
}
if enableHTTPServer {
err = acmeClient.Challenge.SetHTTP01Provider(AcmeHTTPChallengeProvider{challengeCache})
if err != nil {
log.Printf("[ERROR] Can't create HTTP-01 provider: %s", err)
log.Error().Err(err).Msg("Can't create HTTP-01 provider")
}
}
}
mainDomainAcmeClient, err = lego.NewClient(acmeConfig)
if err != nil {
log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
} else {
if dnsProvider == "" {
// using mock server, don't use wildcard certs
err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
if err != nil {
log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err)
log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider")
}
} else {
provider, err := dns.NewDNSChallengeProviderByName(dnsProvider)
if err != nil {
log.Printf("[ERROR] Can't create DNS Challenge provider: %s", err)
log.Error().Err(err).Msg("Can't create DNS Challenge provider")
}
err = mainDomainAcmeClient.Challenge.SetDNS01Provider(provider)
if err != nil {
log.Printf("[ERROR] Can't create DNS-01 provider: %s", err)
log.Error().Err(err).Msg("Can't create DNS-01 provider")
}
}
}
@ -453,7 +453,7 @@ func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *
if mainCertBytes == nil {
_, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, nil, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
if err != nil {
log.Printf("[ERROR] Couldn't renew main domain certificate, continuing with mock certs only: %s", err)
log.Error().Err(err).Msg("Couldn't renew main domain certificate, continuing with mock certs only")
}
}
@ -481,7 +481,7 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
if err != nil || !tlsCertificates[0].NotAfter.After(now) {
err := certDB.Delete(string(key))
if err != nil {
log.Printf("[ERROR] Deleting expired certificate for %s failed: %s", string(key), err)
log.Error().Err(err).Msgf("Deleting expired certificate for %q failed", string(key))
} else {
expiredCertCount++
}
@ -489,22 +489,22 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
}
key, resBytes, err = keyDatabaseIterator.Next()
}
log.Printf("[INFO] Removed %d expired certificates from the database", expiredCertCount)
log.Debug().Msgf("Removed %d expired certificates from the database", expiredCertCount)
// compact the database
msg, err := certDB.Compact()
if err != nil {
log.Printf("[ERROR] Compacting key database failed: %s", err)
log.Error().Err(err).Msg("Compacting key database failed")
} else {
log.Printf("[INFO] Compacted key database (%s)", msg)
log.Debug().Msgf("Compacted key database: %s", msg)
}
// update main cert
res, err := certDB.Get(string(mainDomainSuffix))
if err != nil {
log.Err(err).Msgf("could not get cert for domain '%s'", mainDomainSuffix)
log.Error().Msgf("Couldn't get cert for domain %q", mainDomainSuffix)
} else if res == nil {
log.Error().Msgf("Couldn't renew certificate for main domain: %s", "expected main domain cert to exist, but it's missing - seems like the database is corrupted")
log.Error().Msgf("Couldn't renew certificate for main domain %q expected main domain cert to exist, but it's missing - seems like the database is corrupted", string(mainDomainSuffix))
} else {
tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
@ -513,7 +513,7 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
go (func() {
_, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
if err != nil {
log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", err)
log.Error().Err(err).Msg("Couldn't renew certificate for main domain")
}
})()
}

View file

@ -72,7 +72,7 @@ func (p aDB) sync() {
for {
err := p.intern.Sync()
if err != nil {
log.Err(err).Msg("Syncing cert database failed")
log.Error().Err(err).Msg("Syncing cert database failed")
}
select {
case <-p.ctx.Done():

View file

@ -85,7 +85,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
// also disallow search indexing and add a Link header to the canonical URL.
tryBranch := func(log zerolog.Logger, repo, branch string, path []string, canonicalLink string) bool {
if repo == "" {
log.Debug().Msg("tryBranch: repo == ''")
log.Debug().Msg("tryBranch: repo is empty")
return false
}
@ -120,10 +120,10 @@ func Handler(mainDomainSuffix, rawDomain []byte,
return true
}
log.Debug().Msg("preparations")
log.Debug().Msg("Preparing")
if rawDomain != nil && bytes.Equal(trimmedHost, rawDomain) {
// Serve raw content from RawDomain
log.Debug().Msg("raw domain")
log.Debug().Msg("Serving raw domain")
targetOptions.TryIndexPages = false
if targetOptions.ForbiddenMimeTypes == nil {
@ -143,28 +143,28 @@ func Handler(mainDomainSuffix, rawDomain []byte,
// raw.codeberg.org/example/myrepo/@main/index.html
if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") {
log.Debug().Msg("raw domain preparations, now trying with specified branch")
log.Debug().Msg("Preparing raw domain, now trying with specified branch")
if tryBranch(log,
targetRepo, pathElements[2][1:], pathElements[3:],
giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
) {
log.Debug().Msg("tryBranch, now trying upstream 1")
log.Info().Msg("tryBranch, now trying upstream 1")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
return
}
log.Debug().Msg("missing branch")
log.Warn().Msg("Path missed a branch")
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
return
}
log.Debug().Msg("raw domain preparations, now trying with default branch")
log.Debug().Msg("Preparing raw domain, now trying with default branch")
tryBranch(log,
targetRepo, "", pathElements[2:],
giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
)
log.Debug().Msg("tryBranch, now trying upstream 2")
log.Info().Msg("tryBranch, now trying upstream 2")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
@ -172,7 +172,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
} else if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
// Serve pages from subdomains of MainDomainSuffix
log.Debug().Msg("main domain suffix")
log.Info().Msg("Serve pages from main domain suffix")
pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
targetOwner = string(bytes.TrimSuffix(trimmedHost, mainDomainSuffix))
@ -194,16 +194,17 @@ func Handler(mainDomainSuffix, rawDomain []byte,
return
}
log.Debug().Msg("main domain preparations, now trying with specified repo & branch")
log.Debug().Msg("Preparing main domain, now trying with specified repo & branch")
if tryBranch(log,
pathElements[0], pathElements[1][1:], pathElements[2:],
"/"+pathElements[0]+"/%p",
) {
log.Debug().Msg("tryBranch, now trying upstream 3")
log.Info().Msg("tryBranch, now trying upstream 3")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
} else {
log.Warn().Msg("tryBranch: upstream 3 failed")
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
}
return
@ -212,14 +213,15 @@ func Handler(mainDomainSuffix, rawDomain []byte,
// Check if the first directory is a branch for the "pages" repo
// example.codeberg.page/@main/index.html
if strings.HasPrefix(pathElements[0], "@") {
log.Debug().Msg("main domain preparations, now trying with specified branch")
log.Debug().Msg("Preparing main domain, now trying with specified branch")
if tryBranch(log,
"pages", pathElements[0][1:], pathElements[1:], "/%p") {
log.Debug().Msg("tryBranch, now trying upstream 4")
log.Info().Msg("tryBranch, now trying upstream 4")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
} else {
log.Warn().Msg("tryBranch: upstream 4 failed")
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
}
return
@ -231,7 +233,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
log.Debug().Msg("main domain preparations, now trying with specified repo")
if pathElements[0] != "pages" && tryBranch(log,
pathElements[0], "pages", pathElements[1:], "") {
log.Debug().Msg("tryBranch, now trying upstream 5")
log.Info().Msg("tryBranch, now trying upstream 5")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
@ -243,7 +245,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
log.Debug().Msg("main domain preparations, now trying with default repo/branch")
if tryBranch(log,
"pages", "", pathElements, "") {
log.Debug().Msg("tryBranch, now trying upstream 6")
log.Info().Msg("tryBranch, now trying upstream 6")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
@ -251,6 +253,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
}
// Couldn't find a valid repo/branch
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
return
} else {
@ -272,11 +275,12 @@ func Handler(mainDomainSuffix, rawDomain []byte,
}
// Try to use the given repo on the given branch or the default branch
log.Debug().Msg("custom domain preparations, now trying with details from DNS")
log.Debug().Msg("Preparing custom domain, now trying with details from DNS")
if tryBranch(log,
targetRepo, targetBranch, pathElements, canonicalLink) {
canonicalDomain, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), canonicalDomainCache)
if !valid {
log.Warn().Msg("Custom domains, domain from DNS isn't valid/canonical")
html.ReturnErrorPage(ctx, fasthttp.StatusMisdirectedRequest)
return
} else if canonicalDomain != trimmedHostStr {
@ -287,17 +291,19 @@ func Handler(mainDomainSuffix, rawDomain []byte,
return
}
log.Warn().Msg("Custom domains, targetOwner from DNS is empty")
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
return
}
log.Debug().Msg("tryBranch, now trying upstream 7")
log.Info().Msg("tryBranch, now trying upstream 7 %s")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
return
}
log.Warn().Msg("Couldn't handle request, none of the options succeed")
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
return
}

View file

@ -2,11 +2,11 @@ package server
import (
"bytes"
"fmt"
"net/http"
"time"
"github.com/rs/zerolog/log"
"github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/server/cache"
@ -16,7 +16,7 @@ import (
type fasthttpLogger struct{}
func (fasthttpLogger) Printf(format string, args ...interface{}) {
log.Printf("[FASTHTTP] "+format, args...)
log.Printf("FastHTTP: %s", fmt.Sprintf(format, args...))
}
func SetupServer(handler fasthttp.RequestHandler) *fasthttp.Server {

View file

@ -41,6 +41,7 @@ func tryUpstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
targetOptions.TargetRepo = targetRepo
targetOptions.TargetBranch = targetBranch
targetOptions.TargetPath = targetPath
targetOptions.Host = string(trimmedHost)
// Try to request the file from the Gitea API
if !targetOptions.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {

View file

@ -3,7 +3,6 @@ package upstream
import (
"bytes"
"errors"
"fmt"
"io"
"strings"
"time"
@ -33,7 +32,10 @@ type Options struct {
TargetBranch,
TargetPath,
DefaultMimeType string
// Used for debugging purposes.
Host string
DefaultMimeType string
ForbiddenMimeTypes map[string]bool
TryIndexPages bool
BranchTimestamp time.Time
@ -44,7 +46,7 @@ type Options struct {
// Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) {
log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger()
log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath, o.Host}).Logger()
// Check if the branch exists and when it was modified
if o.BranchTimestamp.IsZero() {
@ -70,7 +72,8 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
return true
}
}
log.Debug().Msg("preparations")
log.Debug().Msg("Preparing")
// Make a GET request to the upstream URL
uri := o.generateUri()
@ -82,7 +85,7 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
} else {
res, err = giteaClient.ServeRawContent(o.generateUriClientArgs())
}
log.Debug().Msg("acquisition")
log.Debug().Msg("Aquisting")
// Handle errors
if (err != nil && errors.Is(err, gitea.ErrorNotFound)) || (res == nil && !cachedResponse.Exists) {
@ -136,7 +139,7 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
return false
}
if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) {
fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", uri, err, res.StatusCode())
log.Warn().Msgf("Couldn't fetch contents from %q: %v (status code %d)", uri, err, res.StatusCode())
html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
return true
}
@ -155,7 +158,7 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
ctx.Redirect(o.redirectIfExists, fasthttp.StatusTemporaryRedirect)
return true
}
log.Debug().Msg("error handling")
log.Debug().Msg("Handling error")
// Set the MIME type
mimeType := o.getMimeTypeByExtension()
@ -175,7 +178,7 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
}
ctx.Response.Header.SetLastModified(o.BranchTimestamp)
log.Debug().Msg("response preparations")
log.Debug().Msg("Prepare response")
// Write the response body to the original request
var cacheBodyWriter bytes.Buffer
@ -193,11 +196,11 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
_, err = ctx.Write(cachedResponse.Body)
}
if err != nil {
fmt.Printf("Couldn't write body for \"%s\": %s\n", uri, err)
log.Error().Err(err).Msgf("Couldn't write body for %q", uri)
html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
return true
}
log.Debug().Msg("response")
log.Debug().Msg("Sending response")
if res != nil && res.Header.ContentLength() <= fileCacheSizeLimit && ctx.Err() == nil {
cachedResponse.Exists = true