pages/pages-server
1
0
Fork 0
mirror of https://codeberg.org/Codeberg/pages-server.git synced 2025-04-25 06:16:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'main' into std-http

Browse source
This commit is contained in:
6543 2022-08-28 14:35:27 +02:00
commit e1c78d6069
No known key found for this signature in database
GPG key ID: C99B82E40B027BAE
17 changed files with 138 additions and 79 deletions
Download patch file Download diff file Expand all files Collapse all files

59
server/certificates/certificates.go
View file

@ -12,7 +12,6 @@ import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"os"
"strconv"
"strings"
@ -229,7 +228,7 @@ func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUs
res.CSR = nil // acme client doesn't like CSR to be set
tlsCertificate, err = obtainCert(acmeClient, []string{string(sni)}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
if err != nil {
log.Printf("Couldn't renew certificate for %s: %s", sni, err)
log.Error().Msgf("Couldn't renew certificate for %s: %v", string(sni), err)
}
})()
}
@ -272,10 +271,10 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re
if acmeUseRateLimits {
acmeClientRequestLimit.Take()
}
log.Printf("Renewing certificate for %v", domains)
log.Debug().Msgf("Renewing certificate for: %v", domains)
res, err = acmeClient.Certificate.Renew(*renew, true, false, "")
if err != nil {
log.Printf("Couldn't renew certificate for %v, trying to request a new one: %s", domains, err)
log.Error().Err(err).Msgf("Couldn't renew certificate for %v, trying to request a new one", domains)
res = nil
}
}
@ -290,7 +289,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re
acmeClientOrderLimit.Take()
acmeClientRequestLimit.Take()
}
log.Printf("Requesting new certificate for %v", domains)
log.Debug().Msgf("Re-requesting new certificate for %v", domains)
res, err = acmeClient.Certificate.Obtain(certificate.ObtainRequest{
Domains: domains,
Bundle: true,
@ -298,7 +297,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re
})
}
if err != nil {
log.Printf("Couldn't obtain certificate for %v: %s", domains, err)
log.Error().Err(err).Msgf("Couldn't obtain again a certificate or %v", domains)
if renew != nil && renew.CertURL != "" {
tlsCertificate, err := tls.X509KeyPair(renew.Certificate, renew.PrivateKey)
if err == nil && tlsCertificate.Leaf.NotAfter.After(time.Now()) {
@ -312,7 +311,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re
}
return mockCert(domains[0], err.Error(), string(mainDomainSuffix), keyDatabase), err
}
log.Printf("Obtained certificate for %v", domains)
log.Debug().Msgf("Obtained certificate for %v", domains)
if err := keyDatabase.Put(name, res); err != nil {
return tls.Certificate{}, err
@ -329,7 +328,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
var myAcmeAccount AcmeAccount
var myAcmeConfig *lego.Config
if account, err := ioutil.ReadFile(configFile); err == nil {
if account, err := os.ReadFile(configFile); err == nil {
if err := json.Unmarshal(account, &myAcmeAccount); err != nil {
return nil, err
}
@ -345,7 +344,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
_, err := lego.NewClient(myAcmeConfig)
if err != nil {
// TODO: should we fail hard instead?
log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
}
return myAcmeConfig, nil
} else if !os.IsNotExist(err) {
@ -366,13 +365,13 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
tempClient, err := lego.NewClient(myAcmeConfig)
if err != nil {
log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
} else {
// accept terms & log in to EAB
if acmeEabKID == "" || acmeEabHmac == "" {
reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: acmeAcceptTerms})
if err != nil {
log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err)
log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
} else {
myAcmeAccount.Registration = reg
}
@ -383,7 +382,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
HmacEncoded: acmeEabHmac,
})
if err != nil {
log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err)
log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
} else {
myAcmeAccount.Registration = reg
}
@ -392,12 +391,12 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
if myAcmeAccount.Registration != nil {
acmeAccountJSON, err := json.Marshal(myAcmeAccount)
if err != nil {
log.Printf("[FAIL] Error during json.Marshal(myAcmeAccount), waiting for manual restart to avoid rate limits: %s", err)
log.Error().Err(err).Msg("json.Marshalfailed, waiting for manual restart to avoid rate limits")
select {}
}
err = ioutil.WriteFile(configFile, acmeAccountJSON, 0o600)
err = os.WriteFile(configFile, acmeAccountJSON, 0o600)
if err != nil {
log.Printf("[FAIL] Error during ioutil.WriteFile(\"acme-account.json\"), waiting for manual restart to avoid rate limits: %s", err)
log.Error().Err(err).Msg("os.WriteFile failed, waiting for manual restart to avoid rate limits")
select {}
}
}
@ -415,38 +414,38 @@ func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *
acmeClient, err = lego.NewClient(acmeConfig)
if err != nil {
log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
} else {
err = acmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
if err != nil {
log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err)
log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider")
}
if enableHTTPServer {
err = acmeClient.Challenge.SetHTTP01Provider(AcmeHTTPChallengeProvider{challengeCache})
if err != nil {
log.Printf("[ERROR] Can't create HTTP-01 provider: %s", err)
log.Error().Err(err).Msg("Can't create HTTP-01 provider")
}
}
}
mainDomainAcmeClient, err = lego.NewClient(acmeConfig)
if err != nil {
log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
} else {
if dnsProvider == "" {
// using mock server, don't use wildcard certs
err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
if err != nil {
log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err)
log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider")
}
} else {
provider, err := dns.NewDNSChallengeProviderByName(dnsProvider)
if err != nil {
log.Printf("[ERROR] Can't create DNS Challenge provider: %s", err)
log.Error().Err(err).Msg("Can't create DNS Challenge provider")
}
err = mainDomainAcmeClient.Challenge.SetDNS01Provider(provider)
if err != nil {
log.Printf("[ERROR] Can't create DNS-01 provider: %s", err)
log.Error().Err(err).Msg("Can't create DNS-01 provider")
}
}
}
@ -454,7 +453,7 @@ func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *
if mainCertBytes == nil {
_, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, nil, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
if err != nil {
log.Printf("[ERROR] Couldn't renew main domain certificate, continuing with mock certs only: %s", err)
log.Error().Err(err).Msg("Couldn't renew main domain certificate, continuing with mock certs only")
}
}
@ -482,7 +481,7 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
if err != nil || !tlsCertificates[0].NotAfter.After(now) {
err := certDB.Delete(string(key))
if err != nil {
log.Printf("[ERROR] Deleting expired certificate for %s failed: %s", string(key), err)
log.Error().Err(err).Msgf("Deleting expired certificate for %q failed", string(key))
} else {
expiredCertCount++
}
@ -490,22 +489,22 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
}
key, resBytes, err = keyDatabaseIterator.Next()
}
log.Printf("[INFO] Removed %d expired certificates from the database", expiredCertCount)
log.Debug().Msgf("Removed %d expired certificates from the database", expiredCertCount)
// compact the database
msg, err := certDB.Compact()
if err != nil {
log.Printf("[ERROR] Compacting key database failed: %s", err)
log.Error().Err(err).Msg("Compacting key database failed")
} else {
log.Printf("[INFO] Compacted key database (%s)", msg)
log.Debug().Msgf("Compacted key database: %s", msg)
}
// update main cert
res, err := certDB.Get(string(mainDomainSuffix))
if err != nil {
log.Err(err).Msgf("could not get cert for domain '%s'", mainDomainSuffix)
log.Error().Msgf("Couldn't get cert for domain %q", mainDomainSuffix)
} else if res == nil {
log.Error().Msgf("Couldn't renew certificate for main domain: %s", "expected main domain cert to exist, but it's missing - seems like the database is corrupted")
log.Error().Msgf("Couldn't renew certificate for main domain %q expected main domain cert to exist, but it's missing - seems like the database is corrupted", string(mainDomainSuffix))
} else {
tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
@ -514,7 +513,7 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
go (func() {
_, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
if err != nil {
log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", err)
log.Error().Err(err).Msg("Couldn't renew certificate for main domain")
}
})()
}

2
server/database/setup.go
View file

@ -72,7 +72,7 @@ func (p aDB) sync() {
for {
err := p.intern.Sync()
if err != nil {
log.Err(err).Msg("Syncing cert database failed")
log.Error().Err(err).Msg("Syncing cert database failed")
}
select {
case <-p.ctx.Done():

1
server/gitea/client.go
View file

@ -9,4 +9,5 @@ var ErrorNotFound = errors.New("not found")
const (
branchTimestampCacheKeyPrefix = "branchTime"
defaultBranchCacheKeyPrefix = "defaultBranch"
giteaObjectTypeHeader = "X-Gitea-Object-Type"
)

8
server/gitea/client_fasthttp.go
View file

@ -16,8 +16,7 @@ import (
)
const (
giteaAPIRepos = "/api/v1/repos/"
giteaObjectTypeHeader = "X-Gitea-Object-Type"
giteaAPIRepos = "/api/v1/repos/"
)
type Client struct {
@ -71,7 +70,10 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
switch resp.StatusCode() {
case fasthttp.StatusOK:
if client.followSymlinks && string(resp.Header.Peek(giteaObjectTypeHeader)) == "symlink" {
objType := string(resp.Header.Peek(giteaObjectTypeHeader))
log.Trace().Msgf("server raw content object: %s", objType)
if client.followSymlinks && objType == "symlink" {
// TODO: limit to 1000 chars if we switched to std
linkDest := strings.TrimSpace(string(resp.Body()))
log.Debug().Msgf("follow symlink from '%s' to '%s'", resource, linkDest)
return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest)

25
server/gitea/client_std.go
View file

@ -11,6 +11,7 @@ import (
"time"
"code.gitea.io/sdk/gitea"
"github.com/rs/zerolog/log"
"codeberg.org/codeberg/pages/server/cache"
)
@ -25,14 +26,19 @@ type Client struct {
func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, followSymlinks, supportLFS bool) (*Client, error) {
rootURL, err := url.Parse(giteaRoot)
if err != nil {
return nil, err
}
giteaRoot = strings.Trim(rootURL.String(), "/")
stdClient := http.Client{Timeout: 10 * time.Second}
sdk, err := gitea.NewClient(giteaRoot, gitea.SetHTTPClient(&stdClient), gitea.SetToken(giteaAPIToken))
return &Client{
sdkClient: sdk,
responseCache: respCache,
sdkClient: sdk,
responseCache: respCache,
followSymlinks: followSymlinks,
supportLFS: supportLFS,
}, err
}
@ -78,6 +84,21 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
// _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), cachedResponse, fileCacheTimeout)
// }
objType := resp.Header.Get(giteaObjectTypeHeader)
log.Trace().Msgf("server raw content object: %s", objType)
if client.followSymlinks && objType == "symlink" {
// limit to 1000 chars
defer reader.Close()
linkDestBytes, err := io.ReadAll(io.LimitReader(reader, 10000))
if err != nil {
return nil, nil, err
}
linkDest := strings.TrimSpace(string(linkDestBytes))
log.Debug().Msgf("follow symlink from '%s' to '%s'", resource, linkDest)
return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest)
}
return reader, resp.Response, err
case http.StatusNotFound:

2
server/handler.go
View file

@ -28,7 +28,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
dnsLookupCache, canonicalDomainCache cache.SetGetKey,
) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
log := log.With().Str("Handler", req.RequestURI).Logger()
log := log.With().Strs("Handler", []string{string(req.Host), req.RequestURI}).Logger()
w.Header().Set("Server", "CodebergPages/"+version.Version)

2
server/handler_fasthttp.go
View file

@ -27,7 +27,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
dnsLookupCache, canonicalDomainCache cache.SetGetKey,
) func(ctx *fasthttp.RequestCtx) {
return func(ctx *fasthttp.RequestCtx) {
log := log.With().Str("Handler", string(ctx.Request.Header.RequestURI())).Logger()
log := log.With().Strs("Handler", []string{string(ctx.Request.Host()), string(ctx.Request.Header.RequestURI())}).Logger()
ctx.Response.Header.Set("Server", "CodebergPages/"+version.Version)

4
server/setup.go
View file

@ -4,11 +4,11 @@ package server
import (
"bytes"
"fmt"
"net/http"
"time"
"github.com/rs/zerolog/log"
"github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/server/cache"
@ -18,7 +18,7 @@ import (
type fasthttpLogger struct{}
func (fasthttpLogger) Printf(format string, args ...interface{}) {
log.Printf("[FASTHTTP] "+format, args...)
log.Printf("FastHTTP: %s", fmt.Sprintf(format, args...))
}
func SetupServer(handler fasthttp.RequestHandler) *fasthttp.Server {

1
server/try.go
View file

@ -43,6 +43,7 @@ func tryUpstream(ctx *context.Context, giteaClient *gitea.Client,
targetOptions.TargetRepo = targetRepo
targetOptions.TargetBranch = targetBranch
targetOptions.TargetPath = targetPath
targetOptions.Host = string(trimmedHost)
// Try to request the file from the Gitea API
if !targetOptions.Upstream(ctx, giteaClient) {