Add redis for caching, first try during a train ride so expect it to not be working yet

server/cache/interface.go

@@ -4,7 +4,7 @@ import "time"
 
 // ICache is an interface that defines how the pages server interacts with the cache.
 type ICache interface {
-	Set(key string, value interface{}, ttl time.Duration) error
-	Get(key string) (interface{}, bool)
+	Set(key string, value string, ttl time.Duration) error
+	Get(key string) (string, bool)
 	Remove(key string)
 }

server/cache/memory.go

@@ -1,7 +1,31 @@
 package cache
 
-import "github.com/OrlovEvgeny/go-mcache"
+import (
+	"github.com/OrlovEvgeny/go-mcache"
+	"time"
+)
+
+type MCache struct {
+	mcache *mcache.CacheDriver
+}
+
+func (m *MCache) Set(key string, value string, ttl time.Duration) error {
+	return m.mcache.Set(key, value, ttl)
+}
+
+func (m *MCache) Get(key string) (string, bool) {
+	val, ok := m.mcache.Get(key)
+	if ok {
+		return val.(string), true
+	} else {
+		return "", false
+	}
+}
+
+func (m *MCache) Remove(key string) {
+	m.mcache.Remove(key)
+}
 
 func NewInMemoryCache() ICache {
-	return mcache.New()
+	return &MCache{mcache.New()}
 }

server/cache/redis.go

@@ -0,0 +1,47 @@
+package cache
+
+import (
+	"context"
+	"github.com/redis/go-redis/v9"
+	"github.com/rs/zerolog/log"
+	"time"
+)
+
+type RedisCache struct {
+	ctx context.Context
+	rdb *redis.Client
+}
+
+func (r *RedisCache) Set(key string, value string, ttl time.Duration) error {
+	return r.rdb.Set(r.ctx, key, value, ttl).Err()
+}
+
+func (r *RedisCache) Get(key string) (string, bool) {
+	val, err := r.rdb.Get(r.ctx, key).Result()
+	if err != nil {
+		if err == redis.Nil {
+			log.Error().Err(err).Str("key", key).Msg("Couldn't request key from cache.")
+		}
+		return "", false
+	} else {
+		return val, true
+	}
+}
+
+func (r *RedisCache) Remove(key string) {
+	err := r.rdb.Del(r.ctx, key).Err()
+	if err == nil {
+		log.Error().Err(err).Str("key", key).Msg("Couldn't delete key from cache.")
+	}
+}
+
+func NewRedisCache() ICache {
+	return &RedisCache{
+		context.Background(),
+		redis.NewClient(&redis.Options{
+			Addr:     "localhost:6379",
+			Password: "", // no password set
+			DB:       0,  // use default DB
+		}),
+	}
+}

server/certificates/cached_challengers.go

@@ -60,12 +60,12 @@ func SetupHTTPACMEChallengeServer(challengeCache cache.ICache, sslPort uint) htt
 		// it's an acme request
 		if strings.HasPrefix(ctx.Path(), challengePath) {
 			challenge, ok := challengeCache.Get(domain + "/" + strings.TrimPrefix(ctx.Path(), challengePath))
-			if !ok || challenge == nil {
+			if !ok || challenge == "" {
 				log.Info().Msgf("HTTP-ACME challenge for '%s' failed: token not found", domain)
 				ctx.String("no challenge for this token", http.StatusNotFound)
 			}
 			log.Info().Msgf("HTTP-ACME challenge for '%s' succeeded", domain)
-			ctx.String(challenge.(string))
+			ctx.String(challenge)
 			return
 		}
 

server/certificates/certificates.go

@@ -6,6 +6,7 @@ import (
 	"crypto/x509"
 	"errors"
 	"fmt"
+	"github.com/OrlovEvgeny/go-mcache"
 	"strconv"
 	"strings"
 	"time"
@@ -31,7 +32,7 @@ func TLSConfig(mainDomainSuffix string,
 	giteaClient *gitea.Client,
 	acmeClient *AcmeClient,
 	firstDefaultBranch string,
-	keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.ICache,
+	keyCache *mcache.CacheDriver, challengeCache cache.ICache, dnsLookupCache *mcache.CacheDriver, canonicalDomainCache cache.ICache,
 	certDB database.CertDB,
 	noDNS01 bool,
 	rawDomain string,
@@ -56,7 +57,7 @@ func TLSConfig(mainDomainSuffix string,
 					if !ok {
 						return nil, errors.New("no challenge for this domain")
 					}
-					cert, err := tlsalpn01.ChallengeCert(domain, challenge.(string))
+					cert, err := tlsalpn01.ChallengeCert(domain, challenge)
 					if err != nil {
 						return nil, err
 					}

server/dns/dns.go

@@ -1,11 +1,10 @@
 package dns
 
 import (
+	"github.com/OrlovEvgeny/go-mcache"
 	"net"
 	"strings"
 	"time"
-
-	"codeberg.org/codeberg/pages/server/cache"
 )
 
 // lookupCacheTimeout specifies the timeout for the DNS lookup cache.
@@ -15,7 +14,7 @@ var defaultPagesRepo = "pages"
 
 // GetTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix.
 // If everything is fine, it returns the target data.
-func GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch string, dnsLookupCache cache.ICache) (targetOwner, targetRepo, targetBranch string) {
+func GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch string, dnsLookupCache *mcache.CacheDriver) (targetOwner, targetRepo, targetBranch string) {
 	// Get CNAME or TXT
 	var cname string
 	var err error

server/gitea/client.go

@@ -115,8 +115,17 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
 	log := log.With().Str("cache_key", cacheKey).Logger()
 	log.Trace().Msg("try file in cache")
 	// handle if cache entry exist
-	if cache, ok := client.responseCache.Get(cacheKey); ok {
-		cache := cache.(FileResponse)
+	if cacheMetadata, ok := client.responseCache.Get(cacheKey + "|Metadata"); ok {
+		cacheMetadataParts := strings.Split(cacheMetadata, "\n")
+		cache := FileResponse{
+			Exists:    cacheMetadataParts[0] == "true",
+			IsSymlink: cacheMetadataParts[1] == "true",
+			ETag:      cacheMetadataParts[2],
+		}
+		cacheBodyString, _ := client.responseCache.Get(cacheKey + "|Body")
+		cache.Body = []byte(cacheBodyString)
+		// TODO: don't grab the content from the cache if the ETag matches?!
+
 		cachedHeader, cachedStatusCode := cache.createHttpResponse(cacheKey)
 		// TODO: check against some timestamp mismatch?!?
 		if cache.Exists {
@@ -130,6 +139,7 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
 				return io.NopCloser(bytes.NewReader(cache.Body)), cachedHeader, cachedStatusCode, nil
 			} else if cache.IsEmpty() {
 				log.Debug().Msg("[cache] is empty")
+				// TODO: empty files aren't cached anyways; but when closing the issue please make sure that a missing body cache key is also handled correctly.
 			}
 		}
 	}
@@ -164,7 +174,12 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
 						ETag:      resp.Header.Get(ETagHeader),
 					}
 					log.Trace().Msgf("file response has %d bytes", len(fileResponse.Body))
-					if err := client.responseCache.Set(cacheKey, fileResponse, fileCacheTimeout); err != nil {
+					metadataStr := strconv.FormatBool(fileResponse.Exists) + "\n" + strconv.FormatBool(fileResponse.IsSymlink) + "\n" + fileResponse.ETag
+					if err := client.responseCache.Set(cacheKey+"|Metadata", metadataStr, fileCacheTimeout); err != nil {
+						log.Error().Err(err).Msg("[cache] error on cache write")
+					}
+					// TODO: Test with binary files, as we convert []byte to string! Using []byte values might makes more sense anyways.
+					if err := client.responseCache.Set(cacheKey+"|Body", string(fileResponse.Body), fileCacheTimeout); err != nil {
 						log.Error().Err(err).Msg("[cache] error on cache write")
 					}
 
@@ -187,13 +202,11 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
 				ETag:     resp.Header.Get(ETagHeader),
 				MimeType: mimeType,
 			}
+			// TODO: dafuq...
 			return fileResp.CreateCacheReader(reader, client.responseCache, cacheKey), resp.Response.Header, resp.StatusCode, nil
 
 		case http.StatusNotFound:
-			if err := client.responseCache.Set(cacheKey, FileResponse{
-				Exists: false,
-				ETag:   resp.Header.Get(ETagHeader),
-			}, fileCacheTimeout); err != nil {
+			if err := client.responseCache.Set(cacheKey+"|Metadata", "false\nfalse\n"+resp.Header.Get(ETagHeader), fileCacheTimeout); err != nil {
 				log.Error().Err(err).Msg("[cache] error on cache write")
 			}
 
@@ -208,21 +221,28 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
 func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchName string) (*BranchTimestamp, error) {
 	cacheKey := fmt.Sprintf("%s/%s/%s/%s", branchTimestampCacheKeyPrefix, repoOwner, repoName, branchName)
 
-	if stamp, ok := client.responseCache.Get(cacheKey); ok && stamp != nil {
-		branchTimeStamp := stamp.(*BranchTimestamp)
-		if branchTimeStamp.notFound {
+	if stamp, ok := client.responseCache.Get(cacheKey); ok {
+		if stamp == "" {
 			log.Trace().Msgf("[cache] use branch %q not found", branchName)
 			return &BranchTimestamp{}, ErrorNotFound
 		}
 		log.Trace().Msgf("[cache] use branch %q exist", branchName)
-		return branchTimeStamp, nil
+		// This comes from the refactoring of the caching library.
+		// The branch as reported by the API was stored in the cache, and I'm not sure if there are
+		// situations where it differs from the name in the request, hence this is left here.
+		stampParts := strings.SplitN(stamp, "", 2)
+		stampTime, _ := time.Parse(time.RFC3339, stampParts[0])
+		return &BranchTimestamp{
+			Branch:    stampParts[1],
+			Timestamp: stampTime,
+		}, nil
 	}
 
 	branch, resp, err := client.sdkClient.GetRepoBranch(repoOwner, repoName, branchName)
 	if err != nil {
 		if resp != nil && resp.StatusCode == http.StatusNotFound {
 			log.Trace().Msgf("[cache] set cache branch %q not found", branchName)
-			if err := client.responseCache.Set(cacheKey, &BranchTimestamp{Branch: branchName, notFound: true}, branchExistenceCacheTimeout); err != nil {
+			if err := client.responseCache.Set(cacheKey, "", branchExistenceCacheTimeout); err != nil {
 				log.Error().Err(err).Msg("[cache] error on cache write")
 			}
 			return &BranchTimestamp{}, ErrorNotFound
@@ -239,7 +259,7 @@ func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchNam
 	}
 
 	log.Trace().Msgf("set cache branch [%s] exist", branchName)
-	if err := client.responseCache.Set(cacheKey, stamp, branchExistenceCacheTimeout); err != nil {
+	if err := client.responseCache.Set(cacheKey, stamp.Timestamp.Format(time.RFC3339)+"|"+stamp.Branch, branchExistenceCacheTimeout); err != nil {
 		log.Error().Err(err).Msg("[cache] error on cache write")
 	}
 	return stamp, nil
@@ -248,8 +268,8 @@ func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchNam
 func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (string, error) {
 	cacheKey := fmt.Sprintf("%s/%s/%s", defaultBranchCacheKeyPrefix, repoOwner, repoName)
 
-	if branch, ok := client.responseCache.Get(cacheKey); ok && branch != nil {
-		return branch.(string), nil
+	if branch, ok := client.responseCache.Get(cacheKey); ok {
+		return branch, nil
 	}
 
 	repo, resp, err := client.sdkClient.GetRepo(repoOwner, repoName)

server/handler/handler_custom_domain.go

@@ -1,6 +1,7 @@
 package handler
 
 import (
+	"github.com/OrlovEvgeny/go-mcache"
 	"net/http"
 	"path"
 	"strings"
@@ -19,7 +20,7 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g
 	trimmedHost string,
 	pathElements []string,
 	firstDefaultBranch string,
-	dnsLookupCache, canonicalDomainCache, redirectsCache cache.ICache,
+	dnsLookupCache *mcache.CacheDriver, canonicalDomainCache, redirectsCache cache.ICache,
 ) {
 	// Serve pages from custom domains
 	targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, mainDomainSuffix, firstDefaultBranch, dnsLookupCache)

server/upstream/domains.go

@@ -20,7 +20,7 @@ const canonicalDomainConfig = ".domains"
 func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.ICache) (domain string, valid bool) {
 	// Check if this request is cached.
 	if cachedValue, ok := canonicalDomainCache.Get(o.TargetOwner + "/" + o.TargetRepo + "/" + o.TargetBranch); ok {
-		domains := cachedValue.([]string)
+		domains := strings.Split(cachedValue, "\n")
 		for _, domain := range domains {
 			if domain == actualDomain {
 				valid = true
@@ -33,6 +33,7 @@ func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain,
 	body, err := giteaClient.GiteaRawContent(o.TargetOwner, o.TargetRepo, o.TargetBranch, canonicalDomainConfig)
 	if err != nil && !errors.Is(err, gitea.ErrorNotFound) {
 		log.Error().Err(err).Msgf("could not read %s of %s/%s", canonicalDomainConfig, o.TargetOwner, o.TargetRepo)
+		// TODO: WTF we just continue?!
 	}
 
 	var domains []string
@@ -62,7 +63,7 @@ func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain,
 	}
 
 	// Add result to cache.
-	_ = canonicalDomainCache.Set(o.TargetOwner+"/"+o.TargetRepo+"/"+o.TargetBranch, domains, canonicalDomainCacheTimeout)
+	_ = canonicalDomainCache.Set(o.TargetOwner+"/"+o.TargetRepo+"/"+o.TargetBranch, strings.Join(domains, "\n"), canonicalDomainCacheTimeout)
 
 	// Return the first domain from the list and return if any of the domains
 	// matched the requested domain.

server/upstream/redirects.go

@@ -1,6 +1,7 @@
 package upstream
 
 import (
+	"encoding/json"
 	"strconv"
 	"strings"
 	"time"
@@ -29,7 +30,12 @@ func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.I
 
 	// Check for cached redirects
 	if cachedValue, ok := redirectsCache.Get(cacheKey); ok {
-		redirects = cachedValue.([]Redirect)
+		redirects := []Redirect{}
+		err := json.Unmarshal([]byte(cachedValue), redirects)
+		if err != nil {
+			log.Error().Err(err).Msgf("could not parse redirects for key %s", cacheKey)
+			// It's okay to continue, the array stays empty.
+		}
 	} else {
 		// Get _redirects file and parse
 		body, err := giteaClient.GiteaRawContent(o.TargetOwner, o.TargetRepo, o.TargetBranch, redirectsConfig)
@@ -58,7 +64,12 @@ func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.I
 				})
 			}
 		}
-		_ = redirectsCache.Set(cacheKey, redirects, redirectsCacheTimeout)
+		redirectsJson, err := json.Marshal(redirects)
+		if err != nil {
+			log.Error().Err(err).Msgf("could not store redirects for key %s", cacheKey)
+		} else {
+			_ = redirectsCache.Set(cacheKey, string(redirectsJson), redirectsCacheTimeout)
+		}
 	}
 	return redirects
 }