Move redis config to CacheConfig struct, add cache prefixes & trace logging

This commit is contained in:
Moritz Marquardt 2024-03-29 23:15:08 +01:00
parent e1a22d5f4c
commit 46c8daacba
7 changed files with 45 additions and 30 deletions

View file

@ -82,6 +82,7 @@ and especially have a look at [this section of the haproxy.cfg](https://codeberg
See <https://go-acme.github.io/lego/dns/> for available values & additional environment variables.
- `NO_DNS_01` (default: `false`): Disable the use of ACME DNS. This means that the wildcard certificate is self-signed and all domains and subdomains will have a distinct certificate. Because this may lead to a rate limit from the ACME provider, this option is not recommended for Gitea/Forgejo instances with open registrations or a great number of users/orgs.
- `LOG_LEVEL` (default: warn): Set this to specify the level of logging.
- `REDIS_URL` (default: use in-memory cache): Set this to use Redis as a cache server.
## Contributing to the development

View file

@ -6,7 +6,7 @@ type Config struct {
Gitea GiteaConfig
Database DatabaseConfig
ACME ACMEConfig
RedisURL string `default:""`
Cache CacheConfig
}
type ServerConfig struct {
@ -46,3 +46,7 @@ type ACMEConfig struct {
NoDNS01 bool `default:"false"`
AccountConfigFile string `default:"acme-account.json"`
}
type CacheConfig struct {
RedisURL string `default:""`
}

View file

@ -54,9 +54,7 @@ func MergeConfig(ctx *cli.Context, config *Config) {
mergeGiteaConfig(ctx, &config.Gitea)
mergeDatabaseConfig(ctx, &config.Database)
mergeACMEConfig(ctx, &config.ACME)
if ctx.IsSet("redis-url") {
config.RedisURL = ctx.String("redis-url")
}
mergeCacheConfig(ctx, &config.Cache)
}
func mergeServerConfig(ctx *cli.Context, config *ServerConfig) {
@ -151,3 +149,9 @@ func mergeACMEConfig(ctx *cli.Context, config *ACMEConfig) {
config.AccountConfigFile = ctx.String("acme-account-config")
}
}
func mergeCacheConfig(ctx *cli.Context, config *CacheConfig) {
if ctx.IsSet("redis-url") {
config.RedisURL = ctx.String("redis-url")
}
}

26
server/cache/redis.go vendored
View file

@ -9,35 +9,41 @@ import (
)
type RedisCache struct {
ctx context.Context
rdb *redis.Client
name string
ctx context.Context
rdb *redis.Client
}
func (r *RedisCache) Set(key string, value []byte, ttl time.Duration) error {
return r.rdb.Set(r.ctx, key, value, ttl).Err()
log.Trace().Str("key", r.name+"::"+key).Int("len(value)", len(value)).Bytes("value", value).Msg("Set in Redis.")
return r.rdb.Set(r.ctx, r.name+"::"+key, value, ttl).Err()
}
func (r *RedisCache) Get(key string) ([]byte, bool) {
val, err := r.rdb.Get(r.ctx, key).Bytes()
val, err := r.rdb.Get(r.ctx, r.name+"::"+key).Bytes()
if err != nil {
if errors.Is(err, redis.Nil) {
log.Error().Err(err).Str("key", key).Msg("Couldn't request key from cache.")
if !errors.Is(err, redis.Nil) {
log.Error().Err(err).Str("key", r.name+"::"+key).Msg("Couldn't request key from cache.")
} else {
log.Trace().Str("key", r.name+"::"+key).Msg("Get from Redis, doesn't exist.")
}
return nil, false
} else {
log.Trace().Str("key", r.name+"::"+key).Int("len(value)", len(val)).Msg("Get from Redis.")
return val, true
}
}
func (r *RedisCache) Remove(key string) {
err := r.rdb.Del(r.ctx, key).Err()
if err == nil {
log.Error().Err(err).Str("key", key).Msg("Couldn't delete key from cache.")
err := r.rdb.Del(r.ctx, r.name+"::"+key).Err()
if err != nil {
log.Error().Err(err).Str("key", r.name+"::"+key).Msg("Couldn't delete key from cache.")
}
}
func NewRedisCache(opts *redis.Options) ICache {
func NewRedisCache(name string, opts *redis.Options) ICache {
return &RedisCache{
name,
context.Background(),
redis.NewClient(opts),
}

View file

@ -117,8 +117,7 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
// handle if cache entry exist
if cacheMetadata, ok := client.responseCache.Get(cacheKey + "|Metadata"); ok {
cache := FileResponseFromMetadataString(string(cacheMetadata))
cacheBodyString, _ := client.responseCache.Get(cacheKey + "|Body")
cache.Body = []byte(cacheBodyString)
cache.Body, _ = client.responseCache.Get(cacheKey + "|Body")
// TODO: don't grab the content from the cache if the ETag matches?!
cachedHeader, cachedStatusCode := cache.createHttpResponse(cacheKey)
@ -136,8 +135,9 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
log.Debug().Msg("[cache] is empty")
// TODO: empty files aren't cached anyways; but when closing the issue please make sure that a missing body cache key is also handled correctly.
}
}
} // TODO: handle missing pages if they redirect to a index.html
}
// TODO: metadata not written, is close ever called?
log.Trace().Msg("file not in cache")
// not in cache, open reader via gitea api
reader, resp, err := client.sdkClient.GetFileReader(targetOwner, targetRepo, ref, resource, client.supportLFS)
@ -284,12 +284,12 @@ func (client *Client) GiteaCheckIfOwnerExists(owner string) (bool, error) {
cacheKey := fmt.Sprintf("%s/%s", ownerExistenceKeyPrefix, owner)
if exist, ok := client.responseCache.Get(cacheKey); ok && exist != nil {
return exist.(bool), nil
return string(exist) == "true", nil
}
_, resp, err := client.sdkClient.GetUserInfo(owner)
if resp.StatusCode == http.StatusOK && err == nil {
if err := client.responseCache.Set(cacheKey, true, ownerExistenceCacheTimeout); err != nil {
if err := client.responseCache.Set(cacheKey, []byte("true"), ownerExistenceCacheTimeout); err != nil {
log.Error().Err(err).Msg("[cache] error on cache write")
}
return true, nil
@ -299,14 +299,14 @@ func (client *Client) GiteaCheckIfOwnerExists(owner string) (bool, error) {
_, resp, err = client.sdkClient.GetOrg(owner)
if resp.StatusCode == http.StatusOK && err == nil {
if err := client.responseCache.Set(cacheKey, true, ownerExistenceCacheTimeout); err != nil {
if err := client.responseCache.Set(cacheKey, []byte("true"), ownerExistenceCacheTimeout); err != nil {
log.Error().Err(err).Msg("[cache] error on cache write")
}
return true, nil
} else if resp.StatusCode != http.StatusNotFound {
return false, err
}
if err := client.responseCache.Set(cacheKey, false, ownerExistenceCacheTimeout); err != nil {
if err := client.responseCache.Set(cacheKey, []byte("false"), ownerExistenceCacheTimeout); err != nil {
log.Error().Err(err).Msg("[cache] error on cache write")
}
return false, nil

View file

@ -78,24 +78,24 @@ func Serve(ctx *cli.Context) error {
dnsLookupCache := mcache.New()
var redisErr error = nil
createCache := func() cache.ICache {
if cfg.RedisURL != "" {
opts, err := redis.ParseURL(cfg.RedisURL)
createCache := func(name string) cache.ICache {
if cfg.Cache.RedisURL != "" {
opts, err := redis.ParseURL(cfg.Cache.RedisURL)
if err != nil {
redisErr = err
}
return cache.NewRedisCache(opts)
return cache.NewRedisCache(name, opts)
}
return cache.NewInMemoryCache()
}
// challengeCache stores the certificate challenges
challengeCache := createCache()
challengeCache := createCache("challenge")
// canonicalDomainCache stores canonical domains
canonicalDomainCache := createCache()
canonicalDomainCache := createCache("canonicalDomain")
// redirectsCache stores redirects in _redirects files
redirectsCache := createCache()
redirectsCache := createCache("redirects")
// clientResponseCache stores responses from the Gitea server
clientResponseCache := createCache()
clientResponseCache := createCache("clientResponse")
if redisErr != nil {
return redisErr
}

View file

@ -31,7 +31,7 @@ func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.I
// Check for cached redirects
if cachedValue, ok := redirectsCache.Get(cacheKey); ok {
redirects := []Redirect{}
err := json.Unmarshal(cachedValue, redirects)
err := json.Unmarshal(cachedValue, &redirects)
if err != nil {
log.Error().Err(err).Msgf("could not parse redirects for key %s", cacheKey)
// It's okay to continue, the array stays empty.