use /proc/self/statm to get current usage

This commit is contained in:
crapStone 2024-03-15 20:35:39 +01:00
parent a9b6615286
commit bb41a4abfa
3 changed files with 35 additions and 13 deletions

View file

@ -140,8 +140,8 @@ var (
}, },
&cli.Uint64Flag{ &cli.Uint64Flag{
Name: "memory-limit", Name: "memory-limit",
Usage: "maximum size of memory in bytes to use for caching, default: 512MB", Usage: "maximum size of memory in KiB to use for caching, default: 512MiB",
Value: 512 * 1024 * 1024, Value: 512 * 1024,
EnvVars: []string{"MAX_MEMORY_SIZE"}, EnvVars: []string{"MAX_MEMORY_SIZE"},
}, },

View file

@ -1,7 +1,9 @@
package cache package cache
import ( import (
"runtime" "os"
"strconv"
"strings"
"time" "time"
"github.com/OrlovEvgeny/go-mcache" "github.com/OrlovEvgeny/go-mcache"
@ -10,7 +12,7 @@ import (
type Cache struct { type Cache struct {
mcache *mcache.CacheDriver mcache *mcache.CacheDriver
memoryLimit uint64 memoryLimit int
lastCheck time.Time lastCheck time.Time
} }
@ -21,7 +23,7 @@ func NewInMemoryCache() ICache {
// NewInMemoryCache returns a new mcache with a memory limit. // NewInMemoryCache returns a new mcache with a memory limit.
// If the limit is exceeded, the cache will be cleared. // If the limit is exceeded, the cache will be cleared.
func NewInMemoryCacheWithLimit(memoryLimit uint64) ICache { func NewInMemoryCacheWithLimit(memoryLimit int) ICache {
return &Cache{ return &Cache{
mcache: mcache.New(), mcache: mcache.New(),
memoryLimit: memoryLimit, memoryLimit: memoryLimit,
@ -31,11 +33,10 @@ func NewInMemoryCacheWithLimit(memoryLimit uint64) ICache {
func (c *Cache) Set(key string, value interface{}, ttl time.Duration) error { func (c *Cache) Set(key string, value interface{}, ttl time.Duration) error {
now := time.Now() now := time.Now()
// checking memory limit is a "stop the world" operation // we don't want to do it too often
// so we don't want to do it too often
if now.Sub(c.lastCheck) > (time.Second * 3) { if now.Sub(c.lastCheck) > (time.Second * 3) {
if c.memoryLimitOvershot() { if c.memoryLimitOvershot() {
log.Debug().Msg("memory limit exceeded, clearing cache") log.Info().Msg("[cache] memory limit exceeded, clearing cache")
c.mcache.Truncate() c.mcache.Truncate()
} }
c.lastCheck = now c.lastCheck = now
@ -53,10 +54,31 @@ func (c *Cache) Remove(key string) {
} }
func (c *Cache) memoryLimitOvershot() bool { func (c *Cache) memoryLimitOvershot() bool {
var stats runtime.MemStats mem := getCurrentMemory()
runtime.ReadMemStats(&stats)
log.Debug().Uint64("bytes", stats.HeapAlloc).Msg("current memory usage") log.Debug().Int("kB", mem).Msg("[cache] current memory usage")
return stats.HeapAlloc > c.memoryLimit return mem > c.memoryLimit
}
// getCurrentMemory returns the current memory in KB
func getCurrentMemory() int {
b, err := os.ReadFile("/proc/self/statm")
if err != nil {
log.Error().Err(err).Msg("[cache] could not read /proc/self/statm")
return 0
}
str := string(b)
arr := strings.Split(str, " ")
// convert to pages
res, err := strconv.Atoi(arr[1])
if err != nil {
log.Error().Err(err).Msg("[cache] could not convert string to int")
return 0
}
// convert to KB
return (res * os.Getpagesize()) / 1024
} }

View file

@ -79,7 +79,7 @@ func Serve(ctx *cli.Context) error {
// redirectsCache stores redirects in _redirects files // redirectsCache stores redirects in _redirects files
redirectsCache := cache.NewInMemoryCache() redirectsCache := cache.NewInMemoryCache()
// clientResponseCache stores responses from the Gitea server // clientResponseCache stores responses from the Gitea server
clientResponseCache := cache.NewInMemoryCacheWithLimit(ctx.Uint64("memory-limit")) clientResponseCache := cache.NewInMemoryCacheWithLimit(ctx.Int("memory-limit"))
giteaClient, err := gitea.NewClient(cfg.Gitea, clientResponseCache) giteaClient, err := gitea.NewClient(cfg.Gitea, clientResponseCache)
if err != nil { if err != nil {