diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go
index 7c9a035..6e41ee4 100644
--- a/server/upstream/upstream.go
+++ b/server/upstream/upstream.go
@@ -196,7 +196,7 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
 	}
 	log.Debug().Msg("response")
 
-	if res != nil && ctx.Err() == nil {
+	if res != nil && res.Header.ContentLength() > fileCacheSizeLimit && ctx.Err() == nil {
 		cachedResponse.Exists = true
 		cachedResponse.MimeType = mimeType
 		cachedResponse.Body = cacheBodyWriter.Bytes()