mirror of
https://codeberg.org/Codeberg/pages-server.git
synced 2025-04-19 11:36:57 +00:00
cache pages, taken from https://codeberg.org/Codeberg/pages-server/pulls/301
Co-authored-by: Moritz Marquardt <git@momar.de>
This commit is contained in:
crapStone
parent
23a8e83e80
commit
81bdbaf261
2 changed files with 102 additions and 110 deletions
|
|
@ -2,6 +2,7 @@ package gitea
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
|
@ -39,10 +40,9 @@ const (
|
|||
objTypeSymlink = "symlink"
|
||||
|
||||
// std
|
||||
ETagHeader = "ETag"
|
||||
ContentTypeHeader = "Content-Type"
|
||||
ContentLengthHeader = "Content-Length"
|
||||
ContentEncodingHeader = "Content-Encoding"
|
||||
ETagHeader = "ETag"
|
||||
ContentTypeHeader = "Content-Type"
|
||||
ContentLengthHeader = "Content-Length"
|
||||
)
|
||||
|
||||
type Client struct {
|
||||
|
|
@ -104,7 +104,7 @@ func (client *Client) ContentWebLink(targetOwner, targetRepo, branch, resource s
|
|||
}
|
||||
|
||||
func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource string) ([]byte, error) {
|
||||
reader, _, _, err := client.ServeRawContent(targetOwner, targetRepo, ref, resource, false)
|
||||
reader, _, _, err := client.ServeRawContent(targetOwner, targetRepo, ref, resource)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -112,29 +112,42 @@ func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource str
|
|||
return io.ReadAll(reader)
|
||||
}
|
||||
|
||||
func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource string, decompress bool) (io.ReadCloser, http.Header, int, error) {
|
||||
func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource string) (io.ReadCloser, http.Header, int, error) {
|
||||
cacheKey := fmt.Sprintf("%s/%s/%s|%s|%s", rawContentCacheKeyPrefix, targetOwner, targetRepo, ref, resource)
|
||||
log := log.With().Str("cache_key", cacheKey).Logger()
|
||||
log.Trace().Msg("try file in cache")
|
||||
// handle if cache entry exist
|
||||
if cache, ok := client.responseCache.Get(cacheKey); ok {
|
||||
cache := cache.(FileResponse)
|
||||
cachedHeader, cachedStatusCode := cache.createHttpResponse(cacheKey, decompress)
|
||||
if cacheMetadata, ok := client.responseCache.Get(cacheKey + "|Metadata"); ok {
|
||||
var cache FileResponse
|
||||
err := json.Unmarshal(cacheMetadata.([]byte), &cache)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msgf("[cache] failed to unmarshal metadata for: %s", cacheKey)
|
||||
return nil, nil, http.StatusInternalServerError, err
|
||||
}
|
||||
body, ok := client.responseCache.Get(cacheKey + "|Body")
|
||||
if !ok {
|
||||
log.Error().Msgf("[cache] failed to get body for: %s", cacheKey)
|
||||
return nil, nil, http.StatusInternalServerError, ErrorNotFound
|
||||
}
|
||||
cache.Body = body.([]byte)
|
||||
// TODO: don't grab the content from the cache if the ETag matches?!
|
||||
|
||||
cachedHeader, cachedStatusCode := cache.createHttpResponse(cacheKey)
|
||||
// TODO: check against some timestamp mismatch?!?
|
||||
if cache.Exists {
|
||||
log.Debug().Msg("[cache] exists")
|
||||
if cache.IsSymlink {
|
||||
linkDest := string(cache.Body)
|
||||
log.Debug().Msgf("[cache] follow symlink from %q to %q", resource, linkDest)
|
||||
return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest, decompress)
|
||||
} else if !cache.IsEmpty() {
|
||||
return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest)
|
||||
} else {
|
||||
log.Debug().Msgf("[cache] return %d bytes", len(cache.Body))
|
||||
return io.NopCloser(bytes.NewReader(cache.Body)), cachedHeader, cachedStatusCode, nil
|
||||
} else if cache.IsEmpty() {
|
||||
log.Debug().Msg("[cache] is empty")
|
||||
}
|
||||
} else {
|
||||
return nil, nil, http.StatusNotFound, ErrorNotFound
|
||||
}
|
||||
}
|
||||
// TODO: metadata not written, is close ever called?
|
||||
log.Trace().Msg("file not in cache")
|
||||
// not in cache, open reader via gitea api
|
||||
reader, resp, err := client.sdkClient.GetFileReader(targetOwner, targetRepo, ref, resource, client.supportLFS)
|
||||
|
|
@ -166,41 +179,40 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
|
|||
ETag: resp.Header.Get(ETagHeader),
|
||||
}
|
||||
log.Trace().Msgf("file response has %d bytes", len(fileResponse.Body))
|
||||
if err := client.responseCache.Set(cacheKey, fileResponse, fileCacheTimeout); err != nil {
|
||||
jsonToCache, err := json.Marshal(fileResponse)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msgf("[cache] marshaling json metadata for %q has returned an error", cacheKey)
|
||||
}
|
||||
if err := client.responseCache.Set(cacheKey+"|Metadata", jsonToCache, fileCacheTimeout); err != nil {
|
||||
log.Error().Err(err).Msg("[cache] error on cache write")
|
||||
}
|
||||
if err := client.responseCache.Set(cacheKey+"|Body", fileResponse.Body, fileCacheTimeout); err != nil {
|
||||
log.Error().Err(err).Msg("[cache] error on cache write")
|
||||
}
|
||||
|
||||
log.Debug().Msgf("follow symlink from %q to %q", resource, linkDest)
|
||||
return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest, decompress)
|
||||
return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest)
|
||||
}
|
||||
}
|
||||
|
||||
// now we are sure it's content so set the MIME type
|
||||
mimeType, rawType := client.getMimeTypeByExtension(resource)
|
||||
if decompress {
|
||||
resp.Response.Header.Set(ContentTypeHeader, mimeType)
|
||||
} else {
|
||||
resp.Response.Header.Set(ContentTypeHeader, rawType)
|
||||
}
|
||||
|
||||
if !shouldRespBeSavedToCache(resp.Response) {
|
||||
return reader, resp.Response.Header, resp.StatusCode, err
|
||||
}
|
||||
mimeType := client.getMimeTypeByExtension(resource)
|
||||
resp.Response.Header.Set(ContentTypeHeader, mimeType)
|
||||
|
||||
// now we write to cache and respond at the same time
|
||||
fileResp := FileResponse{
|
||||
Exists: true,
|
||||
ETag: resp.Header.Get(ETagHeader),
|
||||
MimeType: mimeType,
|
||||
RawMime: rawType,
|
||||
}
|
||||
return fileResp.CreateCacheReader(reader, client.responseCache, cacheKey), resp.Response.Header, resp.StatusCode, nil
|
||||
|
||||
case http.StatusNotFound:
|
||||
if err := client.responseCache.Set(cacheKey, FileResponse{
|
||||
Exists: false,
|
||||
ETag: resp.Header.Get(ETagHeader),
|
||||
}, fileCacheTimeout); err != nil {
|
||||
jsonToCache, err := json.Marshal(FileResponse{ETag: resp.Header.Get(ETagHeader)})
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msgf("[cache] marshaling json metadata for %q has returned an error", cacheKey)
|
||||
}
|
||||
if err := client.responseCache.Set(cacheKey+"|Metadata", jsonToCache, fileCacheTimeout); err != nil {
|
||||
log.Error().Err(err).Msg("[cache] error on cache write")
|
||||
}
|
||||
|
||||
|
|
@ -215,21 +227,25 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
|
|||
func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchName string) (*BranchTimestamp, error) {
|
||||
cacheKey := fmt.Sprintf("%s/%s/%s/%s", branchTimestampCacheKeyPrefix, repoOwner, repoName, branchName)
|
||||
|
||||
if stamp, ok := client.responseCache.Get(cacheKey); ok && stamp != nil {
|
||||
branchTimeStamp := stamp.(*BranchTimestamp)
|
||||
if branchTimeStamp.notFound {
|
||||
log.Trace().Msgf("[cache] use branch %q not found", branchName)
|
||||
if stampRaw, ok := client.responseCache.Get(cacheKey); ok {
|
||||
var stamp BranchTimestamp
|
||||
err := json.Unmarshal(stampRaw.([]byte), &stamp)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msgf("[cache] failed to unmarshal metadata for: %s", cacheKey)
|
||||
return &BranchTimestamp{}, ErrorNotFound
|
||||
}
|
||||
log.Trace().Msgf("[cache] use branch %q exist", branchName)
|
||||
return branchTimeStamp, nil
|
||||
// This comes from the refactoring of the caching library.
|
||||
// The branch as reported by the API was stored in the cache, and I'm not sure if there are
|
||||
// situations where it differs from the name in the request, hence this is left here.
|
||||
return &stamp, nil
|
||||
}
|
||||
|
||||
branch, resp, err := client.sdkClient.GetRepoBranch(repoOwner, repoName, branchName)
|
||||
if err != nil {
|
||||
if resp != nil && resp.StatusCode == http.StatusNotFound {
|
||||
log.Trace().Msgf("[cache] set cache branch %q not found", branchName)
|
||||
if err := client.responseCache.Set(cacheKey, &BranchTimestamp{Branch: branchName, notFound: true}, branchExistenceCacheTimeout); err != nil {
|
||||
if err := client.responseCache.Set(cacheKey, []byte{}, branchExistenceCacheTimeout); err != nil {
|
||||
log.Error().Err(err).Msg("[cache] error on cache write")
|
||||
}
|
||||
return &BranchTimestamp{}, ErrorNotFound
|
||||
|
|
@ -246,7 +262,11 @@ func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchNam
|
|||
}
|
||||
|
||||
log.Trace().Msgf("set cache branch [%s] exist", branchName)
|
||||
if err := client.responseCache.Set(cacheKey, stamp, branchExistenceCacheTimeout); err != nil {
|
||||
jsonToCache, err := json.Marshal(stamp)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msgf("[cache] marshaling json timestamp for %q has returned an error", cacheKey)
|
||||
}
|
||||
if err := client.responseCache.Set(cacheKey, jsonToCache, branchExistenceCacheTimeout); err != nil {
|
||||
log.Error().Err(err).Msg("[cache] error on cache write")
|
||||
}
|
||||
return stamp, nil
|
||||
|
|
@ -255,7 +275,7 @@ func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchNam
|
|||
func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (string, error) {
|
||||
cacheKey := fmt.Sprintf("%s/%s/%s", defaultBranchCacheKeyPrefix, repoOwner, repoName)
|
||||
|
||||
if branch, ok := client.responseCache.Get(cacheKey); ok && branch != nil {
|
||||
if branch, ok := client.responseCache.Get(cacheKey); ok {
|
||||
return branch.(string), nil
|
||||
}
|
||||
|
||||
|
|
@ -268,7 +288,7 @@ func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (str
|
|||
}
|
||||
|
||||
branch := repo.DefaultBranch
|
||||
if err := client.responseCache.Set(cacheKey, branch, defaultBranchCacheTimeout); err != nil {
|
||||
if err := client.responseCache.Set(cacheKey, []byte(branch), defaultBranchCacheTimeout); err != nil {
|
||||
log.Error().Err(err).Msg("[cache] error on cache write")
|
||||
}
|
||||
return branch, nil
|
||||
|
|
@ -277,13 +297,14 @@ func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (str
|
|||
func (client *Client) GiteaCheckIfOwnerExists(owner string) (bool, error) {
|
||||
cacheKey := fmt.Sprintf("%s/%s", ownerExistenceKeyPrefix, owner)
|
||||
|
||||
if exist, ok := client.responseCache.Get(cacheKey); ok && exist != nil {
|
||||
return exist.(bool), nil
|
||||
if existRaw, ok := client.responseCache.Get(cacheKey); ok && existRaw != nil {
|
||||
exist, err := strconv.ParseBool(existRaw.(string))
|
||||
return exist, err
|
||||
}
|
||||
|
||||
_, resp, err := client.sdkClient.GetUserInfo(owner)
|
||||
if resp.StatusCode == http.StatusOK && err == nil {
|
||||
if err := client.responseCache.Set(cacheKey, true, ownerExistenceCacheTimeout); err != nil {
|
||||
if err := client.responseCache.Set(cacheKey, []byte("true"), ownerExistenceCacheTimeout); err != nil {
|
||||
log.Error().Err(err).Msg("[cache] error on cache write")
|
||||
}
|
||||
return true, nil
|
||||
|
|
@ -293,59 +314,26 @@ func (client *Client) GiteaCheckIfOwnerExists(owner string) (bool, error) {
|
|||
|
||||
_, resp, err = client.sdkClient.GetOrg(owner)
|
||||
if resp.StatusCode == http.StatusOK && err == nil {
|
||||
if err := client.responseCache.Set(cacheKey, true, ownerExistenceCacheTimeout); err != nil {
|
||||
if err := client.responseCache.Set(cacheKey, []byte("true"), ownerExistenceCacheTimeout); err != nil {
|
||||
log.Error().Err(err).Msg("[cache] error on cache write")
|
||||
}
|
||||
return true, nil
|
||||
} else if resp.StatusCode != http.StatusNotFound {
|
||||
return false, err
|
||||
}
|
||||
if err := client.responseCache.Set(cacheKey, false, ownerExistenceCacheTimeout); err != nil {
|
||||
if err := client.responseCache.Set(cacheKey, []byte("false"), ownerExistenceCacheTimeout); err != nil {
|
||||
log.Error().Err(err).Msg("[cache] error on cache write")
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (client *Client) extToMime(ext string) string {
|
||||
mimeType := mime.TypeByExtension(ext)
|
||||
func (client *Client) getMimeTypeByExtension(resource string) string {
|
||||
mimeType := mime.TypeByExtension(path.Ext(resource))
|
||||
mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
|
||||
if client.forbiddenMimeTypes[mimeTypeSplit[0]] || mimeType == "" {
|
||||
mimeType = client.defaultMimeType
|
||||
}
|
||||
log.Trace().Msgf("probe mime of %q is %q", resource, mimeType)
|
||||
|
||||
return mimeType
|
||||
}
|
||||
|
||||
func (client *Client) getMimeTypeByExtension(resource string) (mimeType, rawType string) {
|
||||
rawExt := path.Ext(resource)
|
||||
innerExt := rawExt
|
||||
switch rawExt {
|
||||
case ".gz", ".br", ".zst":
|
||||
innerExt = path.Ext(resource[:len(resource)-len(rawExt)])
|
||||
}
|
||||
rawType = client.extToMime(rawExt)
|
||||
mimeType = rawType
|
||||
if innerExt != rawExt {
|
||||
mimeType = client.extToMime(innerExt)
|
||||
}
|
||||
log.Trace().Msgf("probe mime of %q is (%q / raw %q)", resource, mimeType, rawType)
|
||||
return mimeType, rawType
|
||||
}
|
||||
|
||||
func shouldRespBeSavedToCache(resp *http.Response) bool {
|
||||
if resp == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
contentLengthRaw := resp.Header.Get(ContentLengthHeader)
|
||||
if contentLengthRaw == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
contentLength, err := strconv.ParseInt(contentLengthRaw, 10, 64)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("could not parse content length")
|
||||
}
|
||||
|
||||
// if content to big or could not be determined we not cache it
|
||||
return contentLength > 0 && contentLength < fileCacheSizeLimit
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue