Merge branch 'std-http' into refactor_split-long-funcs

This commit is contained in:
6543 2022-11-12 01:08:24 +01:00
commit 3d9ffcf8d7
No known key found for this signature in database
GPG key ID: B8BE6D610E61C862
9 changed files with 73 additions and 72 deletions

View file

@ -15,6 +15,10 @@ pipeline:
- "[ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }" - "[ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }"
- golangci-lint run --timeout 5m --build-tags integration - golangci-lint run --timeout 5m --build-tags integration
editor-config:
group: compliant
image: mstruebing/editorconfig-checker
build: build:
group: compliant group: compliant
image: codeberg.org/6543/docker-images/golang_just image: codeberg.org/6543/docker-images/golang_just
@ -90,6 +94,7 @@ pipeline:
from_secret: bot_token from_secret: bot_token
when: when:
event: [ "push" ] event: [ "push" ]
branch: ${CI_REPO_DEFAULT_BRANCH}
docker-tag: docker-tag:
image: plugins/kaniko image: plugins/kaniko

View file

@ -1,7 +1,6 @@
package html package html
import ( import (
"io"
"net/http" "net/http"
"strconv" "strconv"
"strings" "strings"
@ -9,8 +8,8 @@ import (
"codeberg.org/codeberg/pages/server/context" "codeberg.org/codeberg/pages/server/context"
) )
// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced // ReturnErrorPage sets the response status code and writes NotFoundPage to the response body,
// with the provided status code. // with "%status%" and %message% replaced with the provided statusCode and msg
func ReturnErrorPage(ctx *context.Context, msg string, statusCode int) { func ReturnErrorPage(ctx *context.Context, msg string, statusCode int) {
ctx.RespWriter.Header().Set("Content-Type", "text/html; charset=utf-8") ctx.RespWriter.Header().Set("Content-Type", "text/html; charset=utf-8")
ctx.RespWriter.WriteHeader(statusCode) ctx.RespWriter.WriteHeader(statusCode)
@ -22,7 +21,7 @@ func ReturnErrorPage(ctx *context.Context, msg string, statusCode int) {
msg = strings.ReplaceAll(strings.ReplaceAll(ErrorPage, "%message%", msg), "%status%", http.StatusText(statusCode)) msg = strings.ReplaceAll(strings.ReplaceAll(ErrorPage, "%message%", msg), "%status%", http.StatusText(statusCode))
} }
_, _ = io.Copy(ctx.RespWriter, strings.NewReader(msg)) _, _ = ctx.RespWriter.Write([]byte(msg))
} }
func errorMessage(statusCode int) string { func errorMessage(statusCode int) string {

View file

@ -2,9 +2,7 @@ package context
import ( import (
stdContext "context" stdContext "context"
"io"
"net/http" "net/http"
"strings"
) )
type Context struct { type Context struct {
@ -41,7 +39,7 @@ func (c *Context) String(raw string, status ...int) {
code = status[0] code = status[0]
} }
c.RespWriter.WriteHeader(code) c.RespWriter.WriteHeader(code)
_, _ = io.Copy(c.RespWriter, strings.NewReader(raw)) _, _ = c.RespWriter.Write([]byte(raw))
} }
func (c *Context) IsMethod(m string) bool { func (c *Context) IsMethod(m string) bool {

View file

@ -28,7 +28,7 @@ func (p tmpDB) Put(name string, cert *certificate.Resource) error {
func (p tmpDB) Get(name string) (*certificate.Resource, error) { func (p tmpDB) Get(name string) (*certificate.Resource, error) {
cert, has := p.intern.Get(name) cert, has := p.intern.Get(name)
if !has { if !has {
return nil, fmt.Errorf("cert for '%s' not found", name) return nil, fmt.Errorf("cert for %q not found", name)
} }
return cert.(*certificate.Resource), nil return cert.(*certificate.Resource), nil
} }

View file

@ -27,7 +27,7 @@ const (
fileCacheTimeout = 5 * time.Minute fileCacheTimeout = 5 * time.Minute
// fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default. // fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default.
fileCacheSizeLimit = int64(1024 * 1024) fileCacheSizeLimit = int64(1000 * 1000)
) )
type FileResponse struct { type FileResponse struct {
@ -60,7 +60,7 @@ func (f FileResponse) createHttpResponse(cacheKey string) (http.Header, int) {
header.Set(ContentLengthHeader, fmt.Sprintf("%d", len(f.Body))) header.Set(ContentLengthHeader, fmt.Sprintf("%d", len(f.Body)))
header.Set(PagesCacheIndicatorHeader, "true") header.Set(PagesCacheIndicatorHeader, "true")
log.Trace().Msgf("fileCache for '%s' used", cacheKey) log.Trace().Msgf("fileCache for %q used", cacheKey)
return header, statusCode return header, statusCode
} }
@ -71,45 +71,46 @@ type BranchTimestamp struct {
} }
type writeCacheReader struct { type writeCacheReader struct {
rc io.ReadCloser originalReader io.ReadCloser
buff *bytes.Buffer buffer *bytes.Buffer
f *FileResponse rileResponse *FileResponse
cacheKey string cacheKey string
cache cache.SetGetKey cache cache.SetGetKey
hasErr bool hasError bool
} }
func (t *writeCacheReader) Read(p []byte) (n int, err error) { func (t *writeCacheReader) Read(p []byte) (n int, err error) {
n, err = t.rc.Read(p) n, err = t.originalReader.Read(p)
if err != nil { if err != nil {
t.hasErr = true log.Trace().Err(err).Msgf("[cache] original reader for %q has returned an error", t.cacheKey)
t.hasError = true
} else if n > 0 { } else if n > 0 {
_, _ = t.buff.Write(p[:n]) _, _ = t.buffer.Write(p[:n])
} }
return return
} }
func (t *writeCacheReader) Close() error { func (t *writeCacheReader) Close() error {
if !t.hasErr { if !t.hasError {
fc := *t.f fc := *t.rileResponse
fc.Body = t.buff.Bytes() fc.Body = t.buffer.Bytes()
_ = t.cache.Set(t.cacheKey, fc, fileCacheTimeout) _ = t.cache.Set(t.cacheKey, fc, fileCacheTimeout)
} }
log.Trace().Msgf("cacheReader for '%s' saved=%v closed", t.cacheKey, !t.hasErr) log.Trace().Msgf("cacheReader for %q saved=%t closed", t.cacheKey, !t.hasError)
return t.rc.Close() return t.originalReader.Close()
} }
func (f FileResponse) CreateCacheReader(r io.ReadCloser, cache cache.SetGetKey, cacheKey string) io.ReadCloser { func (f FileResponse) CreateCacheReader(r io.ReadCloser, cache cache.SetGetKey, cacheKey string) io.ReadCloser {
buf := []byte{}
if r == nil || cache == nil || cacheKey == "" { if r == nil || cache == nil || cacheKey == "" {
log.Error().Msg("could not create CacheReader") log.Error().Msg("could not create CacheReader")
return r return nil
} }
return &writeCacheReader{ return &writeCacheReader{
rc: r, originalReader: r,
buff: bytes.NewBuffer(buf), buffer: bytes.NewBuffer(make([]byte, 0)),
f: &f, rileResponse: &f,
cache: cache, cache: cache,
cacheKey: cacheKey, cacheKey: cacheKey,
} }
} }

View file

@ -29,6 +29,7 @@ const (
// pages server // pages server
PagesCacheIndicatorHeader = "X-Pages-Cache" PagesCacheIndicatorHeader = "X-Pages-Cache"
symlinkReadLimit = 10000
// gitea // gitea
giteaObjectTypeHeader = "X-Gitea-Object-Type" giteaObjectTypeHeader = "X-Gitea-Object-Type"
@ -107,7 +108,7 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
if cache.Exists { if cache.Exists {
if cache.IsSymlink { if cache.IsSymlink {
linkDest := string(cache.Body) linkDest := string(cache.Body)
log.Debug().Msgf("[cache] follow symlink from'%s' to '%s'", resource, linkDest) log.Debug().Msgf("[cache] follow symlink from %q to %q", resource, linkDest)
return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest) return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest)
} else { } else {
log.Debug().Msg("[cache] return bytes") log.Debug().Msg("[cache] return bytes")
@ -118,8 +119,7 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
} }
} }
// if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() { // not in cache, open reader via gitea api
// cachedResponse = cachedValue.(gitea.FileResponse)
reader, resp, err := client.sdkClient.GetFileReader(targetOwner, targetRepo, ref, resource, client.supportLFS) reader, resp, err := client.sdkClient.GetFileReader(targetOwner, targetRepo, ref, resource, client.supportLFS)
if resp != nil { if resp != nil {
switch resp.StatusCode { switch resp.StatusCode {
@ -127,55 +127,53 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
// first handle symlinks // first handle symlinks
{ {
objType := resp.Header.Get(giteaObjectTypeHeader) objType := resp.Header.Get(giteaObjectTypeHeader)
log.Trace().Msgf("server raw content object: %s", objType) log.Trace().Msgf("server raw content object %q", objType)
if client.followSymlinks && objType == objTypeSymlink { if client.followSymlinks && objType == objTypeSymlink {
// limit to 1000 chars
defer reader.Close() defer reader.Close()
linkDestBytes, err := io.ReadAll(io.LimitReader(reader, 10000)) // read limited chars for symlink
linkDestBytes, err := io.ReadAll(io.LimitReader(reader, symlinkReadLimit))
if err != nil { if err != nil {
return nil, nil, http.StatusInternalServerError, err return nil, nil, http.StatusInternalServerError, err
} }
linkDest := strings.TrimSpace(string(linkDestBytes)) linkDest := strings.TrimSpace(string(linkDestBytes))
// we store symlink not content to reduce duplicates in cache
if err := client.responseCache.Set(cacheKey, FileResponse{ if err := client.responseCache.Set(cacheKey, FileResponse{
Exists: true, Exists: true,
IsSymlink: true, IsSymlink: true,
Body: []byte(linkDest), Body: []byte(linkDest),
ETag: resp.Header.Get(ETagHeader), ETag: resp.Header.Get(ETagHeader),
}, fileCacheTimeout); err != nil { }, fileCacheTimeout); err != nil {
log.Error().Err(err).Msg("could not save symlink in cache") log.Error().Err(err).Msg("[cache] error on cache write")
} }
log.Debug().Msgf("follow symlink from '%s' to '%s'", resource, linkDest) log.Debug().Msgf("follow symlink from %q to %q", resource, linkDest)
return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest) return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest)
} }
} }
// now we are sure it's content // now we are sure it's content so set the MIME type
{ mimeType := client.getMimeTypeByExtension(resource)
// Set the MIME type resp.Response.Header.Set(ContentTypeHeader, mimeType)
mimeType := client.getMimeTypeByExtension(resource)
resp.Response.Header.Set(ContentTypeHeader, mimeType)
if !shouldRespBeSavedToCache(resp.Response) { if !shouldRespBeSavedToCache(resp.Response) {
return reader, resp.Response.Header, resp.StatusCode, err return reader, resp.Response.Header, resp.StatusCode, err
}
// now we write to cache and respond at the sime time
fileResp := FileResponse{
Exists: true,
ETag: resp.Header.Get(ETagHeader),
MimeType: mimeType,
}
return fileResp.CreateCacheReader(reader, client.responseCache, cacheKey), resp.Response.Header, resp.StatusCode, nil
} }
// now we write to cache and respond at the sime time
fileResp := FileResponse{
Exists: true,
ETag: resp.Header.Get(ETagHeader),
MimeType: mimeType,
}
return fileResp.CreateCacheReader(reader, client.responseCache, cacheKey), resp.Response.Header, resp.StatusCode, nil
case http.StatusNotFound: case http.StatusNotFound:
if err := client.responseCache.Set(cacheKey, FileResponse{ if err := client.responseCache.Set(cacheKey, FileResponse{
Exists: false, Exists: false,
ETag: resp.Header.Get(ETagHeader), ETag: resp.Header.Get(ETagHeader),
}, fileCacheTimeout); err != nil { }, fileCacheTimeout); err != nil {
log.Error().Err(err).Msg("could not save 404 in cache") log.Error().Err(err).Msg("[cache] error on cache write")
} }
return nil, resp.Response.Header, http.StatusNotFound, ErrorNotFound return nil, resp.Response.Header, http.StatusNotFound, ErrorNotFound
@ -192,19 +190,19 @@ func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchNam
if stamp, ok := client.responseCache.Get(cacheKey); ok && stamp != nil { if stamp, ok := client.responseCache.Get(cacheKey); ok && stamp != nil {
branchTimeStamp := stamp.(*BranchTimestamp) branchTimeStamp := stamp.(*BranchTimestamp)
if branchTimeStamp.notFound { if branchTimeStamp.notFound {
log.Trace().Msgf("use cache branch [%s] not found", branchName) log.Trace().Msgf("[cache] use branch %q not found", branchName)
return &BranchTimestamp{}, ErrorNotFound return &BranchTimestamp{}, ErrorNotFound
} }
log.Trace().Msgf("use cache branch [%s] exist", branchName) log.Trace().Msgf("[cache] use branch %q exist", branchName)
return branchTimeStamp, nil return branchTimeStamp, nil
} }
branch, resp, err := client.sdkClient.GetRepoBranch(repoOwner, repoName, branchName) branch, resp, err := client.sdkClient.GetRepoBranch(repoOwner, repoName, branchName)
if err != nil { if err != nil {
if resp != nil && resp.StatusCode == http.StatusNotFound { if resp != nil && resp.StatusCode == http.StatusNotFound {
log.Trace().Msgf("set cache branch [%s] not found", branchName) log.Trace().Msgf("[cache] set cache branch %q not found", branchName)
if err := client.responseCache.Set(cacheKey, &BranchTimestamp{Branch: branchName, notFound: true}, branchExistenceCacheTimeout); err != nil { if err := client.responseCache.Set(cacheKey, &BranchTimestamp{Branch: branchName, notFound: true}, branchExistenceCacheTimeout); err != nil {
log.Error().Err(err).Msgf("error on store of repo branch timestamp [%s/%s@%s]", repoOwner, repoName, branchName) log.Error().Err(err).Msg("[cache] error on cache write")
} }
return &BranchTimestamp{}, ErrorNotFound return &BranchTimestamp{}, ErrorNotFound
} }
@ -221,7 +219,7 @@ func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchNam
log.Trace().Msgf("set cache branch [%s] exist", branchName) log.Trace().Msgf("set cache branch [%s] exist", branchName)
if err := client.responseCache.Set(cacheKey, stamp, branchExistenceCacheTimeout); err != nil { if err := client.responseCache.Set(cacheKey, stamp, branchExistenceCacheTimeout); err != nil {
log.Error().Err(err).Msgf("error on store of repo branch timestamp [%s/%s@%s]", repoOwner, repoName, branchName) log.Error().Err(err).Msg("[cache] error on cache write")
} }
return stamp, nil return stamp, nil
} }
@ -243,7 +241,7 @@ func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (str
branch := repo.DefaultBranch branch := repo.DefaultBranch
if err := client.responseCache.Set(cacheKey, branch, defaultBranchCacheTimeout); err != nil { if err := client.responseCache.Set(cacheKey, branch, defaultBranchCacheTimeout); err != nil {
log.Error().Err(err).Msgf("error on store of repo default branch [%s/%s]", repoOwner, repoName) log.Error().Err(err).Msg("[cache] error on cache write")
} }
return branch, nil return branch, nil
} }
@ -254,7 +252,7 @@ func (client *Client) getMimeTypeByExtension(resource string) string {
if client.forbiddenMimeTypes[mimeTypeSplit[0]] || mimeType == "" { if client.forbiddenMimeTypes[mimeTypeSplit[0]] || mimeType == "" {
mimeType = client.defaultMimeType mimeType = client.defaultMimeType
} }
log.Trace().Msgf("probe mime: %s", mimeType) log.Trace().Msgf("probe mime of %q is %q", resource, mimeType)
return mimeType return mimeType
} }
@ -263,12 +261,12 @@ func shouldRespBeSavedToCache(resp *http.Response) bool {
return false return false
} }
contentLengRaw := resp.Header.Get(ContentLengthHeader) contentLengthRaw := resp.Header.Get(ContentLengthHeader)
if contentLengRaw == "" { if contentLengthRaw == "" {
return false return false
} }
contentLeng, err := strconv.ParseInt(contentLengRaw, 10, 64) contentLeng, err := strconv.ParseInt(contentLengthRaw, 10, 64)
if err != nil { if err != nil {
log.Error().Err(err).Msg("could not parse content length") log.Error().Err(err).Msg("could not parse content length")
} }

View file

@ -95,7 +95,7 @@ func Handler(mainDomainSuffix, rawDomain string,
// TODO: move into external func to not alert vars indirectly // TODO: move into external func to not alert vars indirectly
tryBranch1 := func(log zerolog.Logger, repo, branch string, _path []string, canonicalLink string) bool { tryBranch1 := func(log zerolog.Logger, repo, branch string, _path []string, canonicalLink string) bool {
if repo == "" { if repo == "" {
log.Debug().Msg("tryBranch: repo == ''") log.Debug().Msg("tryBranch: repo is empty")
return false return false
} }
@ -213,7 +213,7 @@ func Handler(mainDomainSuffix, rawDomain string,
canonicalDomainCache) canonicalDomainCache)
} else { } else {
html.ReturnErrorPage(ctx, html.ReturnErrorPage(ctx,
fmt.Sprintf("explizite set branch '%s' do not exist at '%s/%s'", branch, targetOwner, targetRepo), fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", branch, targetOwner, targetRepo),
http.StatusFailedDependency) http.StatusFailedDependency)
} }
return return
@ -232,7 +232,7 @@ func Handler(mainDomainSuffix, rawDomain string,
canonicalDomainCache) canonicalDomainCache)
} else { } else {
html.ReturnErrorPage(ctx, html.ReturnErrorPage(ctx,
fmt.Sprintf("explizite set branch '%s' do not exist at '%s/%s'", branch, targetOwner, "pages"), fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", branch, targetOwner, "pages"),
http.StatusFailedDependency) http.StatusFailedDependency)
} }
return return

View file

@ -20,7 +20,7 @@ func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string) *
log.Err(err).Msg("Could't fetch default branch from repository") log.Err(err).Msg("Could't fetch default branch from repository")
return nil return nil
} }
log.Debug().Msgf("Succesfully fetched default branch '%s' from Gitea", defaultBranch) log.Debug().Msgf("Succesfully fetched default branch %q from Gitea", defaultBranch)
branch = defaultBranch branch = defaultBranch
} }

View file

@ -66,7 +66,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin
if branch == nil || branch.Branch == "" { if branch == nil || branch.Branch == "" {
html.ReturnErrorPage(ctx, html.ReturnErrorPage(ctx,
fmt.Sprintf("could not get timestamp of branch '%s'", o.TargetBranch), fmt.Sprintf("could not get timestamp of branch %q", o.TargetBranch),
http.StatusFailedDependency) http.StatusFailedDependency)
return true return true
} }
@ -182,7 +182,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin
if reader != nil { if reader != nil {
_, err := io.Copy(ctx.RespWriter, reader) _, err := io.Copy(ctx.RespWriter, reader)
if err != nil { if err != nil {
log.Printf("Couldn't write body: %s\n", err) log.Error().Err(err).Msgf("Couldn't write body for %q", o.TargetPath)
html.ReturnErrorPage(ctx, "", http.StatusInternalServerError) html.ReturnErrorPage(ctx, "", http.StatusInternalServerError)
return true return true
} }