pages/pages-server
1
0
Fork 0
mirror of https://codeberg.org/Codeberg/pages-server.git synced 2024-11-05 14:07:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'std-http' into refactor_split-long-funcs

Browse source
This commit is contained in:
6543 2022-11-12 01:08:24 +01:00
commit 3d9ffcf8d7
No known key found for this signature in database
GPG key ID: B8BE6D610E61C862
9 changed files with 73 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.woodpecker.yml
View file

@ -15,6 +15,10 @@ pipeline:
- "[ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }"
- golangci-lint run --timeout 5m --build-tags integration
editor-config:
group: compliant
image: mstruebing/editorconfig-checker
build:
group: compliant
image: codeberg.org/6543/docker-images/golang_just
@ -90,6 +94,7 @@ pipeline:
from_secret: bot_token
when:
event: [ "push" ]
branch: ${CI_REPO_DEFAULT_BRANCH}
docker-tag:
image: plugins/kaniko

7
html/error.go
View file

@ -1,7 +1,6 @@
package html
import (
"io"
"net/http"
"strconv"
"strings"
@ -9,8 +8,8 @@ import (
"codeberg.org/codeberg/pages/server/context"
)
// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced
// with the provided status code.
// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body,
// with "%status%" and %message% replaced with the provided statusCode and msg
func ReturnErrorPage(ctx *context.Context, msg string, statusCode int) {
ctx.RespWriter.Header().Set("Content-Type", "text/html; charset=utf-8")
ctx.RespWriter.WriteHeader(statusCode)
@ -22,7 +21,7 @@ func ReturnErrorPage(ctx *context.Context, msg string, statusCode int) {
msg = strings.ReplaceAll(strings.ReplaceAll(ErrorPage, "%message%", msg), "%status%", http.StatusText(statusCode))
}
_, _ = io.Copy(ctx.RespWriter, strings.NewReader(msg))
_, _ = ctx.RespWriter.Write([]byte(msg))
}
func errorMessage(statusCode int) string {

4
server/context/context.go
View file

@ -2,9 +2,7 @@ package context
import (
stdContext "context"
"io"
"net/http"
"strings"
)
type Context struct {
@ -41,7 +39,7 @@ func (c *Context) String(raw string, status ...int) {
code = status[0]
}
c.RespWriter.WriteHeader(code)
_, _ = io.Copy(c.RespWriter, strings.NewReader(raw))
_, _ = c.RespWriter.Write([]byte(raw))
}
func (c *Context) IsMethod(m string) bool {

2
server/database/mock.go
View file

@ -28,7 +28,7 @@ func (p tmpDB) Put(name string, cert *certificate.Resource) error {
func (p tmpDB) Get(name string) (*certificate.Resource, error) {
cert, has := p.intern.Get(name)
if !has {
return nil, fmt.Errorf("cert for '%s' not found", name)
return nil, fmt.Errorf("cert for %q not found", name)
}
return cert.(*certificate.Resource), nil
}

47
server/gitea/cache.go
View file

@ -27,7 +27,7 @@ const (
fileCacheTimeout = 5 * time.Minute
// fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default.
fileCacheSizeLimit = int64(1024 * 1024)
fileCacheSizeLimit = int64(1000 * 1000)
)
type FileResponse struct {
@ -60,7 +60,7 @@ func (f FileResponse) createHttpResponse(cacheKey string) (http.Header, int) {
header.Set(ContentLengthHeader, fmt.Sprintf("%d", len(f.Body)))
header.Set(PagesCacheIndicatorHeader, "true")
log.Trace().Msgf("fileCache for '%s' used", cacheKey)
log.Trace().Msgf("fileCache for %q used", cacheKey)
return header, statusCode
}
@ -71,45 +71,46 @@ type BranchTimestamp struct {
}
type writeCacheReader struct {
rc io.ReadCloser
buff *bytes.Buffer
f *FileResponse
cacheKey string
cache cache.SetGetKey
hasErr bool
originalReader io.ReadCloser
buffer *bytes.Buffer
rileResponse *FileResponse
cacheKey string
cache cache.SetGetKey
hasError bool
}
func (t *writeCacheReader) Read(p []byte) (n int, err error) {
n, err = t.rc.Read(p)
n, err = t.originalReader.Read(p)
if err != nil {
t.hasErr = true
log.Trace().Err(err).Msgf("[cache] original reader for %q has returned an error", t.cacheKey)
t.hasError = true
} else if n > 0 {
_, _ = t.buff.Write(p[:n])
_, _ = t.buffer.Write(p[:n])
}
return
}
func (t *writeCacheReader) Close() error {
if !t.hasErr {
fc := *t.f
fc.Body = t.buff.Bytes()
if !t.hasError {
fc := *t.rileResponse
fc.Body = t.buffer.Bytes()
_ = t.cache.Set(t.cacheKey, fc, fileCacheTimeout)
}
log.Trace().Msgf("cacheReader for '%s' saved=%v closed", t.cacheKey, !t.hasErr)
return t.rc.Close()
log.Trace().Msgf("cacheReader for %q saved=%t closed", t.cacheKey, !t.hasError)
return t.originalReader.Close()
}
func (f FileResponse) CreateCacheReader(r io.ReadCloser, cache cache.SetGetKey, cacheKey string) io.ReadCloser {
buf := []byte{}
if r == nil || cache == nil || cacheKey == "" {
log.Error().Msg("could not create CacheReader")
return r
return nil
}
return &writeCacheReader{
rc: r,
buff: bytes.NewBuffer(buf),
f: &f,
cache: cache,
cacheKey: cacheKey,
originalReader: r,
buffer: bytes.NewBuffer(make([]byte, 0)),
rileResponse: &f,
cache: cache,
cacheKey: cacheKey,
}
}

68
server/gitea/client.go
View file

@ -29,6 +29,7 @@ const (
// pages server
PagesCacheIndicatorHeader = "X-Pages-Cache"
symlinkReadLimit = 10000
// gitea
giteaObjectTypeHeader = "X-Gitea-Object-Type"
@ -107,7 +108,7 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
if cache.Exists {
if cache.IsSymlink {
linkDest := string(cache.Body)
log.Debug().Msgf("[cache] follow symlink from'%s' to '%s'", resource, linkDest)
log.Debug().Msgf("[cache] follow symlink from %q to %q", resource, linkDest)
return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest)
} else {
log.Debug().Msg("[cache] return bytes")
@ -118,8 +119,7 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
}
}
// if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() {
// cachedResponse = cachedValue.(gitea.FileResponse)
// not in cache, open reader via gitea api
reader, resp, err := client.sdkClient.GetFileReader(targetOwner, targetRepo, ref, resource, client.supportLFS)
if resp != nil {
switch resp.StatusCode {
@ -127,55 +127,53 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
// first handle symlinks
{
objType := resp.Header.Get(giteaObjectTypeHeader)
log.Trace().Msgf("server raw content object: %s", objType)
log.Trace().Msgf("server raw content object %q", objType)
if client.followSymlinks && objType == objTypeSymlink {
// limit to 1000 chars
defer reader.Close()
linkDestBytes, err := io.ReadAll(io.LimitReader(reader, 10000))
// read limited chars for symlink
linkDestBytes, err := io.ReadAll(io.LimitReader(reader, symlinkReadLimit))
if err != nil {
return nil, nil, http.StatusInternalServerError, err
}
linkDest := strings.TrimSpace(string(linkDestBytes))
// we store symlink not content to reduce duplicates in cache
if err := client.responseCache.Set(cacheKey, FileResponse{
Exists: true,
IsSymlink: true,
Body: []byte(linkDest),
ETag: resp.Header.Get(ETagHeader),
}, fileCacheTimeout); err != nil {
log.Error().Err(err).Msg("could not save symlink in cache")
log.Error().Err(err).Msg("[cache] error on cache write")
}
log.Debug().Msgf("follow symlink from '%s' to '%s'", resource, linkDest)
log.Debug().Msgf("follow symlink from %q to %q", resource, linkDest)
return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest)
}
}
// now we are sure it's content
{
// Set the MIME type
mimeType := client.getMimeTypeByExtension(resource)
resp.Response.Header.Set(ContentTypeHeader, mimeType)
// now we are sure it's content so set the MIME type
mimeType := client.getMimeTypeByExtension(resource)
resp.Response.Header.Set(ContentTypeHeader, mimeType)
if !shouldRespBeSavedToCache(resp.Response) {
return reader, resp.Response.Header, resp.StatusCode, err
}
// now we write to cache and respond at the sime time
fileResp := FileResponse{
Exists: true,
ETag: resp.Header.Get(ETagHeader),
MimeType: mimeType,
}
return fileResp.CreateCacheReader(reader, client.responseCache, cacheKey), resp.Response.Header, resp.StatusCode, nil
if !shouldRespBeSavedToCache(resp.Response) {
return reader, resp.Response.Header, resp.StatusCode, err
}
// now we write to cache and respond at the sime time
fileResp := FileResponse{
Exists: true,
ETag: resp.Header.Get(ETagHeader),
MimeType: mimeType,
}
return fileResp.CreateCacheReader(reader, client.responseCache, cacheKey), resp.Response.Header, resp.StatusCode, nil
case http.StatusNotFound:
if err := client.responseCache.Set(cacheKey, FileResponse{
Exists: false,
ETag: resp.Header.Get(ETagHeader),
}, fileCacheTimeout); err != nil {
log.Error().Err(err).Msg("could not save 404 in cache")
log.Error().Err(err).Msg("[cache] error on cache write")
}
return nil, resp.Response.Header, http.StatusNotFound, ErrorNotFound
@ -192,19 +190,19 @@ func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchNam
if stamp, ok := client.responseCache.Get(cacheKey); ok && stamp != nil {
branchTimeStamp := stamp.(*BranchTimestamp)
if branchTimeStamp.notFound {
log.Trace().Msgf("use cache branch [%s] not found", branchName)
log.Trace().Msgf("[cache] use branch %q not found", branchName)
return &BranchTimestamp{}, ErrorNotFound
}
log.Trace().Msgf("use cache branch [%s] exist", branchName)
log.Trace().Msgf("[cache] use branch %q exist", branchName)
return branchTimeStamp, nil
}
branch, resp, err := client.sdkClient.GetRepoBranch(repoOwner, repoName, branchName)
if err != nil {
if resp != nil && resp.StatusCode == http.StatusNotFound {
log.Trace().Msgf("set cache branch [%s] not found", branchName)
log.Trace().Msgf("[cache] set cache branch %q not found", branchName)
if err := client.responseCache.Set(cacheKey, &BranchTimestamp{Branch: branchName, notFound: true}, branchExistenceCacheTimeout); err != nil {
log.Error().Err(err).Msgf("error on store of repo branch timestamp [%s/%s@%s]", repoOwner, repoName, branchName)
log.Error().Err(err).Msg("[cache] error on cache write")
}
return &BranchTimestamp{}, ErrorNotFound
}
@ -221,7 +219,7 @@ func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchNam
log.Trace().Msgf("set cache branch [%s] exist", branchName)
if err := client.responseCache.Set(cacheKey, stamp, branchExistenceCacheTimeout); err != nil {
log.Error().Err(err).Msgf("error on store of repo branch timestamp [%s/%s@%s]", repoOwner, repoName, branchName)
log.Error().Err(err).Msg("[cache] error on cache write")
}
return stamp, nil
}
@ -243,7 +241,7 @@ func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (str
branch := repo.DefaultBranch
if err := client.responseCache.Set(cacheKey, branch, defaultBranchCacheTimeout); err != nil {
log.Error().Err(err).Msgf("error on store of repo default branch [%s/%s]", repoOwner, repoName)
log.Error().Err(err).Msg("[cache] error on cache write")
}
return branch, nil
}
@ -254,7 +252,7 @@ func (client *Client) getMimeTypeByExtension(resource string) string {
if client.forbiddenMimeTypes[mimeTypeSplit[0]] || mimeType == "" {
mimeType = client.defaultMimeType
}
log.Trace().Msgf("probe mime: %s", mimeType)
log.Trace().Msgf("probe mime of %q is %q", resource, mimeType)
return mimeType
}
@ -263,12 +261,12 @@ func shouldRespBeSavedToCache(resp *http.Response) bool {
return false
}
contentLengRaw := resp.Header.Get(ContentLengthHeader)
if contentLengRaw == "" {
contentLengthRaw := resp.Header.Get(ContentLengthHeader)
if contentLengthRaw == "" {
return false
}
contentLeng, err := strconv.ParseInt(contentLengRaw, 10, 64)
contentLeng, err := strconv.ParseInt(contentLengthRaw, 10, 64)
if err != nil {
log.Error().Err(err).Msg("could not parse content length")
}

6
server/handler.go
View file

@ -95,7 +95,7 @@ func Handler(mainDomainSuffix, rawDomain string,
// TODO: move into external func to not alert vars indirectly
tryBranch1 := func(log zerolog.Logger, repo, branch string, _path []string, canonicalLink string) bool {
if repo == "" {
log.Debug().Msg("tryBranch: repo == ''")
log.Debug().Msg("tryBranch: repo is empty")
return false
}
@ -213,7 +213,7 @@ func Handler(mainDomainSuffix, rawDomain string,
canonicalDomainCache)
} else {
html.ReturnErrorPage(ctx,
fmt.Sprintf("explizite set branch '%s' do not exist at '%s/%s'", branch, targetOwner, targetRepo),
fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", branch, targetOwner, targetRepo),
http.StatusFailedDependency)
}
return
@ -232,7 +232,7 @@ func Handler(mainDomainSuffix, rawDomain string,
canonicalDomainCache)
} else {
html.ReturnErrorPage(ctx,
fmt.Sprintf("explizite set branch '%s' do not exist at '%s/%s'", branch, targetOwner, "pages"),
fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", branch, targetOwner, "pages"),
http.StatusFailedDependency)
}
return

2
server/upstream/helper.go
View file

@ -20,7 +20,7 @@ func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string) *
log.Err(err).Msg("Could't fetch default branch from repository")
return nil
}
log.Debug().Msgf("Succesfully fetched default branch '%s' from Gitea", defaultBranch)
log.Debug().Msgf("Succesfully fetched default branch %q from Gitea", defaultBranch)
branch = defaultBranch
}

4
server/upstream/upstream.go
View file

@ -66,7 +66,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin
if branch == nil || branch.Branch == "" {
html.ReturnErrorPage(ctx,
fmt.Sprintf("could not get timestamp of branch '%s'", o.TargetBranch),
fmt.Sprintf("could not get timestamp of branch %q", o.TargetBranch),
http.StatusFailedDependency)
return true
}
@ -182,7 +182,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin
if reader != nil {
_, err := io.Copy(ctx.RespWriter, reader)
if err != nil {
log.Printf("Couldn't write body: %s\n", err)
log.Error().Err(err).Msgf("Couldn't write body for %q", o.TargetPath)
html.ReturnErrorPage(ctx, "", http.StatusInternalServerError)
return true
}