From 8e67d28c4fddfc0b4f4f0dd5db1e60e4d80cf56a Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Fri, 11 Nov 2022 23:51:45 +0100 Subject: [PATCH 01/18] Add editorconfig, fix files and lint via ci --- .ecrc | 9 +++++++++ .editorconfig | 17 +++++++++++++++++ .woodpecker.yml | 4 ++++ Dockerfile | 4 ++-- Justfile | 1 + README.md | 8 +++----- haproxy-sni/dhparam.pem | 2 +- haproxy-sni/docker-compose.yml | 2 +- html/404.html | 6 +++--- 9 files changed, 41 insertions(+), 12 deletions(-) create mode 100644 .ecrc create mode 100644 .editorconfig diff --git a/.ecrc b/.ecrc new file mode 100644 index 0000000..d9ee788 --- /dev/null +++ b/.ecrc @@ -0,0 +1,9 @@ +{ + "Exclude": [ + ".git", + "go.mod", "go.sum", + "vendor", + "LICENSE", + "_test.go" + ] +} diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..354a828 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,17 @@ +root = true + +[*] +indent_style = space +indent_size = 2 +tab_width = 2 +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +[*.go] +indent_style = tab + +[*.md] +trim_trailing_whitespace = false +indent_size = 1 diff --git a/.woodpecker.yml b/.woodpecker.yml index db51eba..5c9a7fd 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -15,6 +15,10 @@ pipeline: - "[ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }" - golangci-lint run --timeout 5m --build-tags integration + editor-config: + group: compliant + image: mstruebing/editorconfig-checker + build: group: compliant image: codeberg.org/6543/docker-images/golang_just diff --git a/Dockerfile b/Dockerfile index 71dd236..904d6f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,5 +11,5 @@ COPY --from=build /workspace/pages /pages COPY --from=build \ /etc/ssl/certs/ca-certificates.crt \ /etc/ssl/certs/ca-certificates.crt - -ENTRYPOINT ["/pages"] \ No newline at end of file + +ENTRYPOINT ["/pages"] diff --git a/Justfile b/Justfile index 03a7436..f7ea414 100644 --- a/Justfile +++ b/Justfile @@ -18,6 +18,7 @@ build-tag VERSION: lint: tool-golangci tool-gofumpt [ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }; \ golangci-lint run --timeout 5m --build-tags integration + # TODO: run editorconfig-checker fmt: tool-gofumpt gofumpt -w --extra . diff --git a/README.md b/README.md index c4758ac..50cdc3c 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,6 @@ It is suitable to be deployed by other Gitea instances, too, to offer static pag **End user documentation** can mainly be found at the [Wiki](https://codeberg.org/Codeberg/pages-server/wiki/Overview) and the [Codeberg Documentation](https://docs.codeberg.org/codeberg-pages/). - ## Quickstart This is the new Codeberg Pages server, a solution for serving static pages from Gitea repositories. @@ -20,17 +19,16 @@ first line will be the canonical domain/URL; all other occurrences will be redir 2) add a CNAME entry to your domain, pointing to `[[{branch}.]{repo}.]{owner}.codeberg.page` (repo defaults to "pages", "branch" defaults to the default branch if "repo" is "pages", or to "pages" if "repo" is something else. If the branch name contains slash characters, you need to replace "/" in the branch name to "~"): - `www.example.org. IN CNAME main.pages.example.codeberg.page.` + `www.example.org. IN CNAME main.pages.example.codeberg.page.` 3) if a CNAME is set for "www.example.org", you can redirect there from the naked domain by adding an ALIAS record for "example.org" (if your provider allows ALIAS or similar records, otherwise use A/AAAA), together with a TXT record that points to your repo (just like the CNAME record): - `example.org IN ALIAS codeberg.page.` - `example.org IN TXT main.pages.example.codeberg.page.` + `example.org IN ALIAS codeberg.page.` + `example.org IN TXT main.pages.example.codeberg.page.` Certificates are generated, updated and cleaned up automatically via Let's Encrypt through a TLS challenge. - ## Deployment **Warning: Some Caveats Apply** diff --git a/haproxy-sni/dhparam.pem b/haproxy-sni/dhparam.pem index 088f967..9b182b7 100644 --- a/haproxy-sni/dhparam.pem +++ b/haproxy-sni/dhparam.pem @@ -5,4 +5,4 @@ MIIBCAKCAQEA//////////+t+FRYortKmq/cViAnPTzx2LnFg84tNpWp4TZBFGQz YdEIqUuyyOP7uWrat2DX9GgdT0Kj3jlN9K5W7edjcrsZCwenyO4KbXCeAvzhzffi 7MA0BM0oNC9hkXL+nOmFg/+OTxIy7vKBg8P+OxtMb61zO7X8vC7CIAXFjvGDfRaD ssbzSibBsu/6iGtCOGEoXJf//////////wIBAg== ------END DH PARAMETERS----- \ No newline at end of file +-----END DH PARAMETERS----- diff --git a/haproxy-sni/docker-compose.yml b/haproxy-sni/docker-compose.yml index 4dd8677..66ff52b 100644 --- a/haproxy-sni/docker-compose.yml +++ b/haproxy-sni/docker-compose.yml @@ -19,4 +19,4 @@ services: volumes: - ./pages-www:/srv:ro - ./pages.Caddyfile:/etc/caddy/Caddyfile:ro - + diff --git a/html/404.html b/html/404.html index b7ec96e..21d968e 100644 --- a/html/404.html +++ b/html/404.html @@ -23,11 +23,11 @@

- Page not found! + Page not found!

- Sorry, but this page couldn't be found or is inaccessible (%status).
- We hope this isn't a problem on our end ;) - Make sure to check the troubleshooting section in the Docs! + Sorry, but this page couldn't be found or is inaccessible (%status).
+ We hope this isn't a problem on our end ;) - Make sure to check the troubleshooting section in the Docs!
From 46a343c1823a347ab56092a22b53b21483a345d3 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Fri, 11 Nov 2022 23:57:38 +0100 Subject: [PATCH 02/18] fix merge conflict relict --- server/upstream/upstream.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index 5704583..370c4ee 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -196,7 +196,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin if reader != nil { _, err := io.Copy(ctx.RespWriter, reader) if err != nil { - log.Printf("Couldn't write body: %s\n", err) + log.Error().Err(err).Msgf("Couldn't write body for %q", o.TargetPath) html.ReturnErrorPage(ctx, "", http.StatusInternalServerError) return true } From f635c21c64ee7ad4078170e3ffc2283a53fdf6a1 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 12 Nov 2022 00:05:51 +0100 Subject: [PATCH 03/18] update func desc --- html/error.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/html/error.go b/html/error.go index 4194533..a1537e1 100644 --- a/html/error.go +++ b/html/error.go @@ -9,8 +9,8 @@ import ( "codeberg.org/codeberg/pages/server/context" ) -// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced -// with the provided status code. +// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body, +// with "%status%" and %message% replaced with the provided statusCode and msg func ReturnErrorPage(ctx *context.Context, msg string, statusCode int) { ctx.RespWriter.Header().Set("Content-Type", "text/html; charset=utf-8") ctx.RespWriter.WriteHeader(statusCode) From b759aaf67331965ac4515b23a0d50e2b9bed848d Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 12 Nov 2022 00:06:34 +0100 Subject: [PATCH 04/18] nit --- html/error.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/html/error.go b/html/error.go index a1537e1..33b24c8 100644 --- a/html/error.go +++ b/html/error.go @@ -1,7 +1,6 @@ package html import ( - "io" "net/http" "strconv" "strings" @@ -22,7 +21,7 @@ func ReturnErrorPage(ctx *context.Context, msg string, statusCode int) { msg = strings.ReplaceAll(strings.ReplaceAll(ErrorPage, "%message%", msg), "%status%", http.StatusText(statusCode)) } - _, _ = io.Copy(ctx.RespWriter, strings.NewReader(msg)) + ctx.RespWriter.Write([]byte(msg)) } func errorMessage(statusCode int) string { From cba996978ed89b1bfdba0afa6b98706c3e9d213f Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 12 Nov 2022 00:08:56 +0100 Subject: [PATCH 05/18] nit picking of legacy --- server/context/context.go | 4 +--- server/gitea/cache.go | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/server/context/context.go b/server/context/context.go index 6fffa74..6bb599d 100644 --- a/server/context/context.go +++ b/server/context/context.go @@ -2,9 +2,7 @@ package context import ( stdContext "context" - "io" "net/http" - "strings" ) type Context struct { @@ -41,7 +39,7 @@ func (c *Context) String(raw string, status ...int) { code = status[0] } c.RespWriter.WriteHeader(code) - _, _ = io.Copy(c.RespWriter, strings.NewReader(raw)) + c.RespWriter.Write([]byte(raw)) } func (c *Context) IsMethod(m string) bool { diff --git a/server/gitea/cache.go b/server/gitea/cache.go index 248c858..19a7b1f 100644 --- a/server/gitea/cache.go +++ b/server/gitea/cache.go @@ -27,7 +27,7 @@ const ( fileCacheTimeout = 5 * time.Minute // fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default. - fileCacheSizeLimit = int64(1024 * 1024) + fileCacheSizeLimit = int64(1000 * 1000) ) type FileResponse struct { From ae610d301ab5fc68e14b0e6934fc140e3bd5d3b4 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 12 Nov 2022 00:15:06 +0100 Subject: [PATCH 06/18] code format --- server/gitea/cache.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/gitea/cache.go b/server/gitea/cache.go index 19a7b1f..b1d66f3 100644 --- a/server/gitea/cache.go +++ b/server/gitea/cache.go @@ -60,7 +60,7 @@ func (f FileResponse) createHttpResponse(cacheKey string) (http.Header, int) { header.Set(ContentLengthHeader, fmt.Sprintf("%d", len(f.Body))) header.Set(PagesCacheIndicatorHeader, "true") - log.Trace().Msgf("fileCache for '%s' used", cacheKey) + log.Trace().Msgf("fileCache for %q used", cacheKey) return header, statusCode } @@ -95,19 +95,19 @@ func (t *writeCacheReader) Close() error { fc.Body = t.buff.Bytes() _ = t.cache.Set(t.cacheKey, fc, fileCacheTimeout) } - log.Trace().Msgf("cacheReader for '%s' saved=%v closed", t.cacheKey, !t.hasErr) + log.Trace().Msgf("cacheReader for %q saved=%t closed", t.cacheKey, !t.hasErr) return t.rc.Close() } func (f FileResponse) CreateCacheReader(r io.ReadCloser, cache cache.SetGetKey, cacheKey string) io.ReadCloser { - buf := []byte{} if r == nil || cache == nil || cacheKey == "" { log.Error().Msg("could not create CacheReader") - return r + return nil } + return &writeCacheReader{ rc: r, - buff: bytes.NewBuffer(buf), + buff: bytes.NewBuffer(make([]byte, 0)), f: &f, cache: cache, cacheKey: cacheKey, From 81bc8fec08477827cb8be0be73fc0ebe03f93941 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 12 Nov 2022 00:20:19 +0100 Subject: [PATCH 07/18] '%s' -> %q --- server/database/mock.go | 2 +- server/gitea/client.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/database/mock.go b/server/database/mock.go index e6c1b5a..dfe2316 100644 --- a/server/database/mock.go +++ b/server/database/mock.go @@ -28,7 +28,7 @@ func (p tmpDB) Put(name string, cert *certificate.Resource) error { func (p tmpDB) Get(name string) (*certificate.Resource, error) { cert, has := p.intern.Get(name) if !has { - return nil, fmt.Errorf("cert for '%s' not found", name) + return nil, fmt.Errorf("cert for %q not found", name) } return cert.(*certificate.Resource), nil } diff --git a/server/gitea/client.go b/server/gitea/client.go index ef35f64..575ebbc 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -107,7 +107,7 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str if cache.Exists { if cache.IsSymlink { linkDest := string(cache.Body) - log.Debug().Msgf("[cache] follow symlink from'%s' to '%s'", resource, linkDest) + log.Debug().Msgf("[cache] follow symlink from %q to %q", resource, linkDest) return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest) } else { log.Debug().Msg("[cache] return bytes") @@ -127,7 +127,7 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str // first handle symlinks { objType := resp.Header.Get(giteaObjectTypeHeader) - log.Trace().Msgf("server raw content object: %s", objType) + log.Trace().Msgf("server raw content object %q", objType) if client.followSymlinks && objType == objTypeSymlink { // limit to 1000 chars defer reader.Close() From 24039fd18a5a4830611dab3e79805734e8e1a65b Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 12 Nov 2022 00:21:45 +0100 Subject: [PATCH 08/18] '%s' -> %q --- server/gitea/client.go | 2 +- server/handler.go | 4 ++-- server/upstream/helper.go | 2 +- server/upstream/upstream.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/gitea/client.go b/server/gitea/client.go index 575ebbc..2e5631f 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -146,7 +146,7 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str log.Error().Err(err).Msg("could not save symlink in cache") } - log.Debug().Msgf("follow symlink from '%s' to '%s'", resource, linkDest) + log.Debug().Msgf("follow symlink from %q to %q", resource, linkDest) return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest) } } diff --git a/server/handler.go b/server/handler.go index 460875c..da33702 100644 --- a/server/handler.go +++ b/server/handler.go @@ -212,7 +212,7 @@ func Handler(mainDomainSuffix, rawDomain string, canonicalDomainCache) } else { html.ReturnErrorPage(ctx, - fmt.Sprintf("explizite set branch '%s' do not exist at '%s/%s'", branch, targetOwner, targetRepo), + fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", branch, targetOwner, targetRepo), http.StatusFailedDependency) } return @@ -231,7 +231,7 @@ func Handler(mainDomainSuffix, rawDomain string, canonicalDomainCache) } else { html.ReturnErrorPage(ctx, - fmt.Sprintf("explizite set branch '%s' do not exist at '%s/%s'", branch, targetOwner, "pages"), + fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", branch, targetOwner, "pages"), http.StatusFailedDependency) } return diff --git a/server/upstream/helper.go b/server/upstream/helper.go index fa872c2..6bc23c8 100644 --- a/server/upstream/helper.go +++ b/server/upstream/helper.go @@ -20,7 +20,7 @@ func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string) * log.Err(err).Msg("Could't fetch default branch from repository") return nil } - log.Debug().Msgf("Succesfully fetched default branch '%s' from Gitea", defaultBranch) + log.Debug().Msgf("Succesfully fetched default branch %q from Gitea", defaultBranch) branch = defaultBranch } diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index 370c4ee..d37c35e 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -66,7 +66,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin if branch == nil || branch.Branch == "" { html.ReturnErrorPage(ctx, - fmt.Sprintf("could not get timestamp of branch '%s'", o.TargetBranch), + fmt.Sprintf("could not get timestamp of branch %q", o.TargetBranch), http.StatusFailedDependency) return true } From 69eabb248a26e33b65732604267c486fd0daee84 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 12 Nov 2022 00:23:45 +0100 Subject: [PATCH 09/18] CI publish next only on default branch --- .woodpecker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.woodpecker.yml b/.woodpecker.yml index 5c9a7fd..20254fe 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -94,6 +94,7 @@ pipeline: from_secret: bot_token when: event: [ "push" ] + branch: ${CI_REPO_DEFAULT_BRANCH} docker-tag: image: plugins/kaniko From 9df4a50e10c66e687f072afe974792dc66778ef8 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 12 Nov 2022 00:27:40 +0100 Subject: [PATCH 10/18] rm dead code and add nice code comment --- server/gitea/client.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/gitea/client.go b/server/gitea/client.go index 2e5631f..4ca3b90 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -118,8 +118,7 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str } } - // if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() { - // cachedResponse = cachedValue.(gitea.FileResponse) + // not in cache, open reader via gitea api reader, resp, err := client.sdkClient.GetFileReader(targetOwner, targetRepo, ref, resource, client.supportLFS) if resp != nil { switch resp.StatusCode { From ba00db990bca0bb4d05df6a66686ea488bb59e2d Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 12 Nov 2022 00:29:45 +0100 Subject: [PATCH 11/18] better code comment and move val into its own const --- server/gitea/client.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/gitea/client.go b/server/gitea/client.go index 4ca3b90..e77e26e 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -29,6 +29,7 @@ const ( // pages server PagesCacheIndicatorHeader = "X-Pages-Cache" + symlinkReadLimit = 10000 // gitea giteaObjectTypeHeader = "X-Gitea-Object-Type" @@ -128,9 +129,9 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str objType := resp.Header.Get(giteaObjectTypeHeader) log.Trace().Msgf("server raw content object %q", objType) if client.followSymlinks && objType == objTypeSymlink { - // limit to 1000 chars defer reader.Close() - linkDestBytes, err := io.ReadAll(io.LimitReader(reader, 10000)) + // read limited chars for symlink + linkDestBytes, err := io.ReadAll(io.LimitReader(reader, symlinkReadLimit)) if err != nil { return nil, nil, http.StatusInternalServerError, err } From cf78f5fda27c6aa303d9dfb3a066de65e9949b63 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 12 Nov 2022 00:32:32 +0100 Subject: [PATCH 12/18] format code --- server/gitea/client.go | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/server/gitea/client.go b/server/gitea/client.go index e77e26e..47280f0 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -151,25 +151,22 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str } } - // now we are sure it's content - { - // Set the MIME type - mimeType := client.getMimeTypeByExtension(resource) - resp.Response.Header.Set(ContentTypeHeader, mimeType) + // now we are sure it's content so set the MIME type + mimeType := client.getMimeTypeByExtension(resource) + resp.Response.Header.Set(ContentTypeHeader, mimeType) - if !shouldRespBeSavedToCache(resp.Response) { - return reader, resp.Response.Header, resp.StatusCode, err - } - - // now we write to cache and respond at the sime time - fileResp := FileResponse{ - Exists: true, - ETag: resp.Header.Get(ETagHeader), - MimeType: mimeType, - } - return fileResp.CreateCacheReader(reader, client.responseCache, cacheKey), resp.Response.Header, resp.StatusCode, nil + if !shouldRespBeSavedToCache(resp.Response) { + return reader, resp.Response.Header, resp.StatusCode, err } + // now we write to cache and respond at the sime time + fileResp := FileResponse{ + Exists: true, + ETag: resp.Header.Get(ETagHeader), + MimeType: mimeType, + } + return fileResp.CreateCacheReader(reader, client.responseCache, cacheKey), resp.Response.Header, resp.StatusCode, nil + case http.StatusNotFound: if err := client.responseCache.Set(cacheKey, FileResponse{ Exists: false, From da84be235215187dcc5f7daca75917a238410599 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 12 Nov 2022 00:38:21 +0100 Subject: [PATCH 13/18] writeCacheReader: rename all vars to use full redable words --- server/gitea/cache.go | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/server/gitea/cache.go b/server/gitea/cache.go index b1d66f3..b11a370 100644 --- a/server/gitea/cache.go +++ b/server/gitea/cache.go @@ -71,32 +71,33 @@ type BranchTimestamp struct { } type writeCacheReader struct { - rc io.ReadCloser - buff *bytes.Buffer - f *FileResponse - cacheKey string - cache cache.SetGetKey - hasErr bool + originalReader io.ReadCloser + buffer *bytes.Buffer + rileResponse *FileResponse + cacheKey string + cache cache.SetGetKey + hasError bool } func (t *writeCacheReader) Read(p []byte) (n int, err error) { - n, err = t.rc.Read(p) + n, err = t.originalReader.Read(p) if err != nil { - t.hasErr = true + log.Trace().Err(err).Msgf("[cache] original reader for %q has returned an error", t.cacheKey) + t.hasError = true } else if n > 0 { - _, _ = t.buff.Write(p[:n]) + _, _ = t.buffer.Write(p[:n]) } return } func (t *writeCacheReader) Close() error { - if !t.hasErr { - fc := *t.f - fc.Body = t.buff.Bytes() + if !t.hasError { + fc := *t.rileResponse + fc.Body = t.buffer.Bytes() _ = t.cache.Set(t.cacheKey, fc, fileCacheTimeout) } - log.Trace().Msgf("cacheReader for %q saved=%t closed", t.cacheKey, !t.hasErr) - return t.rc.Close() + log.Trace().Msgf("cacheReader for %q saved=%t closed", t.cacheKey, !t.hasError) + return t.originalReader.Close() } func (f FileResponse) CreateCacheReader(r io.ReadCloser, cache cache.SetGetKey, cacheKey string) io.ReadCloser { @@ -106,10 +107,10 @@ func (f FileResponse) CreateCacheReader(r io.ReadCloser, cache cache.SetGetKey, } return &writeCacheReader{ - rc: r, - buff: bytes.NewBuffer(make([]byte, 0)), - f: &f, - cache: cache, - cacheKey: cacheKey, + originalReader: r, + buffer: bytes.NewBuffer(make([]byte, 0)), + rileResponse: &f, + cache: cache, + cacheKey: cacheKey, } } From 1bbb1b4a10807ecab286a3f118253723db8e6cda Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 12 Nov 2022 00:40:18 +0100 Subject: [PATCH 14/18] fix lint issue --- html/error.go | 2 +- server/context/context.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/html/error.go b/html/error.go index 33b24c8..826c42b 100644 --- a/html/error.go +++ b/html/error.go @@ -21,7 +21,7 @@ func ReturnErrorPage(ctx *context.Context, msg string, statusCode int) { msg = strings.ReplaceAll(strings.ReplaceAll(ErrorPage, "%message%", msg), "%status%", http.StatusText(statusCode)) } - ctx.RespWriter.Write([]byte(msg)) + _, _ = ctx.RespWriter.Write([]byte(msg)) } func errorMessage(statusCode int) string { diff --git a/server/context/context.go b/server/context/context.go index 6bb599d..be01df0 100644 --- a/server/context/context.go +++ b/server/context/context.go @@ -39,7 +39,7 @@ func (c *Context) String(raw string, status ...int) { code = status[0] } c.RespWriter.WriteHeader(code) - c.RespWriter.Write([]byte(raw)) + _, _ = c.RespWriter.Write([]byte(raw)) } func (c *Context) IsMethod(m string) bool { From 834e8544f028eade1717e6fa2a719ac6e775d8e1 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 12 Nov 2022 00:43:09 +0100 Subject: [PATCH 15/18] use message --- server/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/handler.go b/server/handler.go index da33702..894cd25 100644 --- a/server/handler.go +++ b/server/handler.go @@ -95,7 +95,7 @@ func Handler(mainDomainSuffix, rawDomain string, // TODO: move into external func to not alert vars indirectly tryBranch := func(log zerolog.Logger, repo, branch string, _path []string, canonicalLink string) bool { if repo == "" { - log.Debug().Msg("tryBranch: repo == ''") + log.Debug().Msg("tryBranch: repo is empty") return false } From c6326c88afea0b64323eb90716de27ac69c19205 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 12 Nov 2022 00:49:47 +0100 Subject: [PATCH 16/18] cache write error does not care what content is stored --- server/gitea/client.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/server/gitea/client.go b/server/gitea/client.go index 47280f0..3d7fd44 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -143,7 +143,7 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str Body: []byte(linkDest), ETag: resp.Header.Get(ETagHeader), }, fileCacheTimeout); err != nil { - log.Error().Err(err).Msg("could not save symlink in cache") + log.Error().Err(err).Msg("[cache] error on cache write") } log.Debug().Msgf("follow symlink from %q to %q", resource, linkDest) @@ -172,7 +172,7 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str Exists: false, ETag: resp.Header.Get(ETagHeader), }, fileCacheTimeout); err != nil { - log.Error().Err(err).Msg("could not save 404 in cache") + log.Error().Err(err).Msg("[cache] error on cache write") } return nil, resp.Response.Header, http.StatusNotFound, ErrorNotFound @@ -189,19 +189,19 @@ func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchNam if stamp, ok := client.responseCache.Get(cacheKey); ok && stamp != nil { branchTimeStamp := stamp.(*BranchTimestamp) if branchTimeStamp.notFound { - log.Trace().Msgf("use cache branch [%s] not found", branchName) + log.Trace().Msgf("[cache] use branch %q not found", branchName) return &BranchTimestamp{}, ErrorNotFound } - log.Trace().Msgf("use cache branch [%s] exist", branchName) + log.Trace().Msgf("[cache] use branch %q exist", branchName) return branchTimeStamp, nil } branch, resp, err := client.sdkClient.GetRepoBranch(repoOwner, repoName, branchName) if err != nil { if resp != nil && resp.StatusCode == http.StatusNotFound { - log.Trace().Msgf("set cache branch [%s] not found", branchName) + log.Trace().Msgf("[cache] set cache branch %q not found", branchName) if err := client.responseCache.Set(cacheKey, &BranchTimestamp{Branch: branchName, notFound: true}, branchExistenceCacheTimeout); err != nil { - log.Error().Err(err).Msgf("error on store of repo branch timestamp [%s/%s@%s]", repoOwner, repoName, branchName) + log.Error().Err(err).Msg("[cache] error on cache write") } return &BranchTimestamp{}, ErrorNotFound } @@ -218,7 +218,7 @@ func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchNam log.Trace().Msgf("set cache branch [%s] exist", branchName) if err := client.responseCache.Set(cacheKey, stamp, branchExistenceCacheTimeout); err != nil { - log.Error().Err(err).Msgf("error on store of repo branch timestamp [%s/%s@%s]", repoOwner, repoName, branchName) + log.Error().Err(err).Msg("[cache] error on cache write") } return stamp, nil } @@ -240,7 +240,7 @@ func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (str branch := repo.DefaultBranch if err := client.responseCache.Set(cacheKey, branch, defaultBranchCacheTimeout); err != nil { - log.Error().Err(err).Msgf("error on store of repo default branch [%s/%s]", repoOwner, repoName) + log.Error().Err(err).Msg("[cache] error on cache write") } return branch, nil } From eeaff6de2c85f60a0c05de9437ad9fcefcf27a74 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 12 Nov 2022 00:54:02 +0100 Subject: [PATCH 17/18] explain why we cache symlinc and not content --- server/gitea/client.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/gitea/client.go b/server/gitea/client.go index 3d7fd44..79e3c04 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -137,6 +137,7 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str } linkDest := strings.TrimSpace(string(linkDestBytes)) + // we store symlink not content to reduce duplicates in cache if err := client.responseCache.Set(cacheKey, FileResponse{ Exists: true, IsSymlink: true, From ce58fa24a0a6d4ef56fade83c436efabb77975f7 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 12 Nov 2022 00:56:30 +0100 Subject: [PATCH 18/18] fix misspell and more log info --- server/gitea/client.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/gitea/client.go b/server/gitea/client.go index 79e3c04..c63ee21 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -252,7 +252,7 @@ func (client *Client) getMimeTypeByExtension(resource string) string { if client.forbiddenMimeTypes[mimeTypeSplit[0]] || mimeType == "" { mimeType = client.defaultMimeType } - log.Trace().Msgf("probe mime: %s", mimeType) + log.Trace().Msgf("probe mime of %q is %q", resource, mimeType) return mimeType } @@ -261,12 +261,12 @@ func shouldRespBeSavedToCache(resp *http.Response) bool { return false } - contentLengRaw := resp.Header.Get(ContentLengthHeader) - if contentLengRaw == "" { + contentLengthRaw := resp.Header.Get(ContentLengthHeader) + if contentLengthRaw == "" { return false } - contentLeng, err := strconv.ParseInt(contentLengRaw, 10, 64) + contentLeng, err := strconv.ParseInt(contentLengthRaw, 10, 64) if err != nil { log.Error().Err(err).Msg("could not parse content length") }