From 69c2532026bb52c374f1d4832aba21c451b98727 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 21 Jul 2022 22:13:18 +0200 Subject: [PATCH 1/5] Add Support to Follow Symlinks --- server/gitea/client.go | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/server/gitea/client.go b/server/gitea/client.go index b918235..6653d40 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -11,7 +11,10 @@ import ( "github.com/valyala/fastjson" ) -const giteaAPIRepos = "/api/v1/repos/" +const ( + giteaAPIRepos = "/api/v1/repos/" + giteaObjectTypeHeader = "X-Gitea-Object-Type" +) var ErrorNotFound = errors.New("not found") @@ -21,6 +24,8 @@ type Client struct { fastClient *fasthttp.Client infoTimeout time.Duration contentTimeout time.Duration + + followSymlinks bool } // TODO: once golang v1.19 is min requirement, we can switch to 'JoinPath()' of 'net/url' package @@ -47,6 +52,8 @@ func NewClient(giteaRoot, giteaAPIToken string) (*Client, error) { infoTimeout: 5 * time.Second, contentTimeout: 10 * time.Second, fastClient: getFastHTTPClient(), + + followSymlinks: true, }, err } @@ -60,7 +67,7 @@ func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource str func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource string) (*fasthttp.Response, error) { url := joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref)) - res, err := client.do(client.contentTimeout, url) + resp, err := client.do(client.contentTimeout, url) if err != nil { return nil, err } @@ -69,13 +76,19 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str return nil, err } - switch res.StatusCode() { + switch resp.StatusCode() { case fasthttp.StatusOK: - return res, nil + if client.followSymlinks && string(resp.Header.Peek(giteaObjectTypeHeader)) == "symlink" { + return client.ServeRawContent(targetOwner, targetRepo, ref, strings.TrimSpace(string(resp.Body()))) + } + + return resp, nil + case fasthttp.StatusNotFound: return nil, ErrorNotFound + default: - return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode()) + return nil, fmt.Errorf("unexpected status code '%d'", resp.StatusCode()) } } From 3ccaa17707eef5fe2e02439a2b1f3e07904d6e33 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Fri, 22 Jul 2022 00:49:09 +0200 Subject: [PATCH 2/5] add integration test --- integration/get_test.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/integration/get_test.go b/integration/get_test.go index 8af40f0..617c886 100644 --- a/integration/get_test.go +++ b/integration/get_test.go @@ -88,6 +88,21 @@ func TestGetNotFound(t *testing.T) { assert.EqualValues(t, 37, getSize(resp.Body)) } +func TestFollowSymlink(t *testing.T) { + log.Printf("=== TestFollowSymlink ===\n") + + resp, err := getTestHTTPSClient().Get("https://6543.localhost.mock.directory:4430/tests_for_pages-server/@main/link") + assert.NoError(t, err) + if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) { + t.FailNow() + } + assert.EqualValues(t, "application/octet-stream", resp.Header.Get("Content-Type")) + assert.EqualValues(t, "4", resp.Header.Get("Content-Length")) + body := getBytes(resp.Body) + assert.EqualValues(t, 4, len(body)) + assert.EqualValues(t, "abc\n", string(body)) +} + func getTestHTTPSClient() *http.Client { cookieJar, _ := cookiejar.New(nil) return &http.Client{ @@ -101,6 +116,12 @@ func getTestHTTPSClient() *http.Client { } } +func getBytes(stream io.Reader) []byte { + buf := new(bytes.Buffer) + _, _ = buf.ReadFrom(stream) + return buf.Bytes() +} + func getSize(stream io.Reader) int { buf := new(bytes.Buffer) _, _ = buf.ReadFrom(stream) From 7a7cbac171c1f04fcd3a8194675710dd23d4f988 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Fri, 22 Jul 2022 01:00:48 +0200 Subject: [PATCH 3/5] more debug log --- server/gitea/client.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/gitea/client.go b/server/gitea/client.go index 6653d40..9c45e66 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/rs/zerolog/log" "github.com/valyala/fasthttp" "github.com/valyala/fastjson" ) @@ -79,7 +80,9 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str switch resp.StatusCode() { case fasthttp.StatusOK: if client.followSymlinks && string(resp.Header.Peek(giteaObjectTypeHeader)) == "symlink" { - return client.ServeRawContent(targetOwner, targetRepo, ref, strings.TrimSpace(string(resp.Body()))) + linkDest := strings.TrimSpace(string(resp.Body())) + log.Debug().Msgf("follow symlink from '%s' to '%s'", resource, linkDest) + return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest) } return resp, nil From aa83b307db324dd195c4053a56316e127d03f956 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 24 Jul 2022 17:43:34 +0200 Subject: [PATCH 4/5] add TestLFSSupport --- integration/get_test.go | 13 +++++++++++++ server/gitea/client.go | 11 +++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/integration/get_test.go b/integration/get_test.go index 617c886..332096f 100644 --- a/integration/get_test.go +++ b/integration/get_test.go @@ -103,6 +103,19 @@ func TestFollowSymlink(t *testing.T) { assert.EqualValues(t, "abc\n", string(body)) } +func TestLFSSupport(t *testing.T) { + log.Printf("=== TestLFSSupport ===\n") + + resp, err := getTestHTTPSClient().Get("https://6543.localhost.mock.directory:4430/tests_for_pages-server/@main/lfs.txt") + assert.NoError(t, err) + if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) { + t.FailNow() + } + body := strings.TrimSpace(string(getBytes(resp.Body))) + assert.EqualValues(t, 12, len(body)) + assert.EqualValues(t, "actual value", body) +} + func getTestHTTPSClient() *http.Client { cookieJar, _ := cookiejar.New(nil) return &http.Client{ diff --git a/server/gitea/client.go b/server/gitea/client.go index 9c45e66..747bd6a 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -27,6 +27,7 @@ type Client struct { contentTimeout time.Duration followSymlinks bool + supportLFS bool } // TODO: once golang v1.19 is min requirement, we can switch to 'JoinPath()' of 'net/url' package @@ -55,6 +56,7 @@ func NewClient(giteaRoot, giteaAPIToken string) (*Client, error) { fastClient: getFastHTTPClient(), followSymlinks: true, + supportLFS: true, }, err } @@ -67,8 +69,13 @@ func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource str } func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource string) (*fasthttp.Response, error) { - url := joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref)) - resp, err := client.do(client.contentTimeout, url) + var apiURL string + if client.supportLFS { + apiURL = joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "media", resource+"?ref="+url.QueryEscape(ref)) + } else { + apiURL = joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref)) + } + resp, err := client.do(client.contentTimeout, apiURL) if err != nil { return nil, err } From b1b23ce2ec327e3b5fcf473d36349522176de982 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 24 Jul 2022 17:52:07 +0200 Subject: [PATCH 5/5] make it conf via arg or env --- cmd/flags.go | 11 +++++++++++ cmd/main.go | 2 +- integration/get_test.go | 1 + server/gitea/client.go | 6 +++--- server/handler_test.go | 2 +- 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/cmd/flags.go b/cmd/flags.go index 78040be..3c9aef6 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -69,6 +69,17 @@ var ServeFlags = []cli.Flag{ // TODO: desc EnvVars: []string{"ENABLE_HTTP_SERVER"}, }, + // Server Options + &cli.BoolFlag{ + Name: "enable-lfs-support", + Usage: "enable lfs support, require gitea v1.17.0 as backend", + EnvVars: []string{"ENABLE_LFS_SUPPORT"}, + }, + &cli.BoolFlag{ + Name: "enable-symlink-support", + Usage: "follow symlinks if enabled, require gitea v1.18.0 as backend", + EnvVars: []string{"ENABLE_SYMLINK_SUPPORT"}, + }, // ACME &cli.StringFlag{ diff --git a/cmd/main.go b/cmd/main.go index 80fb666..a76b7db 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -82,7 +82,7 @@ func Serve(ctx *cli.Context) error { // TODO: make this an MRU cache with a size limit fileResponseCache := cache.NewKeyValueCache() - giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken) + giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken, ctx.Bool("enable-symlink-support"), ctx.Bool("enable-lfs-support")) if err != nil { return fmt.Errorf("could not create new gitea client: %v", err) } diff --git a/integration/get_test.go b/integration/get_test.go index 332096f..ccb8b6b 100644 --- a/integration/get_test.go +++ b/integration/get_test.go @@ -9,6 +9,7 @@ import ( "io" "net/http" "net/http/cookiejar" + "strings" "testing" "github.com/rs/zerolog/log" diff --git a/server/gitea/client.go b/server/gitea/client.go index 747bd6a..56e1221 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -44,7 +44,7 @@ func joinURL(baseURL string, paths ...string) string { return baseURL + "/" + strings.Join(p, "/") } -func NewClient(giteaRoot, giteaAPIToken string) (*Client, error) { +func NewClient(giteaRoot, giteaAPIToken string, followSymlinks, supportLFS bool) (*Client, error) { rootURL, err := url.Parse(giteaRoot) giteaRoot = strings.Trim(rootURL.String(), "/") @@ -55,8 +55,8 @@ func NewClient(giteaRoot, giteaAPIToken string) (*Client, error) { contentTimeout: 10 * time.Second, fastClient: getFastHTTPClient(), - followSymlinks: true, - supportLFS: true, + followSymlinks: followSymlinks, + supportLFS: supportLFS, }, err } diff --git a/server/handler_test.go b/server/handler_test.go index 23d9af5..f9a721a 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -13,7 +13,7 @@ import ( func TestHandlerPerformance(t *testing.T) { giteaRoot := "https://codeberg.org" - giteaClient, _ := gitea.NewClient(giteaRoot, "") + giteaClient, _ := gitea.NewClient(giteaRoot, "", false, false) testHandler := Handler( []byte("codeberg.page"), []byte("raw.codeberg.org"), giteaClient,