Add Support to Follow Symlinks and LFS (#114)

close #79
close #80
close #91

Co-authored-by: 6543 <6543@obermui.de>
Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/114

cmd/flags.go

@@ -63,6 +63,19 @@ var ServeFlags = []cli.Flag{
 		// TODO: desc
 		EnvVars: []string{"ENABLE_HTTP_SERVER"},
 	},
+	// Server Options
+	&cli.BoolFlag{
+		Name:    "enable-lfs-support",
+		Usage:   "enable lfs support, require gitea v1.17.0 as backend",
+		EnvVars: []string{"ENABLE_LFS_SUPPORT"},
+		Value:   true,
+	},
+	&cli.BoolFlag{
+		Name:    "enable-symlink-support",
+		Usage:   "follow symlinks if enabled, require gitea v1.18.0 as backend",
+		EnvVars: []string{"ENABLE_SYMLINK_SUPPORT"},
+		Value:   true,
+	},
 	&cli.StringFlag{
 		Name:    "log-level",
 		Value:   "warn",

cmd/main.go

@@ -85,7 +85,7 @@ func Serve(ctx *cli.Context) error {
 	// TODO: make this an MRU cache with a size limit
 	fileResponseCache := cache.NewKeyValueCache()
 
-	giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken)
+	giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken, ctx.Bool("enable-symlink-support"), ctx.Bool("enable-lfs-support"))
 	if err != nil {
 		return fmt.Errorf("could not create new gitea client: %v", err)
 	}

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/go-acme/lego/v4 v4.5.3
 	github.com/joho/godotenv v1.4.0
 	github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad
+	github.com/rs/zerolog v1.27.0
 	github.com/stretchr/testify v1.7.0
 	github.com/urfave/cli/v2 v2.3.0
 	github.com/valyala/fasthttp v1.31.0
@@ -92,7 +93,6 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/pquerna/otp v1.3.0 // indirect
-	github.com/rs/zerolog v1.27.0 // indirect
 	github.com/russross/blackfriday/v2 v2.0.1 // indirect
 	github.com/sacloud/libsacloud v1.36.2 // indirect
 	github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210127161313-bd30bebeac4f // indirect

go.sum

@@ -327,7 +327,6 @@ github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
 github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
-github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
@@ -671,8 +670,6 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA=
-golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 h1:foEbQz/B0Oz6YIqu/69kfXPYeFQAuuMYFkjaqXzl5Wo=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=

integration/get_test.go

@@ -10,6 +10,7 @@ import (
 	"log"
 	"net/http"
 	"net/http/cookiejar"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -88,6 +89,34 @@ func TestGetNotFound(t *testing.T) {
 	assert.EqualValues(t, 37, getSize(resp.Body))
 }
 
+func TestFollowSymlink(t *testing.T) {
+	log.Printf("=== TestFollowSymlink ===\n")
+
+	resp, err := getTestHTTPSClient().Get("https://6543.localhost.mock.directory:4430/tests_for_pages-server/@main/link")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, "application/octet-stream", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "4", resp.Header.Get("Content-Length"))
+	body := getBytes(resp.Body)
+	assert.EqualValues(t, 4, len(body))
+	assert.EqualValues(t, "abc\n", string(body))
+}
+
+func TestLFSSupport(t *testing.T) {
+	log.Printf("=== TestLFSSupport ===\n")
+
+	resp, err := getTestHTTPSClient().Get("https://6543.localhost.mock.directory:4430/tests_for_pages-server/@main/lfs.txt")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+		t.FailNow()
+	}
+	body := strings.TrimSpace(string(getBytes(resp.Body)))
+	assert.EqualValues(t, 12, len(body))
+	assert.EqualValues(t, "actual value", body)
+}
+
 func getTestHTTPSClient() *http.Client {
 	cookieJar, _ := cookiejar.New(nil)
 	return &http.Client{
@@ -101,6 +130,12 @@ func getTestHTTPSClient() *http.Client {
 	}
 }
 
+func getBytes(stream io.Reader) []byte {
+	buf := new(bytes.Buffer)
+	_, _ = buf.ReadFrom(stream)
+	return buf.Bytes()
+}
+
 func getSize(stream io.Reader) int {
 	buf := new(bytes.Buffer)
 	_, _ = buf.ReadFrom(stream)

server/gitea/cache.go

@@ -0,0 +1,12 @@
+package gitea
+
+type FileResponse struct {
+	Exists   bool
+	ETag     []byte
+	MimeType string
+	Body     []byte
+}
+
+func (f FileResponse) IsEmpty() bool {
+	return len(f.Body) != 0
+}

server/gitea/client.go

@@ -7,11 +7,15 @@ import (
 	"strings"
 	"time"
 
+	"github.com/rs/zerolog/log"
 	"github.com/valyala/fasthttp"
 	"github.com/valyala/fastjson"
 )
 
-const giteaAPIRepos = "/api/v1/repos/"
+const (
+	giteaAPIRepos         = "/api/v1/repos/"
+	giteaObjectTypeHeader = "X-Gitea-Object-Type"
+)
 
 var ErrorNotFound = errors.New("not found")
 
@@ -21,13 +25,9 @@ type Client struct {
 	fastClient     *fasthttp.Client
 	infoTimeout    time.Duration
 	contentTimeout time.Duration
-}
 
-type FileResponse struct {
+	followSymlinks bool
-	Exists   bool
+	supportLFS     bool
-	ETag     []byte
-	MimeType string
-	Body     []byte
 }
 
 // TODO: once golang v1.19 is min requirement, we can switch to 'JoinPath()' of 'net/url' package
@@ -44,9 +44,7 @@ func joinURL(baseURL string, paths ...string) string {
 	return baseURL + "/" + strings.Join(p, "/")
 }
 
-func (f FileResponse) IsEmpty() bool { return len(f.Body) != 0 }
+func NewClient(giteaRoot, giteaAPIToken string, followSymlinks, supportLFS bool) (*Client, error) {
-
-func NewClient(giteaRoot, giteaAPIToken string) (*Client, error) {
 	rootURL, err := url.Parse(giteaRoot)
 	giteaRoot = strings.Trim(rootURL.String(), "/")
 
@@ -56,29 +54,28 @@ func NewClient(giteaRoot, giteaAPIToken string) (*Client, error) {
 		infoTimeout:    5 * time.Second,
 		contentTimeout: 10 * time.Second,
 		fastClient:     getFastHTTPClient(),
+
+		followSymlinks: followSymlinks,
+		supportLFS:     supportLFS,
 	}, err
 }
 
 func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource string) ([]byte, error) {
-	url := joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref))
+	resp, err := client.ServeRawContent(targetOwner, targetRepo, ref, resource)
-	res, err := client.do(client.contentTimeout, url)
 	if err != nil {
 		return nil, err
 	}
-
+	return resp.Body(), nil
-	switch res.StatusCode() {
-	case fasthttp.StatusOK:
-		return res.Body(), nil
-	case fasthttp.StatusNotFound:
-		return nil, ErrorNotFound
-	default:
-		return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
-	}
 }
 
-func (client *Client) ServeRawContent(uri string) (*fasthttp.Response, error) {
+func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource string) (*fasthttp.Response, error) {
-	url := joinURL(client.giteaRoot, giteaAPIRepos, uri)
+	var apiURL string
-	res, err := client.do(client.contentTimeout, url)
+	if client.supportLFS {
+		apiURL = joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "media", resource+"?ref="+url.QueryEscape(ref))
+	} else {
+		apiURL = joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref))
+	}
+	resp, err := client.do(client.contentTimeout, apiURL)
 	if err != nil {
 		return nil, err
 	}
@@ -87,13 +84,24 @@ func (client *Client) ServeRawContent(uri string) (*fasthttp.Response, error) {
 		return nil, err
 	}
 
-	switch res.StatusCode() {
+	switch resp.StatusCode() {
 	case fasthttp.StatusOK:
-		return res, nil
+		objType := string(resp.Header.Peek(giteaObjectTypeHeader))
+		log.Trace().Msgf("server raw content object: %s", objType)
+		if client.followSymlinks && objType == "symlink" {
+			// TODO: limit to 1000 chars if we switched to std
+			linkDest := strings.TrimSpace(string(resp.Body()))
+			log.Debug().Msgf("follow symlink from '%s' to '%s'", resource, linkDest)
+			return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest)
+		}
+
+		return resp, nil
+
 	case fasthttp.StatusNotFound:
 		return nil, ErrorNotFound
+
 	default:
-		return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
+		return nil, fmt.Errorf("unexpected status code '%d'", resp.StatusCode())
 	}
 }
 

server/handler_test.go

@@ -13,7 +13,7 @@ import (
 
 func TestHandlerPerformance(t *testing.T) {
 	giteaRoot := "https://codeberg.org"
-	giteaClient, _ := gitea.NewClient(giteaRoot, "")
+	giteaClient, _ := gitea.NewClient(giteaRoot, "", false, false)
 	testHandler := Handler(
 		[]byte("codeberg.page"), []byte("raw.codeberg.org"),
 		giteaClient,

server/upstream/helper.go

@@ -67,6 +67,10 @@ func (o *Options) generateUri() string {
 	return path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath)
 }
 
+func (o *Options) generateUriClientArgs() (targetOwner, targetRepo, ref, resource string) {
+	return o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath
+}
+
 func (o *Options) timestamp() string {
 	return strconv.FormatInt(o.BranchTimestamp.Unix(), 10)
 }

server/upstream/upstream.go

@@ -83,7 +83,7 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
 	if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() {
 		cachedResponse = cachedValue.(gitea.FileResponse)
 	} else {
-		res, err = giteaClient.ServeRawContent(uri)
+		res, err = giteaClient.ServeRawContent(o.generateUriClientArgs())
 	}
 	log.Debug().Msg("Aquisting")