Merge branch 'move-more-logic-into-client' into std-http

cmd/flags.go

@@ -69,6 +69,17 @@ var ServeFlags = []cli.Flag{
 		// TODO: desc
 		EnvVars: []string{"ENABLE_HTTP_SERVER"},
 	},
+	// Server Options
+	&cli.BoolFlag{
+		Name:    "enable-lfs-support",
+		Usage:   "enable lfs support, require gitea v1.17.0 as backend",
+		EnvVars: []string{"ENABLE_LFS_SUPPORT"},
+	},
+	&cli.BoolFlag{
+		Name:    "enable-symlink-support",
+		Usage:   "follow symlinks if enabled, require gitea v1.18.0 as backend",
+		EnvVars: []string{"ENABLE_SYMLINK_SUPPORT"},
+	},
 
 	// ACME
 	&cli.StringFlag{

cmd/main.go

@@ -82,7 +82,7 @@ func Serve(ctx *cli.Context) error {
 	// TODO: make this an MRU cache with a size limit
 	fileResponseCache := cache.NewKeyValueCache()
 
-	giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken, fileResponseCache)
+	giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken, fileResponseCache, ctx.Bool("enable-symlink-support"), ctx.Bool("enable-lfs-support"))
 	if err != nil {
 		return fmt.Errorf("could not create new gitea client: %v", err)
 	}

integration/get_test.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"net/http"
 	"net/http/cookiejar"
+	"strings"
 	"testing"
 
 	"github.com/rs/zerolog/log"
@@ -88,6 +89,34 @@ func TestGetNotFound(t *testing.T) {
 	assert.EqualValues(t, 37, getSize(resp.Body))
 }
 
+func TestFollowSymlink(t *testing.T) {
+	log.Printf("=== TestFollowSymlink ===\n")
+
+	resp, err := getTestHTTPSClient().Get("https://6543.localhost.mock.directory:4430/tests_for_pages-server/@main/link")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, "application/octet-stream", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "4", resp.Header.Get("Content-Length"))
+	body := getBytes(resp.Body)
+	assert.EqualValues(t, 4, len(body))
+	assert.EqualValues(t, "abc\n", string(body))
+}
+
+func TestLFSSupport(t *testing.T) {
+	log.Printf("=== TestLFSSupport ===\n")
+
+	resp, err := getTestHTTPSClient().Get("https://6543.localhost.mock.directory:4430/tests_for_pages-server/@main/lfs.txt")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+		t.FailNow()
+	}
+	body := strings.TrimSpace(string(getBytes(resp.Body)))
+	assert.EqualValues(t, 12, len(body))
+	assert.EqualValues(t, "actual value", body)
+}
+
 func getTestHTTPSClient() *http.Client {
 	cookieJar, _ := cookiejar.New(nil)
 	return &http.Client{
@@ -101,6 +130,12 @@ func getTestHTTPSClient() *http.Client {
 	}
 }
 
+func getBytes(stream io.Reader) []byte {
+	buf := new(bytes.Buffer)
+	_, _ = buf.ReadFrom(stream)
+	return buf.Bytes()
+}
+
 func getSize(stream io.Reader) int {
 	buf := new(bytes.Buffer)
 	_, _ = buf.ReadFrom(stream)

server/gitea/client_fasthttp.go

@@ -8,13 +8,17 @@ import (
 	"strings"
 	"time"
 
+	"github.com/rs/zerolog/log"
 	"github.com/valyala/fasthttp"
 	"github.com/valyala/fastjson"
 
 	"codeberg.org/codeberg/pages/server/cache"
 )
 
-const giteaAPIRepos = "/api/v1/repos/"
+const (
+	giteaAPIRepos         = "/api/v1/repos/"
+	giteaObjectTypeHeader = "X-Gitea-Object-Type"
+)
 
 type Client struct {
 	giteaRoot         string
@@ -23,9 +27,12 @@ type Client struct {
 	contentTimeout    time.Duration
 	fastClient        *fasthttp.Client
 	fileResponseCache cache.SetGetKey
+
+	followSymlinks bool
+	supportLFS     bool
 }
 
-func NewClient(giteaRoot, giteaAPIToken string, fileResponseCache cache.SetGetKey) (*Client, error) {
+func NewClient(giteaRoot, giteaAPIToken string, fileResponseCache cache.SetGetKey, followSymlinks, supportLFS bool) (*Client, error) {
 	rootURL, err := url.Parse(giteaRoot)
 	giteaRoot = strings.Trim(rootURL.String(), "/")
 
@@ -36,6 +43,9 @@ func NewClient(giteaRoot, giteaAPIToken string, fileResponseCache cache.SetGetKe
 		contentTimeout:    10 * time.Second,
 		fastClient:        getFastHTTPClient(),
 		fileResponseCache: fileResponseCache,
+
+		followSymlinks: followSymlinks,
+		supportLFS:     supportLFS,
 	}, err
 }
 
@@ -48,19 +58,32 @@ func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource str
 }
 
 func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource string) (*fasthttp.Response, error) {
-	url := joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref))
+	var apiURL string
-	res, err := client.do(client.contentTimeout, url)
+	if client.supportLFS {
+		apiURL = joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "media", resource+"?ref="+url.QueryEscape(ref))
+	} else {
+		apiURL = joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref))
+	}
+	resp, err := client.do(client.contentTimeout, apiURL)
 	if err != nil {
 		return nil, err
 	}
 
-	switch res.StatusCode() {
+	switch resp.StatusCode() {
 	case fasthttp.StatusOK:
-		return res, nil
+		if client.followSymlinks && string(resp.Header.Peek(giteaObjectTypeHeader)) == "symlink" {
+			linkDest := strings.TrimSpace(string(resp.Body()))
+			log.Debug().Msgf("follow symlink from '%s' to '%s'", resource, linkDest)
+			return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest)
+		}
+
+		return resp, nil
+
 	case fasthttp.StatusNotFound:
 		return nil, ErrorNotFound
+
 	default:
-		return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
+		return nil, fmt.Errorf("unexpected status code '%d'", resp.StatusCode())
 	}
 }
 

server/gitea/client_std.go

@@ -19,9 +19,12 @@ import (
 type Client struct {
 	sdkClient         *gitea.Client
 	fileResponseCache cache.SetGetKey
+
+	followSymlinks bool
+	supportLFS     bool
 }
 
-func NewClient(giteaRoot, giteaAPIToken string, fileResponseCache cache.SetGetKey) (*Client, error) {
+func NewClient(giteaRoot, giteaAPIToken string, fileResponseCache cache.SetGetKey, followSymlinks, supportLFS bool) (*Client, error) {
 	rootURL, err := url.Parse(giteaRoot)
 	giteaRoot = strings.Trim(rootURL.String(), "/")
 
@@ -35,6 +38,14 @@ func NewClient(giteaRoot, giteaAPIToken string, fileResponseCache cache.SetGetKe
 }
 
 func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource string) ([]byte, error) {
+	// var apiURL string
+	// if client.supportLFS {
+	// 	apiURL = joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "media", resource+"?ref="+url.QueryEscape(ref))
+	// } else {
+	// 	apiURL = joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref))
+	// }
+	// TODO: sdk client support media api!!!
+
 	rawBytes, resp, err := client.sdkClient.GetFile(targetOwner, targetRepo, ref, resource)
 	if err != nil {
 		return nil, err

server/handler_test.go

@@ -15,7 +15,7 @@ import (
 
 func TestHandlerPerformance(t *testing.T) {
 	giteaRoot := "https://codeberg.org"
-	giteaClient, _ := gitea.NewClient(giteaRoot, "", cache.NewKeyValueCache())
+	giteaClient, _ := gitea.NewClient(giteaRoot, "", cache.NewKeyValueCache(), false, false)
 	testHandler := Handler(
 		[]byte("codeberg.page"), []byte("raw.codeberg.org"),
 		giteaClient,