From f5d0dc7447c5dd5a079b518a457d7559015a3fce Mon Sep 17 00:00:00 2001 From: 6543 <6543@noreply.codeberg.org> Date: Sun, 27 Mar 2022 21:54:06 +0200 Subject: [PATCH] Add pipeline (#65) close #54 Co-authored-by: 6543 <6543@obermui.de> Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/65 Reviewed-by: Andreas Shimokawa --- .gitignore | 1 + .woodpecker.yml | 25 +++++++++++++++++++++++++ cmd/certs.go | 5 +++-- cmd/main.go | 8 ++++---- main.go | 6 ++---- server/certificates/acme_account.go | 2 ++ server/certificates/certificates.go | 13 +++++++++---- server/database/setup.go | 16 ++-------------- server/handler.go | 7 ++++--- server/handler_test.go | 3 ++- server/try.go | 4 ++-- server/upstream/helper.go | 4 ++-- 12 files changed, 58 insertions(+), 36 deletions(-) create mode 100644 .woodpecker.yml diff --git a/.gitignore b/.gitignore index 3117f77..dfe69ac 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ key-database.pogreb/ acme-account.json build/ vendor/ +pages diff --git a/.woodpecker.yml b/.woodpecker.yml new file mode 100644 index 0000000..eafb569 --- /dev/null +++ b/.woodpecker.yml @@ -0,0 +1,25 @@ +pipeline: + # use vendor to cache dependencies + vendor: + image: golang + commands: + - go mod vendor + + lint: + image: golangci/golangci-lint:v1.45.2 + commands: + - go version + - go install mvdan.cc/gofumpt@latest + - "[ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }" + - golangci-lint run + + # # TODO: make tests work + # test: + # image: golang + # commands: + # - go test ./... + + build: + image: golang + commands: + - go build diff --git a/cmd/certs.go b/cmd/certs.go index 071234b..83f2ac5 100644 --- a/cmd/certs.go +++ b/cmd/certs.go @@ -2,6 +2,7 @@ package cmd import ( "fmt" + "github.com/akrylysov/pogreb" "github.com/urfave/cli/v2" @@ -12,12 +13,12 @@ var Certs = &cli.Command{ Name: "certs", Usage: "manage certs manually", Subcommands: []*cli.Command{ - &cli.Command{ + { Name: "list", Usage: "list all certificates in the database", Action: listCerts, }, - &cli.Command{ + { Name: "remove", Usage: "remove a certificate from the database", Action: removeCert, diff --git a/cmd/main.go b/cmd/main.go index fb0c26e..6836cee 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -72,14 +72,14 @@ func Serve(ctx *cli.Context) error { keyCache := cache.NewKeyValueCache() challengeCache := cache.NewKeyValueCache() // canonicalDomainCache stores canonical domains - var canonicalDomainCache = cache.NewKeyValueCache() + canonicalDomainCache := cache.NewKeyValueCache() // dnsLookupCache stores DNS lookups for custom domains - var dnsLookupCache = cache.NewKeyValueCache() + dnsLookupCache := cache.NewKeyValueCache() // branchTimestampCache stores branch timestamps for faster cache checking - var branchTimestampCache = cache.NewKeyValueCache() + branchTimestampCache := cache.NewKeyValueCache() // fileResponseCache stores responses from the Gitea server // TODO: make this an MRU cache with a size limit - var fileResponseCache = cache.NewKeyValueCache() + fileResponseCache := cache.NewKeyValueCache() // Create handler based on settings handler := server.Handler(mainDomainSuffix, []byte(rawDomain), diff --git a/main.go b/main.go index 41aba22..205ec93 100644 --- a/main.go +++ b/main.go @@ -9,10 +9,8 @@ import ( "codeberg.org/codeberg/pages/cmd" ) -var ( - // can be changed with -X on compile - version = "dev" -) +// can be changed with -X on compile +var version = "dev" func main() { app := cli.NewApp() diff --git a/server/certificates/acme_account.go b/server/certificates/acme_account.go index 2ee2e80..57f4d44 100644 --- a/server/certificates/acme_account.go +++ b/server/certificates/acme_account.go @@ -19,9 +19,11 @@ var _ registration.User = &AcmeAccount{} func (u *AcmeAccount) GetEmail() string { return u.Email } + func (u AcmeAccount) GetRegistration() *registration.Resource { return u.Registration } + func (u *AcmeAccount) GetPrivateKey() crypto.PrivateKey { return u.Key } diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index 505212f..a13215c 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -40,7 +40,8 @@ func TLSConfig(mainDomainSuffix []byte, giteaRoot, giteaAPIToken, dnsProvider string, acmeUseRateLimits bool, keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey, - certDB database.CertDB) *tls.Config { + certDB database.CertDB, +) *tls.Config { return &tls.Config{ // check DNS name & get certificate from Let's Encrypt GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { @@ -146,8 +147,10 @@ func checkUserLimit(user string) error { return nil } -var acmeClient, mainDomainAcmeClient *lego.Client -var acmeClientCertificateLimitPerUser = map[string]*equalizer.TokenBucket{} +var ( + acmeClient, mainDomainAcmeClient *lego.Client + acmeClientCertificateLimitPerUser = map[string]*equalizer.TokenBucket{} +) // rate limit is 300 / 3 hours, we want 200 / 2 hours but to refill more often, so that's 25 new domains every 15 minutes // TODO: when this is used a lot, we probably have to think of a somewhat better solution? @@ -166,6 +169,7 @@ var _ challenge.Provider = AcmeTLSChallengeProvider{} func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error { return a.challengeCache.Set(domain, keyAuth, 1*time.Hour) } + func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error { a.challengeCache.Remove(domain) return nil @@ -181,6 +185,7 @@ var _ challenge.Provider = AcmeHTTPChallengeProvider{} func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error { return a.challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour) } + func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error { a.challengeCache.Remove(domain + "/" + token) return nil @@ -388,7 +393,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce log.Printf("[FAIL] Error during json.Marshal(myAcmeAccount), waiting for manual restart to avoid rate limits: %s", err) select {} } - err = ioutil.WriteFile(configFile, acmeAccountJSON, 0600) + err = ioutil.WriteFile(configFile, acmeAccountJSON, 0o600) if err != nil { log.Printf("[FAIL] Error during ioutil.WriteFile(\"acme-account.json\"), waiting for manual restart to avoid rate limits: %s", err) select {} diff --git a/server/database/setup.go b/server/database/setup.go index f3cac16..e48b661 100644 --- a/server/database/setup.go +++ b/server/database/setup.go @@ -13,6 +13,8 @@ import ( "github.com/rs/zerolog/log" ) +var _ CertDB = aDB{} + type aDB struct { ctx context.Context cancel context.CancelFunc @@ -76,20 +78,6 @@ func (p aDB) sync() { } } -func (p aDB) compact() { - for { - err := p.intern.Sync() - if err != nil { - log.Err(err).Msg("Syncing cert database failed") - } - select { - case <-p.ctx.Done(): - return - case <-time.After(p.syncInterval): - } - } -} - func New(path string) (CertDB, error) { if path == "" { return nil, fmt.Errorf("path not set") diff --git a/server/handler.go b/server/handler.go index 1aaf476..60f5809 100644 --- a/server/handler.go +++ b/server/handler.go @@ -18,7 +18,8 @@ import ( func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAPIToken string, blacklistedPaths, allowedCorsDomains [][]byte, - dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey) func(ctx *fasthttp.RequestCtx) { + dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey, +) func(ctx *fasthttp.RequestCtx) { return func(ctx *fasthttp.RequestCtx) { log := log.With().Str("Handler", string(ctx.Request.Header.RequestURI())).Logger() @@ -72,14 +73,14 @@ func Handler(mainDomainSuffix, rawDomain []byte, // Prepare request information to Gitea var targetOwner, targetRepo, targetBranch, targetPath string - var targetOptions = &upstream.Options{ + targetOptions := &upstream.Options{ ForbiddenMimeTypes: map[string]struct{}{}, TryIndexPages: true, } // tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will // also disallow search indexing and add a Link header to the canonical URL. - var tryBranch = func(repo string, branch string, path []string, canonicalLink string) bool { + tryBranch := func(repo, branch string, path []string, canonicalLink string) bool { if repo == "" { return false } diff --git a/server/handler_test.go b/server/handler_test.go index 0ec9fcd..34aed7f 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -2,10 +2,11 @@ package server import ( "fmt" - "github.com/valyala/fasthttp" "testing" "time" + "github.com/valyala/fasthttp" + "codeberg.org/codeberg/pages/server/cache" ) diff --git a/server/try.go b/server/try.go index 31cd7f4..4eda5b2 100644 --- a/server/try.go +++ b/server/try.go @@ -19,8 +19,8 @@ func tryUpstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaAPIToken string, - canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey) { - + canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey, +) { // check if a canonical domain exists on a request on MainDomain if bytes.HasSuffix(trimmedHost, mainDomainSuffix) { canonicalDomain, _ := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache) diff --git a/server/upstream/helper.go b/server/upstream/helper.go index b5ee77a..f61d746 100644 --- a/server/upstream/helper.go +++ b/server/upstream/helper.go @@ -27,7 +27,7 @@ func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string, br result.Branch = branch if branch == "" { // Get default branch - var body = make([]byte, 0) + body := make([]byte, 0) // TODO: use header for API key? status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"?access_token="+giteaApiToken, 5*time.Second) if err != nil || status != 200 { @@ -37,7 +37,7 @@ func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string, br result.Branch = fastjson.GetString(body, "default_branch") } - var body = make([]byte, 0) + body := make([]byte, 0) status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"/branches/"+branch+"?access_token="+giteaApiToken, 5*time.Second) if err != nil || status != 200 { return nil