From 52a3b48016fa1b650bc6988fcd76f56d356ff8b1 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 9 Feb 2023 17:52:30 +0100 Subject: [PATCH] next --- Dockerfile | 4 +- Justfile | 14 +++---- cmd/certs.go | 23 ++++++----- server/certificates/certificates.go | 54 +++++++++++-------------- server/database/interface.go | 63 +++++++++++++++++++++++++---- server/database/migrate.go | 1 + server/database/mock.go | 5 +-- server/database/pogreb.go | 29 ++++++++++++- server/database/xorm.go | 55 +++++++++++++++++++------ 9 files changed, 174 insertions(+), 74 deletions(-) create mode 100644 server/database/migrate.go diff --git a/Dockerfile b/Dockerfile index 904d6f4..380c373 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ -FROM golang:alpine as build +FROM techknowlogick/xgo as build WORKDIR /workspace RUN apk add ca-certificates COPY . . -RUN CGO_ENABLED=0 go build . +RUN CGO_ENABLED=1 go build -tags 'sqlite sqlite_unlock_notify netgo' -ldflags '-s -w -extldflags "-static" -linkmode external' . FROM scratch COPY --from=build /workspace/pages /pages diff --git a/Justfile b/Justfile index 7d72fe8..579a604 100644 --- a/Justfile +++ b/Justfile @@ -7,13 +7,13 @@ dev: export RAW_DOMAIN=raw.localhost.mock.directory export PORT=4430 export LOG_LEVEL=trace - go run . + go run -tags 'sqlite sqlite_unlock_notify -ldflags '-s -w -extldflags "-static" -linkmode external' netgo' . build: - CGO_ENABLED=0 go build -ldflags '-s -w' -v -o build/codeberg-pages-server ./ + CGO_ENABLED=1 go build -tags 'sqlite sqlite_unlock_notify netgo' -ldflags '-s -w -extldflags "-static" -linkmode external' -v -o build/codeberg-pages-server ./ build-tag VERSION: - CGO_ENABLED=0 go build -ldflags '-s -w -X "codeberg.org/codeberg/pages/server/version.Version={{VERSION}}"' -v -o build/codeberg-pages-server ./ + CGO_ENABLED=1 go build -tags 'sqlite sqlite_unlock_notify netgo' '-ldflags '-s -w -X "codeberg.org/codeberg/pages/server/version.Version={{VERSION}}" -extldflags "-static" -linkmode external' -v -o build/codeberg-pages-server ./ lint: tool-golangci tool-gofumpt [ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }; \ @@ -38,13 +38,13 @@ tool-gofumpt: fi test: - go test -race codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/ + go test -race -tags 'sqlite sqlite_unlock_notify netgo' codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/ test-run TEST: - go test -race -run "^{{TEST}}$" codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/ + go test -race -tags 'sqlite sqlite_unlock_notify netgo' -run "^{{TEST}}$" codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/ integration: - go test -race -tags integration codeberg.org/codeberg/pages/integration/... + go test -race -tags 'integration sqlite sqlite_unlock_notify netgo' codeberg.org/codeberg/pages/integration/... integration-run TEST: - go test -race -tags integration -run "^{{TEST}}$" codeberg.org/codeberg/pages/integration/... + go test -race -tags 'integration sqlite sqlite_unlock_notify netgo' -run "^{{TEST}}$" codeberg.org/codeberg/pages/integration/... diff --git a/cmd/certs.go b/cmd/certs.go index ef82002..46a9c96 100644 --- a/cmd/certs.go +++ b/cmd/certs.go @@ -2,8 +2,8 @@ package cmd import ( "fmt" + "time" - "github.com/akrylysov/pogreb" "github.com/urfave/cli/v2" "codeberg.org/codeberg/pages/server/database" @@ -52,15 +52,20 @@ func listCerts(ctx *cli.Context) error { return fmt.Errorf("could not create database: %v", err) } - items := keyDatabase.Items() - for domain, _, err := items.Next(); err != pogreb.ErrIterationDone; domain, _, err = items.Next() { - if err != nil { - return err + items, err := keyDatabase.Items(0, 0) + if err != nil { + return err + } + + fmt.Printf("Name\tDomain\tValidTill\n\n") + for _, cert := range items { + if cert.Name[0] == '.' { + cert.Name = "*" + cert.Name } - if domain[0] == '.' { - fmt.Printf("*") - } - fmt.Printf("%s\n", domain) + fmt.Printf("%s\t%s\t%s\n", + cert.Name, + cert.Domain, + time.Unix(cert.ValidTill, 0).Format(time.RFC3339)) } return nil } diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index 1aa90a0..3096c3c 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -1,14 +1,12 @@ package certificates import ( - "bytes" "context" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" "crypto/tls" "crypto/x509" - "encoding/gob" "encoding/json" "errors" "fmt" @@ -478,41 +476,35 @@ func SetupCertificates(mainDomainSuffix, dnsProvider string, acmeConfig *lego.Co func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffix, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) { for { - // clean up expired certs - now := time.Now() + // delete expired certs that will be invalid until next clean up + threshold := time.Now().Add(-interval) expiredCertCount := 0 - keyDatabaseIterator := certDB.Items() - key, resBytes, err := keyDatabaseIterator.Next() - for err == nil { - if !strings.EqualFold(string(key), mainDomainSuffix) { - resGob := bytes.NewBuffer(resBytes) - resDec := gob.NewDecoder(resGob) - res := &certificate.Resource{} - err = resDec.Decode(res) - if err != nil { - panic(err) - } - tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate) - if err != nil || tlsCertificates[0].NotAfter.Before(now) { - err := certDB.Delete(string(key)) - if err != nil { - log.Error().Err(err).Msgf("Deleting expired certificate for %q failed", string(key)) - } else { - expiredCertCount++ + certs, err := certDB.Items(0, 0) + if err != nil { + log.Error().Err(err).Msg("could not get certs from list") + } else { + for _, cert := range certs { + if !strings.EqualFold(cert.Name, mainDomainSuffix) { + if time.Unix(cert.ValidTill, 0).Before(threshold) { + err := certDB.Delete(cert.Name) + if err != nil { + log.Error().Err(err).Msgf("Deleting expired certificate for %q failed", cert.Name) + } else { + expiredCertCount++ + } } } } - key, resBytes, err = keyDatabaseIterator.Next() - } - log.Debug().Msgf("Removed %d expired certificates from the database", expiredCertCount) + log.Debug().Msgf("Removed %d expired certificates from the database", expiredCertCount) - // compact the database - msg, err := certDB.Compact() - if err != nil { - log.Error().Err(err).Msg("Compacting key database failed") - } else { - log.Debug().Msgf("Compacted key database: %s", msg) + // compact the database + msg, err := certDB.Compact() + if err != nil { + log.Error().Err(err).Msg("Compacting key database failed") + } else { + log.Debug().Msgf("Compacted key database: %s", msg) + } } // update main cert diff --git a/server/database/interface.go b/server/database/interface.go index 84adc19..15d5de6 100644 --- a/server/database/interface.go +++ b/server/database/interface.go @@ -1,8 +1,11 @@ package database import ( - "github.com/akrylysov/pogreb" + "fmt" + + "github.com/go-acme/lego/v4/certcrypto" "github.com/go-acme/lego/v4/certificate" + "github.com/rs/zerolog/log" ) type CertDB interface { @@ -10,14 +13,60 @@ type CertDB interface { Put(name string, cert *certificate.Resource) error Get(name string) (*certificate.Resource, error) Delete(key string) error + Items(page, pageSize int) ([]*Cert, error) + // Compact deprecated // TODO: remove in next version Compact() (string, error) - Items() *pogreb.ItemIterator } type Cert struct { - Domain string `xorm:"pk NOT NULL"` - Created int64 `xorm:"created NOT NULL DEFAULT 0"` - Updated int64 `xorm:"updated NOT NULL DEFAULT 0"` - ValidTill int64 `xorm:"NOT NULL DEFAULT 0"` - Raw []byte `xorm:"NOT NULL"` + Name string `xorm:"pk NOT NULL 'name'"` + Domain string `xorm:" NOT NULL UNIQUE 'domain'"` // TODO: check: is name always same as domain? + Created int64 `xorm:"created NOT NULL DEFAULT 0 'created'"` + Updated int64 `xorm:"updated NOT NULL DEFAULT 0 'updated'"` + ValidTill int64 `xorm:" NOT NULL DEFAULT 0 'valid_till'"` + // certificate.Resource + certURL string `xorm:"'cert_url'"` + certStableURL string `xorm:"'cert_stable_url''"` + privateKey []byte `xorm:"'private_key'"` + certificate []byte `xorm:"'certificate'"` + issuerCertificate []byte `xorm:"'issuer_certificate'"` // TODO: dedup ? + csr []byte `xorm:"'csr'"` +} + +func (c Cert) Raw() *certificate.Resource { + return &certificate.Resource{ + Domain: c.Domain, + CertURL: c.certURL, + CertStableURL: c.certStableURL, + PrivateKey: c.privateKey, + Certificate: c.certificate, + IssuerCertificate: c.issuerCertificate, + CSR: c.csr, + } +} + +func toCert(name string, c *certificate.Resource) (*Cert, error) { + tlsCertificates, err := certcrypto.ParsePEMBundle(c.Certificate) + if err != nil { + return nil, err + } + if len(tlsCertificates) != 1 || tlsCertificates[0] == nil { + err := fmt.Errorf("parsed cert resource has no or more than one cert") + log.Error().Err(err).Str("name", name).Msgf("cert: %v", c) + return nil, err + } + validTill := tlsCertificates[0].NotAfter.Unix() + + return &Cert{ + Name: name, + Domain: c.Domain, + ValidTill: validTill, + + certURL: c.CertURL, + certStableURL: c.CertStableURL, + privateKey: c.PrivateKey, + certificate: c.Certificate, + issuerCertificate: c.IssuerCertificate, + csr: c.CSR, + }, nil } diff --git a/server/database/migrate.go b/server/database/migrate.go new file mode 100644 index 0000000..636bab8 --- /dev/null +++ b/server/database/migrate.go @@ -0,0 +1 @@ +package database diff --git a/server/database/mock.go b/server/database/mock.go index dfe2316..5b36bec 100644 --- a/server/database/mock.go +++ b/server/database/mock.go @@ -5,7 +5,6 @@ import ( "time" "github.com/OrlovEvgeny/go-mcache" - "github.com/akrylysov/pogreb" "github.com/go-acme/lego/v4/certificate" ) @@ -43,8 +42,8 @@ func (p tmpDB) Compact() (string, error) { return "Truncate done", nil } -func (p tmpDB) Items() *pogreb.ItemIterator { - panic("ItemIterator not implemented for tmpDB") +func (p tmpDB) Items(page, pageSize int) ([]*Cert, error) { + panic("Items not implemented for tmpDB") } func NewTmpDB() (CertDB, error) { diff --git a/server/database/pogreb.go b/server/database/pogreb.go index eeea9cc..9a53faf 100644 --- a/server/database/pogreb.go +++ b/server/database/pogreb.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/gob" + "errors" "fmt" "time" @@ -62,8 +63,32 @@ func (p aDB) Compact() (string, error) { return fmt.Sprintf("%+v", result), nil } -func (p aDB) Items() *pogreb.ItemIterator { - return p.intern.Items() +func (p aDB) Items(_, _ int) ([]*Cert, error) { + items := make([]*Cert, 0, p.intern.Count()) + iterator := p.intern.Items() + for { + key, resBytes, err := iterator.Next() + if err != nil { + if errors.Is(err, pogreb.ErrIterationDone) { + break + } + return nil, err + } + + res := &certificate.Resource{} + if err := gob.NewDecoder(bytes.NewBuffer(resBytes)).Decode(res); err != nil { + return nil, err + } + + cert, err := toCert(string(key), res) + if err != nil { + return nil, err + } + + items = append(items, cert) + } + + return items, nil } var _ CertDB = &aDB{} diff --git a/server/database/xorm.go b/server/database/xorm.go index d378f15..9e4ed84 100644 --- a/server/database/xorm.go +++ b/server/database/xorm.go @@ -1,9 +1,11 @@ package database import ( + "errors" "fmt" - "github.com/akrylysov/pogreb" + "github.com/rs/zerolog/log" + "github.com/go-acme/lego/v4/certificate" "xorm.io/xorm" @@ -15,6 +17,8 @@ import ( var _ CertDB = xDB{} +var ErrNotFound = errors.New("entry not found") + type xDB struct { engine *xorm.Engine } @@ -43,28 +47,53 @@ func (x xDB) Close() error { } func (x xDB) Put(name string, cert *certificate.Resource) error { - // TODO implement me - panic("implement me") + log.Trace().Str("name", name).Msg("inserting cert to db") + c, err := toCert(name, cert) + if err != nil { + return err + } + _, err = x.engine.Insert(c) + return err } func (x xDB) Get(name string) (*certificate.Resource, error) { - // TODO implement me - panic("implement me") + cert := new(Cert) + log.Trace().Str("name", name).Msg("get cert from db") + if _, err := x.engine.ID(name).Get(&cert); err != nil { + return nil, err + } + if cert == nil { + return nil, fmt.Errorf("%w: name='%s'", ErrNotFound, name) + } + return cert.Raw(), nil } -func (x xDB) Delete(key string) error { - // TODO implement me - panic("implement me") +func (x xDB) Delete(name string) error { + log.Trace().Str("name", name).Msg("delete cert from db") + _, err := x.engine.ID(name).Delete(new(Cert)) + return err } func (x xDB) Compact() (string, error) { - // TODO implement me - panic("implement me") + // not needed + return "", nil } -func (x xDB) Items() *pogreb.ItemIterator { - // TODO implement me - panic("implement me") +// Items return al certs from db, if pageSize is 0 it does not use limit +func (x xDB) Items(page, pageSize int) ([]*Cert, error) { + // paginated return + if pageSize >= 0 { + certs := make([]*Cert, 0, pageSize) + if page >= 0 { + page = 1 + } + err := x.engine.Limit(pageSize, (page-1)*pageSize).Find(&certs) + return certs, err + } + + // return all + certs := make([]*Cert, 0, 64) + return certs, x.engine.Find(&certs) } // Supported database drivers