next

Dockerfile

@@ -1,10 +1,10 @@
-FROM golang:alpine as build
+FROM techknowlogick/xgo as build
 
 WORKDIR /workspace
 
 RUN apk add ca-certificates
 COPY . .
-RUN CGO_ENABLED=0 go build .
+RUN CGO_ENABLED=1 go build -tags 'sqlite sqlite_unlock_notify netgo' -ldflags '-s -w -extldflags "-static" -linkmode external' .
 
 FROM scratch
 COPY --from=build /workspace/pages /pages

Justfile

@@ -7,13 +7,13 @@ dev:
     export RAW_DOMAIN=raw.localhost.mock.directory
     export PORT=4430
     export LOG_LEVEL=trace
-    go run .
+    go run -tags 'sqlite sqlite_unlock_notify -ldflags '-s -w -extldflags "-static" -linkmode external' netgo' .
 
 build:
-    CGO_ENABLED=0 go build -ldflags '-s -w' -v -o build/codeberg-pages-server ./
+    CGO_ENABLED=1 go build -tags 'sqlite sqlite_unlock_notify netgo' -ldflags '-s -w -extldflags "-static" -linkmode external' -v -o build/codeberg-pages-server ./
 
 build-tag VERSION:
-    CGO_ENABLED=0 go build -ldflags '-s -w -X "codeberg.org/codeberg/pages/server/version.Version={{VERSION}}"' -v -o build/codeberg-pages-server ./
+    CGO_ENABLED=1 go build -tags 'sqlite sqlite_unlock_notify netgo' '-ldflags '-s -w -X "codeberg.org/codeberg/pages/server/version.Version={{VERSION}}" -extldflags "-static" -linkmode external' -v -o build/codeberg-pages-server ./
 
 lint: tool-golangci tool-gofumpt
     [ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }; \
@@ -38,13 +38,13 @@ tool-gofumpt:
     fi
 
 test:
-    go test -race codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/
+    go test -race -tags 'sqlite sqlite_unlock_notify netgo' codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/
 
 test-run TEST:
-    go test -race -run "^{{TEST}}$" codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/
+    go test -race -tags 'sqlite sqlite_unlock_notify netgo' -run "^{{TEST}}$" codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/
 
 integration:
-    go test -race -tags integration codeberg.org/codeberg/pages/integration/...
+    go test -race -tags 'integration sqlite sqlite_unlock_notify netgo' codeberg.org/codeberg/pages/integration/...
 
 integration-run TEST:
-    go test -race -tags integration -run "^{{TEST}}$" codeberg.org/codeberg/pages/integration/...
+    go test -race -tags 'integration sqlite sqlite_unlock_notify netgo' -run "^{{TEST}}$" codeberg.org/codeberg/pages/integration/...

cmd/certs.go

@@ -2,8 +2,8 @@ package cmd
 
 import (
 	"fmt"
+	"time"
 
-	"github.com/akrylysov/pogreb"
 	"github.com/urfave/cli/v2"
 
 	"codeberg.org/codeberg/pages/server/database"
@@ -52,15 +52,20 @@ func listCerts(ctx *cli.Context) error {
 		return fmt.Errorf("could not create database: %v", err)
 	}
 
-	items := keyDatabase.Items()
+	items, err := keyDatabase.Items(0, 0)
-	for domain, _, err := items.Next(); err != pogreb.ErrIterationDone; domain, _, err = items.Next() {
+	if err != nil {
-		if err != nil {
+		return err
-			return err
+	}
+
+	fmt.Printf("Name\tDomain\tValidTill\n\n")
+	for _, cert := range items {
+		if cert.Name[0] == '.' {
+			cert.Name = "*" + cert.Name
 		}
-		if domain[0] == '.' {
+		fmt.Printf("%s\t%s\t%s\n",
-			fmt.Printf("*")
+			cert.Name,
-		}
+			cert.Domain,
-		fmt.Printf("%s\n", domain)
+			time.Unix(cert.ValidTill, 0).Format(time.RFC3339))
 	}
 	return nil
 }

server/certificates/certificates.go

@@ -1,14 +1,12 @@
 package certificates
 
 import (
-	"bytes"
 	"context"
 	"crypto/ecdsa"
 	"crypto/elliptic"
 	"crypto/rand"
 	"crypto/tls"
 	"crypto/x509"
-	"encoding/gob"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -478,41 +476,35 @@ func SetupCertificates(mainDomainSuffix, dnsProvider string, acmeConfig *lego.Co
 
 func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffix, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) {
 	for {
-		// clean up expired certs
+		// delete expired certs that will be invalid until next clean up
-		now := time.Now()
+		threshold := time.Now().Add(-interval)
 		expiredCertCount := 0
-		keyDatabaseIterator := certDB.Items()
-		key, resBytes, err := keyDatabaseIterator.Next()
-		for err == nil {
-			if !strings.EqualFold(string(key), mainDomainSuffix) {
-				resGob := bytes.NewBuffer(resBytes)
-				resDec := gob.NewDecoder(resGob)
-				res := &certificate.Resource{}
-				err = resDec.Decode(res)
-				if err != nil {
-					panic(err)
-				}
 
-				tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
+		certs, err := certDB.Items(0, 0)
-				if err != nil || tlsCertificates[0].NotAfter.Before(now) {
+		if err != nil {
-					err := certDB.Delete(string(key))
+			log.Error().Err(err).Msg("could not get certs from list")
-					if err != nil {
+		} else {
-						log.Error().Err(err).Msgf("Deleting expired certificate for %q failed", string(key))
+			for _, cert := range certs {
-					} else {
+				if !strings.EqualFold(cert.Name, mainDomainSuffix) {
-						expiredCertCount++
+					if time.Unix(cert.ValidTill, 0).Before(threshold) {
+						err := certDB.Delete(cert.Name)
+						if err != nil {
+							log.Error().Err(err).Msgf("Deleting expired certificate for %q failed", cert.Name)
+						} else {
+							expiredCertCount++
+						}
 					}
 				}
 			}
-			key, resBytes, err = keyDatabaseIterator.Next()
+			log.Debug().Msgf("Removed %d expired certificates from the database", expiredCertCount)
-		}
-		log.Debug().Msgf("Removed %d expired certificates from the database", expiredCertCount)
 
-		// compact the database
+			// compact the database
-		msg, err := certDB.Compact()
+			msg, err := certDB.Compact()
-		if err != nil {
+			if err != nil {
-			log.Error().Err(err).Msg("Compacting key database failed")
+				log.Error().Err(err).Msg("Compacting key database failed")
-		} else {
+			} else {
-			log.Debug().Msgf("Compacted key database: %s", msg)
+				log.Debug().Msgf("Compacted key database: %s", msg)
+			}
 		}
 
 		// update main cert

server/database/interface.go

@@ -1,8 +1,11 @@
 package database
 
 import (
-	"github.com/akrylysov/pogreb"
+	"fmt"
+
+	"github.com/go-acme/lego/v4/certcrypto"
 	"github.com/go-acme/lego/v4/certificate"
+	"github.com/rs/zerolog/log"
 )
 
 type CertDB interface {
@@ -10,14 +13,60 @@ type CertDB interface {
 	Put(name string, cert *certificate.Resource) error
 	Get(name string) (*certificate.Resource, error)
 	Delete(key string) error
+	Items(page, pageSize int) ([]*Cert, error)
+	// Compact deprecated // TODO: remove in next version
 	Compact() (string, error)
-	Items() *pogreb.ItemIterator
 }
 
 type Cert struct {
-	Domain    string `xorm:"pk NOT NULL"`
+	Name      string `xorm:"pk      NOT NULL           'name'"`
-	Created   int64  `xorm:"created NOT NULL DEFAULT 0"`
+	Domain    string `xorm:"        NOT NULL UNIQUE    'domain'"` // TODO: check: is name always same as domain?
-	Updated   int64  `xorm:"updated NOT NULL DEFAULT 0"`
+	Created   int64  `xorm:"created NOT NULL DEFAULT 0 'created'"`
-	ValidTill int64  `xorm:"NOT NULL DEFAULT 0"`
+	Updated   int64  `xorm:"updated NOT NULL DEFAULT 0 'updated'"`
-	Raw       []byte `xorm:"NOT NULL"`
+	ValidTill int64  `xorm:"        NOT NULL DEFAULT 0 'valid_till'"`
+	// certificate.Resource
+	certURL           string `xorm:"'cert_url'"`
+	certStableURL     string `xorm:"'cert_stable_url''"`
+	privateKey        []byte `xorm:"'private_key'"`
+	certificate       []byte `xorm:"'certificate'"`
+	issuerCertificate []byte `xorm:"'issuer_certificate'"` // TODO: dedup ?
+	csr               []byte `xorm:"'csr'"`
+}
+
+func (c Cert) Raw() *certificate.Resource {
+	return &certificate.Resource{
+		Domain:            c.Domain,
+		CertURL:           c.certURL,
+		CertStableURL:     c.certStableURL,
+		PrivateKey:        c.privateKey,
+		Certificate:       c.certificate,
+		IssuerCertificate: c.issuerCertificate,
+		CSR:               c.csr,
+	}
+}
+
+func toCert(name string, c *certificate.Resource) (*Cert, error) {
+	tlsCertificates, err := certcrypto.ParsePEMBundle(c.Certificate)
+	if err != nil {
+		return nil, err
+	}
+	if len(tlsCertificates) != 1 || tlsCertificates[0] == nil {
+		err := fmt.Errorf("parsed cert resource has no or more than one cert")
+		log.Error().Err(err).Str("name", name).Msgf("cert: %v", c)
+		return nil, err
+	}
+	validTill := tlsCertificates[0].NotAfter.Unix()
+
+	return &Cert{
+		Name:      name,
+		Domain:    c.Domain,
+		ValidTill: validTill,
+
+		certURL:           c.CertURL,
+		certStableURL:     c.CertStableURL,
+		privateKey:        c.PrivateKey,
+		certificate:       c.Certificate,
+		issuerCertificate: c.IssuerCertificate,
+		csr:               c.CSR,
+	}, nil
 }

server/database/migrate.go

@@ -0,0 +1 @@
+package database

server/database/mock.go

@@ -5,7 +5,6 @@ import (
 	"time"
 
 	"github.com/OrlovEvgeny/go-mcache"
-	"github.com/akrylysov/pogreb"
 	"github.com/go-acme/lego/v4/certificate"
 )
 
@@ -43,8 +42,8 @@ func (p tmpDB) Compact() (string, error) {
 	return "Truncate done", nil
 }
 
-func (p tmpDB) Items() *pogreb.ItemIterator {
+func (p tmpDB) Items(page, pageSize int) ([]*Cert, error) {
-	panic("ItemIterator not implemented for tmpDB")
+	panic("Items not implemented for tmpDB")
 }
 
 func NewTmpDB() (CertDB, error) {

server/database/pogreb.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/gob"
+	"errors"
 	"fmt"
 	"time"
 
@@ -62,8 +63,32 @@ func (p aDB) Compact() (string, error) {
 	return fmt.Sprintf("%+v", result), nil
 }
 
-func (p aDB) Items() *pogreb.ItemIterator {
+func (p aDB) Items(_, _ int) ([]*Cert, error) {
-	return p.intern.Items()
+	items := make([]*Cert, 0, p.intern.Count())
+	iterator := p.intern.Items()
+	for {
+		key, resBytes, err := iterator.Next()
+		if err != nil {
+			if errors.Is(err, pogreb.ErrIterationDone) {
+				break
+			}
+			return nil, err
+		}
+
+		res := &certificate.Resource{}
+		if err := gob.NewDecoder(bytes.NewBuffer(resBytes)).Decode(res); err != nil {
+			return nil, err
+		}
+
+		cert, err := toCert(string(key), res)
+		if err != nil {
+			return nil, err
+		}
+
+		items = append(items, cert)
+	}
+
+	return items, nil
 }
 
 var _ CertDB = &aDB{}

server/database/xorm.go

@@ -1,9 +1,11 @@
 package database
 
 import (
+	"errors"
 	"fmt"
 
-	"github.com/akrylysov/pogreb"
+	"github.com/rs/zerolog/log"
+
 	"github.com/go-acme/lego/v4/certificate"
 	"xorm.io/xorm"
 
@@ -15,6 +17,8 @@ import (
 
 var _ CertDB = xDB{}
 
+var ErrNotFound = errors.New("entry not found")
+
 type xDB struct {
 	engine *xorm.Engine
 }
@@ -43,28 +47,53 @@ func (x xDB) Close() error {
 }
 
 func (x xDB) Put(name string, cert *certificate.Resource) error {
-	// TODO implement me
+	log.Trace().Str("name", name).Msg("inserting cert to db")
-	panic("implement me")
+	c, err := toCert(name, cert)
+	if err != nil {
+		return err
+	}
+	_, err = x.engine.Insert(c)
+	return err
 }
 
 func (x xDB) Get(name string) (*certificate.Resource, error) {
-	// TODO implement me
+	cert := new(Cert)
-	panic("implement me")
+	log.Trace().Str("name", name).Msg("get cert from db")
+	if _, err := x.engine.ID(name).Get(&cert); err != nil {
+		return nil, err
+	}
+	if cert == nil {
+		return nil, fmt.Errorf("%w: name='%s'", ErrNotFound, name)
+	}
+	return cert.Raw(), nil
 }
 
-func (x xDB) Delete(key string) error {
+func (x xDB) Delete(name string) error {
-	// TODO implement me
+	log.Trace().Str("name", name).Msg("delete cert from db")
-	panic("implement me")
+	_, err := x.engine.ID(name).Delete(new(Cert))
+	return err
 }
 
 func (x xDB) Compact() (string, error) {
-	// TODO implement me
+	// not needed
-	panic("implement me")
+	return "", nil
 }
 
-func (x xDB) Items() *pogreb.ItemIterator {
+// Items return al certs from db, if pageSize is 0 it does not use limit
-	// TODO implement me
+func (x xDB) Items(page, pageSize int) ([]*Cert, error) {
-	panic("implement me")
+	// paginated return
+	if pageSize >= 0 {
+		certs := make([]*Cert, 0, pageSize)
+		if page >= 0 {
+			page = 1
+		}
+		err := x.engine.Limit(pageSize, (page-1)*pageSize).Find(&certs)
+		return certs, err
+	}
+
+	// return all
+	certs := make([]*Cert, 0, 64)
+	return certs, x.engine.Find(&certs)
 }
 
 // Supported database drivers