bskyweb: gzip HTTP responses + some other minor improvements (#826)

* bskyweb: gzip HTTP responses + JSON logging + minor refactoring

* reduce timeout and max header size

* add a security.txt

5 files changed, 146 insertions, 51 deletions

diff --git a/bskyweb/cmd/bskyweb/.gitignore b/bskyweb/cmd/bskyweb/.gitignore
index 45883bf13..c810652a1 100644
--- a/bskyweb/cmd/bskyweb/.gitignore
+++ b/bskyweb/cmd/bskyweb/.gitignore
@@ -1 +1 @@
-bskyweb
+/bskyweb
diff --git a/bskyweb/cmd/bskyweb/mailmodo.go b/bskyweb/cmd/bskyweb/mailmodo.go
index 67ee6a2d6..e892971f9 100644
--- a/bskyweb/cmd/bskyweb/mailmodo.go
+++ b/bskyweb/cmd/bskyweb/mailmodo.go
@@ -14,13 +14,15 @@ type Mailmodo struct {
 	httpClient *http.Client
 	APIKey     string
 	BaseURL    string
+	ListName   string
 }
 
-func NewMailmodo(apiKey string) *Mailmodo {
+func NewMailmodo(apiKey, listName string) *Mailmodo {
 	return &Mailmodo{
 		APIKey:     apiKey,
 		BaseURL:    "https://api.mailmodo.com/api/v1",
 		httpClient: &http.Client{},
+		ListName:   listName,
 	}
 }
 
@@ -56,9 +58,9 @@ func (m *Mailmodo) request(ctx context.Context, httpMethod string, apiMethod str
 	return nil
 }
 
-func (m *Mailmodo) AddToList(ctx context.Context, listName, email string) error {
+func (m *Mailmodo) AddToList(ctx context.Context, email string) error {
 	return m.request(ctx, "POST", "addToList", map[string]any{
-		"listName": listName,
+		"listName": m.ListName,
 		"email":    email,
 		"data": map[string]any{
 			"email_hashed": fmt.Sprintf("%x", sha256.Sum256([]byte(email))),
diff --git a/bskyweb/cmd/bskyweb/server.go b/bskyweb/cmd/bskyweb/server.go
index cdd4dd1d2..e8a71dfe4 100644
--- a/bskyweb/cmd/bskyweb/server.go
+++ b/bskyweb/cmd/bskyweb/server.go
@@ -3,12 +3,16 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/fs"
 	"io/ioutil"
 	"net/http"
 	"os"
+	"os/signal"
 	"strings"
+	"syscall"
+	"time"
 
 	comatproto "github.com/bluesky-social/indigo/api/atproto"
 	appbsky "github.com/bluesky-social/indigo/api/bsky"
@@ -17,13 +21,18 @@ import (
 	"github.com/bluesky-social/social-app/bskyweb"
 
 	"github.com/flosch/pongo2/v6"
+	"github.com/klauspost/compress/gzhttp"
+	"github.com/klauspost/compress/gzip"
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4/middleware"
 	"github.com/urfave/cli/v2"
 )
 
 type Server struct {
-	xrpcc *xrpc.Client
+	echo     *echo.Echo
+	httpd    *http.Server
+	mailmodo *Mailmodo
+	xrpcc    *xrpc.Client
 }
 
 func serve(cctx *cli.Context) error {
@@ -35,8 +44,11 @@ func serve(cctx *cli.Context) error {
 	mailmodoAPIKey := cctx.String("mailmodo-api-key")
 	mailmodoListName := cctx.String("mailmodo-list-name")
 
+	// Echo
+	e := echo.New()
+
 	// Mailmodo client.
-	mailmodo := NewMailmodo(mailmodoAPIKey)
+	mailmodo := NewMailmodo(mailmodoAPIKey, mailmodoListName)
 
 	// create a new session
 	// TODO: does this work with no auth at all?
@@ -60,21 +72,43 @@ func serve(cctx *cli.Context) error {
 	xrpcc.Auth.Did = auth.Did
 	xrpcc.Auth.Handle = auth.Handle
 
-	server := Server{xrpcc}
+	// httpd
+	var (
+		httpTimeout          = 2 * time.Minute
+		httpMaxHeaderBytes   = 2 * (1024 * 1024)
+		gzipMinSizeBytes     = 1024 * 2
+		gzipCompressionLevel = gzip.BestSpeed
+		gzipExceptMIMETypes  = []string{"image/png"}
+	)
+
+	// Wrap the server handler in a gzip handler to compress larger responses.
+	gzipHandler, err := gzhttp.NewWrapper(
+		gzhttp.MinSize(gzipMinSizeBytes),
+		gzhttp.CompressionLevel(gzipCompressionLevel),
+		gzhttp.ExceptContentTypes(gzipExceptMIMETypes),
+	)
+	if err != nil {
+		return err
+	}
 
-	staticHandler := http.FileServer(func() http.FileSystem {
-		if debug {
-			log.Debugf("serving static file from the local file system")
-			return http.FS(os.DirFS("static"))
-		}
-		fsys, err := fs.Sub(bskyweb.StaticFS, "static")
-		if err != nil {
-			log.Fatal(err)
-		}
-		return http.FS(fsys)
-	}())
+	//
+	// server
+	//
+	server := &Server{
+		echo:     e,
+		mailmodo: mailmodo,
+		xrpcc:    xrpcc,
+	}
+
+	// Create the HTTP server.
+	server.httpd = &http.Server{
+		Handler:        gzipHandler(server),
+		Addr:           httpAddress,
+		WriteTimeout:   httpTimeout,
+		ReadTimeout:    httpTimeout,
+		MaxHeaderBytes: httpMaxHeaderBytes,
+	}
 
-	e := echo.New()
 	e.HideBanner = true
 	// SECURITY: Do not modify without due consideration.
 	e.Use(middleware.SecureWithConfig(middleware.SecureConfig{
@@ -90,10 +124,9 @@ func serve(cctx *cli.Context) error {
 		Skipper: func(c echo.Context) bool {
 			return strings.HasPrefix(c.Request().URL.Path, "/static")
 		},
-		Format: "method=${method} path=${uri} status=${status} latency=${latency_human}\n",
 	}))
 	e.Renderer = NewRenderer("templates/", &bskyweb.TemplateFS, debug)
-	e.HTTPErrorHandler = customHTTPErrorHandler
+	e.HTTPErrorHandler = server.errorHandler
 
 	// redirect trailing slash to non-trailing slash.
 	// all of our current endpoints have no trailing slash.
@@ -106,9 +139,23 @@ func serve(cctx *cli.Context) error {
 	//
 
 	// static files
+	staticHandler := http.FileServer(func() http.FileSystem {
+		if debug {
+			log.Debugf("serving static file from the local file system")
+			return http.FS(os.DirFS("static"))
+		}
+		fsys, err := fs.Sub(bskyweb.StaticFS, "static")
+		if err != nil {
+			log.Fatal(err)
+		}
+		return http.FS(fsys)
+	}())
 	e.GET("/robots.txt", echo.WrapHandler(staticHandler))
 	e.GET("/static/*", echo.WrapHandler(http.StripPrefix("/static/", staticHandler)))
 	e.GET("/.well-known/*", echo.WrapHandler(staticHandler))
+	e.GET("/security.txt", func(c echo.Context) error {
+		return c.Redirect(http.StatusMovedPermanently, "/.well-known/security.txt")
+	})
 
 	// home
 	e.GET("/", server.WebHome)
@@ -147,44 +194,54 @@ func serve(cctx *cli.Context) error {
 	e.GET("/profile/:handle/post/:rkey/reposted-by", server.WebGeneric)
 
 	// Mailmodo
-	e.POST("/api/waitlist", func(c echo.Context) error {
-		type jsonError struct {
-			Error string `json:"error"`
-		}
+	e.POST("/api/waitlist", server.apiWaitlist)
 
-		// Read the API request.
-		type apiRequest struct {
-			Email string `json:"email"`
-		}
-
-		bodyReader := http.MaxBytesReader(c.Response(), c.Request().Body, 16*1024)
-		payload, err := ioutil.ReadAll(bodyReader)
-		if err != nil {
-			return err
+	// Start the server.
+	log.Infof("starting server address=%s", httpAddress)
+	go func() {
+		if err := server.httpd.ListenAndServe(); err != nil {
+			if !errors.Is(err, http.ErrServerClosed) {
+				log.Errorf("HTTP server shutting down unexpectedly: %s", err)
+			}
 		}
-		var req apiRequest
-		if err := json.Unmarshal(payload, &req); err != nil {
-			return c.JSON(http.StatusBadRequest, jsonError{Error: "Invalid API request"})
+	}()
+
+	// Wait for a signal to exit.
+	log.Info("registering OS exit signal handler")
+	quit := make(chan struct{})
+	exitSignals := make(chan os.Signal, 1)
+	signal.Notify(exitSignals, syscall.SIGINT, syscall.SIGTERM)
+	go func() {
+		sig := <-exitSignals
+		log.Infof("received OS exit signal: %s", sig)
+
+		// Shut down the HTTP server.
+		if err := server.Shutdown(); err != nil {
+			log.Errorf("HTTP server shutdown error: %s", err)
 		}
 
-		if req.Email == "" {
-			return c.JSON(http.StatusBadRequest, jsonError{Error: "Please enter a valid email address."})
-		}
+		// Trigger the return that causes an exit.
+		close(quit)
+	}()
+	<-quit
+	log.Infof("graceful shutdown complete")
+	return nil
+}
 
-		if err := mailmodo.AddToList(c.Request().Context(), mailmodoListName, req.Email); err != nil {
-			log.Errorf("adding email to waitlist failed: %s", err)
-			return c.JSON(http.StatusBadRequest, jsonError{
-				Error: "Storing email in waitlist failed. Please enter a valid email address.",
-			})
-		}
-		return c.JSON(http.StatusOK, map[string]bool{"success": true})
-	})
+func (srv *Server) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
+	srv.echo.ServeHTTP(rw, req)
+}
 
-	log.Infof("starting server address=%s", httpAddress)
-	return e.Start(httpAddress)
+func (srv *Server) Shutdown() error {
+	log.Info("shutting down")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	return srv.httpd.Shutdown(ctx)
 }
 
-func customHTTPErrorHandler(err error, c echo.Context) {
+func (srv *Server) errorHandler(err error, c echo.Context) {
 	code := http.StatusInternalServerError
 	if he, ok := err.(*echo.HTTPError); ok {
 		code = he.Code
@@ -260,3 +317,36 @@ func (srv *Server) WebProfile(c echo.Context) error {
 
 	return c.Render(http.StatusOK, "profile.html", data)
 }
+
+func (srv *Server) apiWaitlist(c echo.Context) error {
+	type jsonError struct {
+		Error string `json:"error"`
+	}
+
+	// Read the API request.
+	type apiRequest struct {
+		Email string `json:"email"`
+	}
+
+	bodyReader := http.MaxBytesReader(c.Response(), c.Request().Body, 16*1024)
+	payload, err := ioutil.ReadAll(bodyReader)
+	if err != nil {
+		return err
+	}
+	var req apiRequest
+	if err := json.Unmarshal(payload, &req); err != nil {
+		return c.JSON(http.StatusBadRequest, jsonError{Error: "Invalid API request"})
+	}
+
+	if req.Email == "" {
+		return c.JSON(http.StatusBadRequest, jsonError{Error: "Please enter a valid email address."})
+	}
+
+	if err := srv.mailmodo.AddToList(c.Request().Context(), req.Email); err != nil {
+		log.Errorf("adding email to waitlist failed: %s", err)
+		return c.JSON(http.StatusBadRequest, jsonError{
+			Error: "Storing email in waitlist failed. Please enter a valid email address.",
+		})
+	}
+	return c.JSON(http.StatusOK, map[string]bool{"success": true})
+}
diff --git a/bskyweb/go.mod b/bskyweb/go.mod
index b2d49a92b..5f06bfc45 100644
--- a/bskyweb/go.mod
+++ b/bskyweb/go.mod
@@ -7,6 +7,7 @@ require (
 	github.com/flosch/pongo2/v6 v6.0.0
 	github.com/ipfs/go-log v1.0.5
 	github.com/joho/godotenv v1.5.1
+	github.com/klauspost/compress v1.16.5
 	github.com/labstack/echo/v4 v4.10.2
 	github.com/urfave/cli/v2 v2.25.3
 )
diff --git a/bskyweb/go.sum b/bskyweb/go.sum
index fa15187c3..ae5d7defb 100644
--- a/bskyweb/go.sum
+++ b/bskyweb/go.sum
@@ -105,6 +105,8 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
+github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=