diff --git a/server/server.go b/server/server.go index be22f5d910..a2ed477a97 100755 --- a/server/server.go +++ b/server/server.go @@ -252,7 +252,7 @@ func newServer(ctx context.Context, c Config, rotationStrategy rotationStrategy) extra: c.Web.Extra, } - static, theme, tmpls, err := loadWebConfig(web) + static, theme, robots, tmpls, err := loadWebConfig(web) if err != nil { return nil, fmt.Errorf("server: failed to load web static: %v", err) } @@ -390,6 +390,8 @@ func newServer(ctx context.Context, c Config, rotationStrategy rotationStrategy) handlePrefix("/static", static) handlePrefix("/theme", theme) + handleFunc("/robots.txt", robots) + s.mux = r s.startKeyRotation(ctx, rotationStrategy, now) diff --git a/server/templates.go b/server/templates.go index e6ab3a793c..b77663e1f5 100644 --- a/server/templates.go +++ b/server/templates.go @@ -88,7 +88,7 @@ func getFuncMap(c webConfig) (template.FuncMap, error) { // |- themes // | |- (theme name) // |- templates -func loadWebConfig(c webConfig) (http.Handler, http.Handler, *templates, error) { +func loadWebConfig(c webConfig) (http.Handler, http.Handler, http.HandlerFunc, *templates, error) { // fallback to the default theme if the legacy theme name is provided if c.theme == "coreos" || c.theme == "tectonic" { c.theme = "" @@ -105,18 +105,24 @@ func loadWebConfig(c webConfig) (http.Handler, http.Handler, *templates, error) staticFiles, err := fs.Sub(c.webFS, "static") if err != nil { - return nil, nil, nil, fmt.Errorf("read static dir: %v", err) + return nil, nil, nil, nil, fmt.Errorf("read static dir: %v", err) } themeFiles, err := fs.Sub(c.webFS, path.Join("themes", c.theme)) if err != nil { - return nil, nil, nil, fmt.Errorf("read themes dir: %v", err) + return nil, nil, nil, nil, fmt.Errorf("read themes dir: %v", err) + } + robotsContent, err := fs.ReadFile(c.webFS, "robots.txt") + if err != nil { + return nil, nil, nil, nil, fmt.Errorf("read robots.txt dir: %v", err) } static := http.FileServer(http.FS(staticFiles)) theme := http.FileServer(http.FS(themeFiles)) + robots := func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, string(robotsContent)) } templates, err := loadTemplates(c, "templates") - return static, theme, templates, err + + return static, theme, robots, templates, err } // loadTemplates parses the expected templates from the provided directory. diff --git a/web/robots.txt b/web/robots.txt new file mode 100644 index 0000000000..1f53798bb4 --- /dev/null +++ b/web/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / diff --git a/web/web.go b/web/web.go index c5ff751490..0c7e9873a7 100644 --- a/web/web.go +++ b/web/web.go @@ -5,7 +5,7 @@ import ( "io/fs" ) -//go:embed static/* templates/* themes/* +//go:embed static/* templates/* themes/* robots.txt var files embed.FS // FS returns a filesystem with the default web assets.