Create Bot Instances during initial bot join

api/proto/teleport/machineid/v1/bot_instance.proto

@@ -42,15 +42,11 @@
 }
 
 // BotInstanceSpec contains fields
 message BotInstanceSpec {
   // The name of the bot associated with this instance.
   string bot_name = 1;
   // The unique identifier for this instance.
   string instance_id = 2;
-  // The desired expiration offset for this bot instance. The expiration will be
-  // calculated from the current time at creation or authentication plus this
-  // value. A nil `ttl` will not expire.
-  google.protobuf.Duration ttl = 3;
 }
 
 // BotInstanceStatusHeartbeat contains information self-reported by an instance
@@ -85,7 +81,7 @@
 
 // BotInstanceStatusAuthentication contains information about a join or renewal.
 // Ths information is entirely sourced by the Auth Server and can be trusted.
 message BotInstanceStatusAuthentication {
   // The timestamp that the join or renewal was authenticated by the Auth
   // Server.
   google.protobuf.Timestamp authenticated_at = 1;
@@ -101,11 +97,9 @@
   // method, this counter is checked during renewal and the Bot is locked out if
   // the counter in the certificate does not match the counter of the last
   // authentication.
-  int32 generation = 5;
+  uint64 generation = 5;
   // The public key of the Bot instance.
   bytes public_key = 6;
-  // The fingerprint of the public key of the Bot instance.
-  string fingerprint = 7;
 }
 
 // BotInstanceStatus holds the status of a BotInstance.

constants.go

@@ -503,6 +503,10 @@ const (
 	// CertExtensionBotName indicates the name of the Machine ID bot this
 	// certificate was issued to, if any.
 	CertExtensionBotName = "bot-name@goteleport.com"
+	// CertExtensionBotInstanceID indicates the unique identifier of this
+	// Machine ID bot instance, if any. This identifier is persisted through
+	// certificate renewals.
+	CertExtensionBotInstanceID = "bot-instance-id@goteleport.com"
 
 	// CertCriticalOptionSourceAddress is a critical option that defines IP addresses (in CIDR notation)
 	// from which this certificate is accepted for authentication.

lib/auth/auth.go

@@ -2000,6 +2000,9 @@ type certRequest struct {
 	deviceExtensions DeviceExtensions
 	// botName is the name of the bot requesting this cert, if any
 	botName string
+	// botInstanceID is the unique identifier of the bot instance associated
+	// with this cert, if any
+	botInstanceID string
 }
 
 // check verifies the cert request is valid.
@@ -2943,6 +2946,7 @@ func generateCert(ctx context.Context, a *Server, req certRequest, caType types.
 		Renewable:               req.renewable,
 		Generation:              req.generation,
 		BotName:                 req.botName,
+		BotInstanceID:           req.botInstanceID,
 		CertificateExtensions:   req.checker.CertificateExtensions(),
 		AllowedResourceIDs:      requestedResourcesStr,
 		ConnectionDiagnosticID:  req.connectionDiagnosticID,
@@ -3038,6 +3042,7 @@ func generateCert(ctx context.Context, a *Server, req certRequest, caType types.
 		Renewable:               req.renewable,
 		Generation:              req.generation,
 		BotName:                 req.botName,
+		BotInstanceID:           req.botInstanceID,
 		AllowedResourceIDs:      req.checker.GetAllowedResourceIDs(),
 		PrivateKeyPolicy:        attestedKeyPolicy,
 		ConnectionDiagnosticID:  req.connectionDiagnosticID,

lib/auth/auth_with_roles.go

@@ -3251,6 +3251,12 @@ func (a *ServerWithRoles) generateUserCerts(ctx context.Context, req proto.UserC
 		connectionDiagnosticID: req.ConnectionDiagnosticID,
 		attestationStatement:   keys.AttestationStatementFromProto(req.AttestationStatement),
 		botName:                getBotName(user),
+
+		// Always pass through a bot instance ID if available. Note that this
+		// method is only used for bot identity renewals and is not responsible
+		// for issuing new instance IDs; see `generateInitialBotCerts()`
+		// TODO: need to update bot instance with new authentication + generation counter
+		botInstanceID: a.context.Identity.GetIdentity().BotInstanceID,
 	}
 	if user.GetName() != a.context.User.GetName() {
 		certReq.impersonator = a.context.User.GetName()
@@ -3303,6 +3309,10 @@ func (a *ServerWithRoles) generateUserCerts(ctx context.Context, req proto.UserC
 		if err := a.authServer.validateGenerationLabel(ctx, user.GetName(), &certReq, currentIdentityGeneration); err != nil {
 			return nil, trace.Wrap(err)
 		}
+
+		if err := a.updateBotAuthentications(ctx, &certReq); err != nil {
+			return nil, trace.Wrap(err)
+		}
 	}
 
 	certs, err := a.authServer.generateUserCert(ctx, certReq)

lib/auth/bot.go

@@ -26,11 +26,17 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/gravitational/trace"
+	"github.com/sirupsen/logrus"
+	"google.golang.org/protobuf/types/known/timestamppb"
 
 	"github.com/gravitational/teleport/api/client/proto"
+	headerv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/header/v1"
+	machineidv1pb "github.com/gravitational/teleport/api/gen/proto/go/teleport/machineid/v1"
 	"github.com/gravitational/teleport/api/types"
 	apievents "github.com/gravitational/teleport/api/types/events"
 	apiutils "github.com/gravitational/teleport/api/utils"
+	"github.com/gravitational/teleport/lib/auth/machineid/machineidv1"
+	experiment "github.com/gravitational/teleport/lib/auth/machineid/machineidv1/bot_instance_experiment"
 	"github.com/gravitational/teleport/lib/authz"
 	"github.com/gravitational/teleport/lib/defaults"
 	"github.com/gravitational/teleport/lib/events"
@@ -168,14 +174,86 @@ func (a *Server) validateGenerationLabel(ctx context.Context, username string, c
 	return nil
 }
 
+func (a *ServerWithRoles) updateBotAuthentications(ctx context.Context, req *certRequest) error {
+	ident := a.context.Identity.GetIdentity()
+
+	authRecord := &machineidv1pb.BotInstanceStatusAuthentication{
+		AuthenticatedAt: timestamppb.New(a.authServer.GetClock().Now()),
+		PublicKey:       req.publicKey,
+
+		// TODO: for now, this copy of the certificate generation only is
+		// informational. Future changes will transition to trusting (and
+		// verifying) this value in lieu of the old generation label on bot
+		// users.
+		Generation: req.generation,
+
+		// Note: This auth path can only ever be for token joins; all other join
+		// types effectively rejoin every renewal. Other fields will be unset
+		// (metadata, join token name, etc).
+		JoinMethod: string(types.JoinMethodToken),
+	}
+
+	_, err := a.authServer.BotInstance.PatchBotInstance(ctx, ident.BotName, ident.BotInstanceID, func(bi *machineidv1pb.BotInstance) (*machineidv1pb.BotInstance, error) {
+		if bi.Status == nil {
+			bi.Status = &machineidv1pb.BotInstanceStatus{}
+		}
+
+		// If we're at or above the limit, remove enough of the front elements
+		// to make room for the new one at the end.
+		if len(bi.Status.LatestAuthentications) >= machineidv1.AuthenticationHistoryLimit {
+			toRemove := len(bi.Status.LatestAuthentications) - machineidv1.AuthenticationHistoryLimit + 1
+			bi.Status.LatestAuthentications = bi.Status.LatestAuthentications[toRemove:]
+		}
+
+		// An initial auth record should have been added during initial join,
+		// but if not, add it now.
+		if bi.Status.InitialAuthentication == nil {
+			log.WithFields(logrus.Fields{
+				"bot_name":        ident.BotName,
+				"bot_instance_id": ident.BotInstanceID,
+			}).Warn("bot instance is missing its initial authentication record, a new one will be added")
+			bi.Status.InitialAuthentication = authRecord
+		}
+
+		bi.Status.LatestAuthentications = append(bi.Status.LatestAuthentications, authRecord)
+
+		return bi, nil
+	})
+
+	return trace.Wrap(err)
+}
+
+// newBotInstance constructs a new bot instance from a spec and initial authentication
+func newBotInstance(
+	spec *machineidv1pb.BotInstanceSpec,
+	initialAuth *machineidv1pb.BotInstanceStatusAuthentication,
+	expires time.Time,
+) *machineidv1pb.BotInstance {
+	return &machineidv1pb.BotInstance{
+		Kind:    types.KindBotInstance,
+		Version: types.V1,
+		Metadata: &headerv1.Metadata{
+			Expires: timestamppb.New(expires),
+		},
+		Spec: spec,
+		Status: &machineidv1pb.BotInstanceStatus{
+			InitialAuthentication: initialAuth,
+			LatestAuthentications: []*machineidv1pb.BotInstanceStatusAuthentication{initialAuth},
+		},
+	}
+}
+
 // generateInitialBotCerts is used to generate bot certs and overlaps
 // significantly with `generateUserCerts()`. However, it omits a number of
 // options (impersonation, access requests, role requests, actual cert renewal,
 // and most UserCertsRequest options that don't relate to bots) and does not
 // care if the current identity is Nop.  This function does not validate the
 // current identity at all; the caller is expected to validate that the client
 // is allowed to issue the (possibly renewable) certificates.
-func (a *Server) generateInitialBotCerts(ctx context.Context, botName, username, loginIP string, pubKey []byte, expires time.Time, renewable bool) (*proto.Certs, error) {
+func (a *Server) generateInitialBotCerts(
+	ctx context.Context, botName, username, loginIP string, pubKey []byte,
+	expires time.Time, renewable bool, initialAuth *machineidv1pb.BotInstanceStatusAuthentication,
+) (*proto.Certs, error) {
 	var err error
 
 	// Extract the user and role set for whom the certificate will be generated.
@@ -216,6 +294,25 @@ func (a *Server) generateInitialBotCerts(ctx context.Context, botName, username,
 	var generation uint64
 	if renewable {
 		generation = 1
+		initialAuth.Generation = 1
+	}
+
+	var botInstanceID string
+	if experiment.Enabled() {
+		uuid, err := uuid.NewRandom()
+		if err != nil {
+			return nil, trace.Wrap(err)
+		}
+
+		bi := newBotInstance(&machineidv1pb.BotInstanceSpec{
+			BotName:    botName,
+			InstanceId: uuid.String(),
+		}, initialAuth, expires.Add(machineidv1.ExpiryMargin))
+
+		_, err = a.BotInstance.CreateBotInstance(ctx, bi)
+		if err != nil {
+			return nil, trace.Wrap(err)
+		}
 	}
 
 	// Generate certificate
@@ -230,6 +327,7 @@ func (a *Server) generateInitialBotCerts(ctx context.Context, botName, username,
 		generation:    generation,
 		loginIP:       loginIP,
 		botName:       botName,
+		botInstanceID: botInstanceID,
 	}
 
 	if err := a.validateGenerationLabel(ctx, userState.GetName(), &certReq, 0); err != nil {

lib/auth/join.go

@@ -30,8 +30,11 @@ import (
 	"github.com/gravitational/trace"
 	"github.com/sirupsen/logrus"
 	"google.golang.org/grpc/peer"
+	"google.golang.org/protobuf/types/known/structpb"
+	"google.golang.org/protobuf/types/known/timestamppb"
 
 	"github.com/gravitational/teleport/api/client/proto"
+	machineidv1pb "github.com/gravitational/teleport/api/gen/proto/go/teleport/machineid/v1"
 	"github.com/gravitational/teleport/api/types"
 	apievents "github.com/gravitational/teleport/api/types/events"
 	"github.com/gravitational/teleport/lib/auth/machineid/machineidv1"
@@ -346,24 +349,7 @@ func (a *Server) generateCertsBot(
 		expires = *req.Expires
 	}
 
-	certs, err := a.generateInitialBotCerts(
-		ctx, botName, machineidv1.BotResourceName(botName), req.RemoteAddr, req.PublicSSHKey, expires, renewable,
-	)
-	if err != nil {
-		return nil, trace.Wrap(err)
-	}
-
-	if shouldDeleteToken {
-		// delete ephemeral bot join tokens so they can't be re-used
-		if err := a.DeleteToken(ctx, provisionToken.GetName()); err != nil {
-			log.WithError(err).Warnf("Could not delete bot provision token %q after generating certs",
-				provisionToken.GetSafeName(),
-			)
-		}
-	}
-
-	// Emit audit event for bot join.
-	log.Infof("Bot %q has joined the cluster.", botName)
+	// Construct a Join event to be sent later.
 	joinEvent := &apievents.BotJoin{
 		Metadata: apievents.Metadata{
 			Type: events.BotJoinEvent,
@@ -380,6 +366,20 @@ func (a *Server) generateCertsBot(
 			RemoteAddr: req.RemoteAddr,
 		},
 	}
+
+	auth := &machineidv1pb.BotInstanceStatusAuthentication{
+		AuthenticatedAt: timestamppb.New(a.GetClock().Now()),
+		// TODO: GetSafeName may not return an appropriate value for later
+		// comparison / locking purposes, and this also shouldn't contain
+		// secrets. Should we hash it?
+		JoinToken:  provisionToken.GetSafeName(),
+		JoinMethod: string(provisionToken.GetJoinMethod()),
+		PublicKey:  req.PublicTLSKey,
+
+		// Note: Generation will be set during `generateInitialBotCerts()` as
+		// needed.
+	}
+
 	if joinAttributeSrc != nil {
 		attributes, err := joinAttributeSrc.JoinAuditAttributes()
 		if err != nil {
@@ -389,7 +389,32 @@ func (a *Server) generateCertsBot(
 		if err != nil {
 			log.WithError(err).Warn("Unable to encode join attributes for audit event.")
 		}
+
+		auth.Metadata, err = structpb.NewStruct(attributes)
+		if err != nil {
+			log.WithError(err).Warn("Unable to encode struct value for join metadata.")
+		}
 	}
+
+	certs, err := a.generateInitialBotCerts(
+		ctx, botName, machineidv1.BotResourceName(botName), req.RemoteAddr, req.PublicSSHKey, expires, renewable, auth,
+	)
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+
+	if shouldDeleteToken {
+		// delete ephemeral bot join tokens so they can't be re-used
+		if err := a.DeleteToken(ctx, provisionToken.GetName()); err != nil {
+			log.WithError(err).Warnf("Could not delete bot provision token %q after generating certs",
+				provisionToken.GetSafeName(),
+			)
+		}
+	}
+
+	// Emit audit event for bot join.
+	log.Infof("Bot %q has joined the cluster.", botName)
+
 	if err := a.emitter.EmitAuditEvent(ctx, joinEvent); err != nil {
 		log.WithError(err).Warn("Failed to emit bot join event.")
 	}

lib/auth/keygen/keygen.go

@@ -197,6 +197,9 @@ func (k *Keygen) GenerateUserCertWithoutValidation(c services.UserCertParams) ([
 	if c.BotName != "" {
 		cert.Permissions.Extensions[teleport.CertExtensionBotName] = c.BotName
 	}
+	if c.BotInstanceID != "" {
+		cert.Permissions.Extensions[teleport.CertExtensionBotInstanceID] = c.BotInstanceID
+	}
 	if c.AllowedResourceIDs != "" {
 		cert.Permissions.Extensions[teleport.CertExtensionAllowedResources] = c.AllowedResourceIDs
 	}

lib/auth/machineid/machineidv1/bot_instance_experiment/experiment.go

@@ -0,0 +1,40 @@
+/*
+ * Teleport
+ * Copyright (C) 2024  Gravitational, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package experiment
+
+import (
+	"os"
+	"sync/atomic"
+)
+
+var enabled = atomic.Bool{}
+
+func init() {
+	enabled.Store(os.Getenv("BOT_INSTANCE_EXPERIMENT") == "1")
+}
+
+// Enabled returns true if the workload identity experiment is enabled.
+func Enabled() bool {
+	return enabled.Load()
+}
+
+// SetEnabled sets the workload identity experiment to the given value.
+func SetEnabled(value bool) {
+	enabled.Store(value)
+}

lib/auth/machineid/machineidv1/bot_instance_service.go

@@ -21,6 +21,7 @@ package machineidv1
 import (
 	"context"
 	"log/slog"
+	"time"
 
 	"github.com/gravitational/trace"
 	"github.com/jonboulle/clockwork"
@@ -33,6 +34,18 @@ import (
 	"github.com/gravitational/teleport/lib/services"
 )
 
+const (
+	// AuthenticationHistoryLimit is the maximum number of authentication
+	// records to be recorded in a bot instance's .Status.LatestAuthentications
+	// field.
+	AuthenticationHistoryLimit = 10
+
+	// ExpiryMargin is the duration added to bot instance expiration times to
+	// ensure the instance remains accessible until shortly after the last
+	// issued certificate expires.
+	ExpiryMargin = time.Minute * 5
+)
+
 // BotInstanceServiceConfig holds configuration options for the BotInstance gRPC
 // service.
 type BotInstanceServiceConfig struct {