Use Pebble and Protos For Traces and BlockLogs (#122)

Switch to using Pebble as a Key Value store to store traces and block
logs

* To support that we create a Trace proto which uses a oneof field to
store the fields for the different types of traces
* This will make it easier to support using RunMe logs for training #110

Fix #91

app/api/types.go

@@ -5,7 +5,6 @@ import (
 	"time"
 
 	"github.com/go-logr/zapr"
-	"github.com/jlewi/foyle/protos/go/foyle/v1alpha1"
 	"go.uber.org/zap"
 )
 
@@ -118,81 +117,3 @@ func (L *LogEntry) Time() time.Time {
 	timestamp := time.Unix(seconds, nanoseconds)
 	return timestamp
 }
-
-type TraceType string
-
-const (
-	GenerateTraceType TraceType = "Generate"
-	ExecuteTraceType  TraceType = "Execute"
-)
-
-type Trace interface {
-	// ID is the id of this trace
-	ID() string
-	Type() TraceType
-}
-
-// GenerateTrace is the trace of a generation request.
-type GenerateTrace struct {
-	// ID is the id of this trace
-	TraceID   string                     `json:"traceId"`
-	StartTime time.Time                  `json:"startTime"`
-	EndTime   time.Time                  `json:"endTime"`
-	Request   *v1alpha1.GenerateRequest  `json:"request"`
-	Response  *v1alpha1.GenerateResponse `json:"response"`
-	EvalMode  bool                       `json:"evalMode"`
-}
-
-func (g *GenerateTrace) ID() string {
-	return g.TraceID
-}
-
-func (g *GenerateTrace) Type() TraceType {
-	return GenerateTraceType
-}
-
-// ExecuteTrace is the trace of an execution request.
-type ExecuteTrace struct {
-	// ID is the id of this trace
-	TraceID   string                    `json:"traceId"`
-	StartTime time.Time                 `json:"startTime"`
-	EndTime   time.Time                 `json:"endTime"`
-	Request   *v1alpha1.ExecuteRequest  `json:"request"`
-	Response  *v1alpha1.ExecuteResponse `json:"response"`
-	EvalMode  bool                      `json:"evalMode"`
-}
-
-func (e *ExecuteTrace) ID() string {
-	return e.TraceID
-}
-
-func (e *ExecuteTrace) Type() TraceType {
-	return ExecuteTraceType
-}
-
-// BlockLog is the log of what happened to a block. It includes information about how a block was generated (if it
-// was generated by the AI) and how it was executed if it was.
-type BlockLog struct {
-	// ID is the id of this block
-	ID string `json:"id"`
-
-	// GenTraceID is the trace ID of the generation request
-	GenTraceID string `json:"genTraceId"`
-
-	// ExecTraceIDs are the trace IDs of the execution requests
-	ExecTraceIDs []string `json:"execTraceId"`
-
-	// Doc is the doc that triggered the generated block
-	Doc *v1alpha1.Doc `json:"doc"`
-	// GeneratedBlock is the block generated by the AI
-	GeneratedBlock *v1alpha1.Block `json:"generatedBlock"`
-	// ExecutedBlock is the final block that was actually executed
-	// nil if the block was not executed
-	ExecutedBlock *v1alpha1.Block `json:"executedBlock"`
-
-	// ExitCode is the exit code of the executed block
-	ExitCode int `json:"exitCode"`
-
-	// EvalMode is true if the block was generated as part of an evaluation and shouldn't be used for learning
-	EvalMode bool `json:"evalMode"`
-}

app/cmd/logs.go

@@ -46,21 +46,18 @@ func NewLogsProcessCmd() *cobra.Command {
 					logsDir = app.Config.GetRawLogDir()
 				}
 
-				if outDir == "" {
-					outDir = app.Config.GetProcessedLogDir()
-				}
 				a, err := analyze.NewAnalyzer()
 				if err != nil {
 					return err
 				}
 
 				log := zapr.NewLogger(zap.L())
-				log.Info("Processing logs", "logs", logsDir, "out", outDir)
-				resultFiles, err := a.Analyze(context.Background(), logsDir, outDir)
-				if err != nil {
+				log.Info("Processing logs", "logs", logsDir)
+
+				if err := a.Analyze(context.Background(), logsDir, app.Config.GetTracesDBDir(), app.Config.GetBlocksDBDir()); err != nil {
 					return err
 				}
-				log.Info("Processed logs", "resultFiles", resultFiles)
+				log.Info("Processed logs", "logs", logsDir, "traces", app.Config.GetTracesDBDir(), "blocks", app.Config.GetBlocksDBDir())
 				return nil
 			}()