From 4c55927df5f08e08ffdcb0f5a8eb4bb15a083d60 Mon Sep 17 00:00:00 2001 From: Jeff Ortel Date: Thu, 24 Aug 2023 14:42:40 -0700 Subject: [PATCH 1/7] :sparkles: Add support for analysis report download; adds tar package. Signed-off-by: Jeff Ortel --- Dockerfile | 3 + api/analysis.go | 476 +++++++++++++++++++++++++++++++++++++-- api/base.go | 9 + api/bucket.go | 192 ++-------------- api/import.go | 3 +- api/pkg.go | 1 + binding/client.go | 104 +-------- hack/add/all.sh | 1 + hack/add/bucket.sh | 36 +-- settings/hub.go | 9 + tar/filter.go | 32 +++ tar/reader.go | 79 +++++++ tar/writer.go | 238 ++++++++++++++++++++ test/tar/data/pet/cat | 2 + test/tar/data/pet/dog | 2 + test/tar/data/rabbit | 1 + test/tar/data/wild/lion | 2 + test/tar/data/wild/tiger | 1 + test/tar/tar_test.go | 65 ++++++ 19 files changed, 945 insertions(+), 311 deletions(-) create mode 100644 tar/filter.go create mode 100644 tar/reader.go create mode 100644 tar/writer.go create mode 100644 test/tar/data/pet/cat create mode 100644 test/tar/data/pet/dog create mode 100644 test/tar/data/rabbit create mode 100644 test/tar/data/wild/lion create mode 100644 test/tar/data/wild/tiger create mode 100644 test/tar/tar_test.go diff --git a/Dockerfile b/Dockerfile index 3fd98ea4f..81486cf83 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,11 +4,14 @@ COPY --chown=1001:0 . . RUN make docker RUN git clone https://github.com/konveyor/tackle2-seed +FROM quay.io/konveyor/static-report as report + FROM registry.access.redhat.com/ubi9/ubi-minimal COPY --from=builder /opt/app-root/src/bin/hub /usr/local/bin/tackle-hub COPY --from=builder /opt/app-root/src/auth/roles.yaml /tmp/roles.yaml COPY --from=builder /opt/app-root/src/auth/users.yaml /tmp/users.yaml COPY --from=builder /opt/app-root/src/tackle2-seed/resources/ /tmp/seed +COPY --from=report /usr/local/static-report /tmp/report RUN microdnf -y install \ sqlite \ diff --git a/api/analysis.go b/api/analysis.go index 61090555f..1b00ac6d9 100644 --- a/api/analysis.go +++ b/api/analysis.go @@ -1,16 +1,25 @@ package api import ( + "bytes" "encoding/json" "errors" + "fmt" "github.com/gin-gonic/gin" + "github.com/gin-gonic/gin/binding" qf "github.com/konveyor/tackle2-hub/api/filter" "github.com/konveyor/tackle2-hub/model" + "github.com/konveyor/tackle2-hub/tar" + "gopkg.in/yaml.v2" "gorm.io/gorm" "gorm.io/gorm/clause" "gorm.io/gorm/logger" "io" + "k8s.io/apimachinery/pkg/util/rand" "net/http" + "os" + "strconv" + "strings" ) // @@ -83,47 +92,91 @@ func (h AnalysisHandler) AddRoutes(e *gin.Engine) { // @summary Get an analysis (report) by ID. // @description Get an analysis (report) by ID. // @tags analyses -// @produce json +// @produce octet-stream // @success 200 {object} api.Analysis // @router /analyses/{id} [get] // @param id path string true "Analysis ID" func (h AnalysisHandler) Get(ctx *gin.Context) { id := h.pk(ctx) - m := &model.Analysis{} - db := h.preLoad(h.DB(ctx), clause.Associations) - result := db.First(m, id) - if result.Error != nil { - _ = ctx.Error(result.Error) + writer := AnalysisWriter{ctx: ctx} + path, err := writer.Create(id) + if err != nil { + _ = ctx.Error(err) return } - r := Analysis{} - r.With(m) - - h.Respond(ctx, http.StatusOK, r) + defer func() { + _ = os.Remove(path) + }() + h.Status(ctx, http.StatusOK) + ctx.File(path) } // AppLatest godoc // @summary Get the latest analysis. // @description Get the latest analysis for an application. +// @description When Accept: is application/x-tar, the static report returned. // @tags analyses -// @produce json +// @produce octet-stream // @success 200 {object} api.Analysis // @router /applications/{id}/analysis [get] // @param id path string true "Application ID" func (h AnalysisHandler) AppLatest(ctx *gin.Context) { id := h.pk(ctx) m := &model.Analysis{} - db := h.preLoad(h.DB(ctx), clause.Associations) - db = db.Where("ApplicationID = ?", id) - result := db.Last(m) - if result.Error != nil { - _ = ctx.Error(result.Error) + db := h.DB(ctx) + db = db.Preload("Application") + db = db.Preload("Application.Tags") + db = db.Preload("Application.Tags.Category") + db = db.Where("ApplicationID", id) + err := db.Last(&m).Error + if err != nil { + _ = ctx.Error(err) return } - r := Analysis{} - r.With(m) - - h.Respond(ctx, http.StatusOK, r) + offered := append(BindMIMEs, TAR) + accepted := ctx.NegotiateFormat(offered...) + if accepted == TAR { + bundleWriter := BundleWriter{} + bundleWriter.ctx = ctx + path, err := bundleWriter.Create(id) + if err != nil { + _ = ctx.Error(err) + return + } + defer func() { + _ = os.Remove(path) + }() + tarWriter := tar.NewWriter(ctx.Writer) + defer func() { + tarWriter.Close() + }() + err = tarWriter.AssertDir(Settings.Report.Path) + if err != nil { + _ = ctx.Error(err) + return + } + err = tarWriter.AssertFile(path) + if err != nil { + _ = ctx.Error(err) + return + } + h.Attachment(ctx, fmt.Sprintf("%s.tar.gz", m.Application.Name)) + ctx.Status(http.StatusOK) + _ = tarWriter.AddDir(Settings.Report.Path) + _ = tarWriter.AddFile(path, "output.js") + } else { + writer := AnalysisWriter{ctx: ctx} + path, err := writer.Create(id) + if err != nil { + _ = ctx.Error(err) + return + } + defer func() { + _ = os.Remove(path) + }() + h.Status(ctx, http.StatusOK) + ctx.File(path) + } } // AppList godoc @@ -148,7 +201,6 @@ func (h AnalysisHandler) AppList(ctx *gin.Context) { db := h.DB(ctx) db = db.Model(&model.Analysis{}) db = db.Where("ApplicationID = ?", id) - db = db.Preload(clause.Associations) db = sort.Sorted(db) var list []model.Analysis var m model.Analysis @@ -1704,8 +1756,8 @@ func (h *AnalysisHandler) depIDs(ctx *gin.Context, f qf.Filter) (q *gorm.DB) { type Analysis struct { Resource `yaml:",inline"` Effort int `json:"effort"` - Issues []Issue `json:"issues,omitempty"` - Dependencies []TechDependency `json:"dependencies,omitempty"` + Issues []Issue `json:"issues,omitempty" yaml:",omitempty"` + Dependencies []TechDependency `json:"dependencies,omitempty" yaml:",omitempty"` } // @@ -1992,3 +2044,381 @@ type DepAppReport struct { // // FactMap map. type FactMap map[string]interface{} + +// +// AnalysisWriter used to create a file containing an analysis. +type AnalysisWriter struct { + encoder + ctx *gin.Context +} + +// +// db returns a db client. +func (r *AnalysisWriter) db() (db *gorm.DB) { + rtx := WithContext(r.ctx) + db = rtx.DB.Debug() + return +} + +// +// Create an analysis file and returns the path. +func (r *AnalysisWriter) Create(id uint) (path string, err error) { + path = fmt.Sprintf("/tmp/report-%d", rand.Int()) + accepted := r.ctx.NegotiateFormat(BindMIMEs...) + switch accepted { + case "", + binding.MIMEPOSTForm, + binding.MIMEJSON: + path += ".json" + case binding.MIMEYAML: + path += ".yaml" + default: + err = &BadRequestError{"MIME not supported."} + } + file, err := os.Create(path) + if err != nil { + return + } + defer func() { + _ = file.Close() + }() + err = r.Write(id, file) + return +} + +// +// Write the analysis file. +func (r *AnalysisWriter) Write(id uint, output io.Writer) (err error) { + m := &model.Analysis{} + db := r.db() + err = db.First(m, id).Error + if err != nil { + return + } + r.encoder, err = r.newEncoder(output) + if err != nil { + return + } + r.begin() + rx := &Analysis{} + rx.With(m) + r.embed(rx) + err = r.addIssues(m) + if err != nil { + return + } + err = r.addDeps(m) + if err != nil { + return + } + r.end() + + return +} + +// +// newEncoder returns an encoder. +func (r *AnalysisWriter) newEncoder(output io.Writer) (encoder encoder, err error) { + accepted := r.ctx.NegotiateFormat(BindMIMEs...) + switch accepted { + case "", + binding.MIMEPOSTForm, + binding.MIMEJSON: + encoder = &jsonEncoder{output: output} + case binding.MIMEYAML: + encoder = &yamlEncoder{output: output} + default: + err = &BadRequestError{"MIME not supported."} + } + + return +} + +// +// addIssues writes issues. +func (r *AnalysisWriter) addIssues(m *model.Analysis) (err error) { + r.field("issues") + r.beginList() + batch := 10 + for b := 0; ; b += batch { + db := r.db() + db = db.Preload("Incidents") + db = db.Limit(batch) + db = db.Offset(b) + var issues []model.Issue + err = db.Find(&issues, "AnalysisID", m.ID).Error + if err != nil { + return + } + if len(issues) == 0 { + break + } + for i := range issues { + issue := Issue{} + issue.With(&issues[i]) + r.writeItem(b, i, issue) + } + } + r.endList() + return +} + +// +// addDeps writes dependencies. +func (r *AnalysisWriter) addDeps(m *model.Analysis) (err error) { + r.field("dependencies") + r.beginList() + batch := 100 + for b := 0; ; b += batch { + db := r.db() + db = db.Limit(batch) + db = db.Offset(b) + var deps []model.TechDependency + err = db.Find(&deps, "AnalysisID", m.ID).Error + if err != nil { + return + } + if len(deps) == 0 { + break + } + for i := range deps { + d := TechDependency{} + d.With(&deps[i]) + r.writeItem(b, i, d) + } + } + r.endList() + return +} + +// +// BundleWriter analysis (static) report bundle writer. +type BundleWriter struct { + AnalysisWriter +} + +// +// Create the output.js file. +func (r *BundleWriter) Create(id uint) (path string, err error) { + path = fmt.Sprintf("/tmp/ouput-%d.js", rand.Int()) + file, err := os.Create(path) + if err != nil { + return + } + defer func() { + _ = file.Close() + }() + err = r.Write(id, file) + return +} + +// +// Write the bundle output.js file. +func (r *BundleWriter) Write(id uint, output io.Writer) (err error) { + m := &model.Analysis{} + db := r.db() + db = db.Preload("Application") + db = db.Preload("Application.Tags") + db = db.Preload("Application.Tags.Category") + err = db.First(m, id).Error + if err != nil { + return + } + r.encoder = &jsonEncoder{output: output} + r.write("window[\"apps\"]=[") + r.begin() + r.field("id").write(strconv.Itoa(int(m.Application.ID))) + r.field("name").writeStr(m.Application.Name) + err = r.addIssues(m) + if err != nil { + return + } + err = r.addDeps(m) + if err != nil { + return + } + err = r.addTags(m) + if err != nil { + return + } + r.end() + r.write("]") + return +} + +// +// addTags writes tags. +func (r *BundleWriter) addTags(m *model.Analysis) (err error) { + r.field("tags") + r.beginList() + for i := range m.Application.Tags { + m := m.Application.Tags[i] + tag := Tag{} + tag.ID = m.ID + tag.Name = m.Name + tag.Category = Ref{ + ID: m.Category.ID, + Name: m.Category.Name, + } + r.writeItem(0, i, tag) + } + r.endList() + return +} + +type encoder interface { + begin() encoder + end() encoder + write(s string) encoder + writeStr(s string) encoder + field(name string) encoder + beginList() encoder + endList() encoder + writeItem(batch, index int, object any) encoder + encode(object any) encoder + embed(object any) encoder +} + +type jsonEncoder struct { + output io.Writer + fields int +} + +func (r *jsonEncoder) begin() encoder { + r.write("{") + return r +} + +func (r *jsonEncoder) end() encoder { + r.write("}") + return r +} + +func (r *jsonEncoder) write(s string) encoder { + _, _ = r.output.Write([]byte(s)) + return r +} + +func (r *jsonEncoder) writeStr(s string) encoder { + r.write("\"" + s + "\"") + return r +} + +func (r *jsonEncoder) field(s string) encoder { + if r.fields > 0 { + r.write(",") + } + r.writeStr(s).write(":") + r.fields++ + return r +} + +func (r *jsonEncoder) beginList() encoder { + r.write("[") + return r +} + +func (r *jsonEncoder) endList() encoder { + r.write("]") + return r +} + +func (r *jsonEncoder) writeItem(batch, index int, object any) encoder { + if batch > 0 || index > 0 { + r.write(",") + } + r.encode(object) + return r +} + +func (r *jsonEncoder) encode(object any) encoder { + encoder := json.NewEncoder(r.output) + _ = encoder.Encode(object) + return r +} + +func (r *jsonEncoder) embed(object any) encoder { + b := new(bytes.Buffer) + encoder := json.NewEncoder(b) + _ = encoder.Encode(object) + s := b.String() + mp := make(map[string]any) + err := json.Unmarshal([]byte(s), &mp) + if err == nil { + r.fields += len(mp) + s = s[1 : len(s)-2] + } + r.write(s) + return r +} + +type yamlEncoder struct { + output io.Writer + fields int + depth int +} + +func (r *yamlEncoder) begin() encoder { + r.write("---\n") + return r +} + +func (r *yamlEncoder) end() encoder { + return r +} + +func (r *yamlEncoder) write(s string) encoder { + s += strings.Repeat(" ", r.depth) + _, _ = r.output.Write([]byte(s)) + return r +} + +func (r *yamlEncoder) writeStr(s string) encoder { + r.write("\"" + s + "\"") + return r +} + +func (r *yamlEncoder) field(s string) encoder { + if r.fields > 0 { + r.write("\n") + } + r.write(s).write(": ") + r.fields++ + return r +} + +func (r *yamlEncoder) beginList() encoder { + r.write("\n") + r.depth++ + return r +} + +func (r *yamlEncoder) endList() encoder { + r.depth-- + return r +} + +func (r *yamlEncoder) writeItem(batch, index int, object any) encoder { + r.encode([]any{object}) + return r +} + +func (r *yamlEncoder) encode(object any) encoder { + encoder := yaml.NewEncoder(r.output) + _ = encoder.Encode(object) + return r +} + +func (r *yamlEncoder) embed(object any) encoder { + b := new(bytes.Buffer) + encoder := yaml.NewEncoder(b) + _ = encoder.Encode(object) + s := b.String() + mp := make(map[string]any) + err := yaml.Unmarshal([]byte(s), &mp) + if err == nil { + r.fields += len(mp) + } + r.write(s) + return r +} diff --git a/api/base.go b/api/base.go index 31a63eb9b..a9cd6f8e1 100644 --- a/api/base.go +++ b/api/base.go @@ -234,6 +234,15 @@ func (h *BaseHandler) Accepted(ctx *gin.Context, mimes ...string) (b bool) { return } +// +// Attachment sets the Content-Disposition header. +func (h *BaseHandler) Attachment(ctx *gin.Context, name string) { + attachment := fmt.Sprintf("attachment; filename=\"%s\"", name) + ctx.Writer.Header().Set( + "Content-Disposition", + attachment) +} + // // REST resource. type Resource struct { diff --git a/api/bucket.go b/api/bucket.go index 08246f68e..2ca45908f 100644 --- a/api/bucket.go +++ b/api/bucket.go @@ -1,22 +1,15 @@ package api import ( - "archive/tar" - "bufio" - "bytes" - "compress/gzip" - "fmt" "github.com/gin-gonic/gin" "github.com/gin-gonic/gin/binding" - liberr "github.com/jortel/go-utils/error" "github.com/konveyor/tackle2-hub/model" "github.com/konveyor/tackle2-hub/nas" + "github.com/konveyor/tackle2-hub/tar" "io" "net/http" "os" pathlib "path" - "path/filepath" - "strings" "time" ) @@ -238,30 +231,17 @@ func (h *BucketOwner) bucketGet(ctx *gin.Context, id uint) { return } if st.IsDir() { - filter := DirFilter{ - pattern: ctx.Query(Filter), - root: path, + filter := tar.Filter{ + Pattern: ctx.Query(Filter), + Root: path, } if h.Accepted(ctx, binding.MIMEHTML) { - err = h.getFile(ctx, m) - if err != nil { - _ = ctx.Error(err) - } - return + h.getFile(ctx, m) } else { - err := h.getDir(ctx, path, filter) - if err != nil { - _ = ctx.Error(err) - return - } - h.Status(ctx, http.StatusOK) + h.getDir(ctx, path, filter) } } else { - err = h.getFile(ctx, m) - if err != nil { - _ = ctx.Error(err) - } - return + h.getFile(ctx, m) } } @@ -315,7 +295,6 @@ func (h *BucketOwner) putDir(ctx *gin.Context, output string) (err error) { file, err := ctx.FormFile(FileField) if err != nil { err = &BadRequestError{err.Error()} - _ = ctx.Error(err) return } fileReader, err := file.Open() @@ -327,142 +306,41 @@ func (h *BucketOwner) putDir(ctx *gin.Context, output string) (err error) { defer func() { _ = fileReader.Close() }() - zipReader, err := gzip.NewReader(fileReader) - if err != nil { - err = &BadRequestError{err.Error()} - _ = ctx.Error(err) - return - } - defer func() { - _ = zipReader.Close() - }() err = nas.RmDir(output) if err != nil { return } - err = os.MkdirAll(output, 0777) - if err != nil { - err = liberr.Wrap(err) - return - } - tarReader := tar.NewReader(zipReader) - for { - header, nErr := tarReader.Next() - if nErr != nil { - if nErr == io.EOF { - break - } else { - err = liberr.Wrap(nErr) - return - } - } - switch header.Typeflag { - case tar.TypeDir: - path := pathlib.Join(output, header.Name) - err = os.Mkdir(path, 0777) - if err != nil { - err = liberr.Wrap(err) - return - } - case tar.TypeReg: - path := pathlib.Join(output, header.Name) - file, nErr := os.Create(path) - if nErr != nil { - err = liberr.Wrap(nErr) - return - } - _, err = io.Copy(file, tarReader) - if err != nil { - err = liberr.Wrap(err) - return - } - _ = file.Close() - } - } + tarReader := tar.NewReader() + err = tarReader.Extract(output, fileReader) return } // // getDir reads a directory from the bucket. -func (h *BucketOwner) getDir(ctx *gin.Context, input string, filter DirFilter) (err error) { - var tarOutput bytes.Buffer - tarWriter := tar.NewWriter(&tarOutput) - err = filepath.Walk( - input, - func(path string, info os.FileInfo, wErr error) (err error) { - if wErr != nil { - err = liberr.Wrap(wErr) - return - } - if path == input { - return - } - if !filter.Match(path) { - return - } - header, err := tar.FileInfoHeader(info, path) - if err != nil { - err = liberr.Wrap(err) - return - } - header.Name = strings.Replace(path, input, "", 1) - switch header.Typeflag { - case tar.TypeDir: - err = tarWriter.WriteHeader(header) - if err != nil { - err = liberr.Wrap(err) - return - } - case tar.TypeReg: - err = tarWriter.WriteHeader(header) - if err != nil { - err = liberr.Wrap(err) - return - } - file, nErr := os.Open(path) - if err != nil { - err = liberr.Wrap(nErr) - return - } - defer func() { - _ = file.Close() - }() - _, err = io.Copy(tarWriter, file) - if err != nil { - err = liberr.Wrap(err) - return - } - } - return - }) - if err != nil { - return - } - err = tarWriter.Close() +func (h *BucketOwner) getDir(ctx *gin.Context, input string, filter tar.Filter) { + tarWriter := tar.NewWriter(ctx.Writer) + tarWriter.Filter = filter + defer func() { + tarWriter.Close() + }() + err := tarWriter.AssertDir(input) if err != nil { - err = liberr.Wrap(err) + _ = ctx.Error(err) return + } else { + h.Attachment(ctx, pathlib.Base(input)+".tar.gz") + ctx.Status(http.StatusOK) } - ctx.Writer.Header().Set( - "Content-Disposition", - fmt.Sprintf("attachment; filename=\"%s\"", pathlib.Base(input)+".tar.gz")) - ctx.Writer.Header().Set(Directory, DirectoryExpand) - zipReader := bufio.NewReader(&tarOutput) - zipWriter := gzip.NewWriter(ctx.Writer) - defer func() { - _ = zipWriter.Close() - }() - _, err = io.Copy(zipWriter, zipReader) + _ = tarWriter.AddDir(input) return } // // getFile reads a file from the bucket. -func (h *BucketOwner) getFile(ctx *gin.Context, m *model.Bucket) (err error) { +func (h *BucketOwner) getFile(ctx *gin.Context, m *model.Bucket) { rPath := ctx.Param(Wildcard) path := pathlib.Join(m.Path, rPath) ctx.File(path) - return } // @@ -502,29 +380,3 @@ func (h *BucketOwner) putFile(ctx *gin.Context, m *model.Bucket) (err error) { err = os.Chmod(path, 0666) return } - -// -// DirFilter supports glob-style filtering. -type DirFilter struct { - root string - pattern string - cache map[string]bool -} - -// -// Match determines if path matches the filter. -func (r *DirFilter) Match(path string) (b bool) { - if r.pattern == "" { - b = true - return - } - if r.cache == nil { - r.cache = map[string]bool{} - matches, _ := filepath.Glob(pathlib.Join(r.root, r.pattern)) - for _, p := range matches { - r.cache[p] = true - } - } - _, b = r.cache[path] - return -} diff --git a/api/import.go b/api/import.go index 62bd9e6f3..f4d969738 100644 --- a/api/import.go +++ b/api/import.go @@ -3,7 +3,6 @@ package api import ( "bytes" "encoding/csv" - "fmt" "github.com/gin-gonic/gin" "github.com/konveyor/tackle2-hub/model" "io" @@ -315,7 +314,7 @@ func (h ImportHandler) DownloadCSV(ctx *gin.Context) { _ = ctx.Error(result.Error) return } - ctx.Writer.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", m.Filename)) + h.Attachment(ctx, m.Filename) ctx.Data(http.StatusOK, "text/csv", m.Content) } diff --git a/api/pkg.go b/api/pkg.go index b5b3e7b0f..5f3eff137 100644 --- a/api/pkg.go +++ b/api/pkg.go @@ -40,6 +40,7 @@ const ( // MIME Types. const ( MIMEOCTETSTREAM = "application/octet-stream" + TAR = "application/x-tar" ) // diff --git a/binding/client.go b/binding/client.go index 13729bf8b..40940a91d 100644 --- a/binding/client.go +++ b/binding/client.go @@ -1,10 +1,7 @@ package binding import ( - "archive/tar" - "bufio" "bytes" - "compress/gzip" "encoding/json" "errors" "fmt" @@ -12,6 +9,7 @@ import ( liberr "github.com/jortel/go-utils/error" "github.com/konveyor/tackle2-hub/api" qf "github.com/konveyor/tackle2-hub/binding/filter" + "github.com/konveyor/tackle2-hub/tar" "io" "mime/multipart" "net" @@ -20,7 +18,6 @@ import ( "net/url" "os" pathlib "path" - "path/filepath" "strings" "time" ) @@ -539,106 +536,17 @@ func (r *Client) FileSend(path, method string, fields []Field, object interface{ // // getDir downloads and expands a directory. func (r *Client) getDir(body io.Reader, output string) (err error) { - zipReader, err := gzip.NewReader(body) - if err != nil { - err = liberr.Wrap(err) - return - } - defer func() { - _ = zipReader.Close() - }() - tarReader := tar.NewReader(zipReader) - for { - header, nErr := tarReader.Next() - if nErr != nil { - if nErr == io.EOF { - break - } else { - err = liberr.Wrap(nErr) - return - } - } - path := pathlib.Join(output, header.Name) - switch header.Typeflag { - case tar.TypeDir: - err = os.Mkdir(path, 0777) - if err != nil { - err = liberr.Wrap(err) - return - } - case tar.TypeReg: - file, nErr := os.Create(path) - if nErr != nil { - err = liberr.Wrap(nErr) - return - } - _, err = io.Copy(file, tarReader) - _ = file.Close() - default: - } - } + tarReader := tar.NewReader() + err = tarReader.Extract(output, body) return } // // putDir archive and uploads a directory. func (r *Client) putDir(writer io.Writer, input string) (err error) { - var tarOutput bytes.Buffer - tarWriter := tar.NewWriter(&tarOutput) - err = filepath.Walk( - input, - func(path string, entry os.FileInfo, wErr error) (err error) { - if wErr != nil { - err = liberr.Wrap(wErr) - return - } - if path == input { - return - } - header, nErr := tar.FileInfoHeader(entry, "") - if nErr != nil { - err = liberr.Wrap(nErr) - return - } - header.Name = path[len(input)+1:] - switch header.Typeflag { - case tar.TypeDir: - err = tarWriter.WriteHeader(header) - if err != nil { - err = liberr.Wrap(err) - return - } - case tar.TypeReg: - err = tarWriter.WriteHeader(header) - if err != nil { - err = liberr.Wrap(err) - return - } - file, nErr := os.Open(path) - if err != nil { - err = liberr.Wrap(nErr) - return - } - defer func() { - _ = file.Close() - }() - _, err = io.Copy(tarWriter, file) - if err != nil { - err = liberr.Wrap(err) - return - } - } - return - }) - if err != nil { - return - } - zipReader := bufio.NewReader(&tarOutput) - zipWriter := gzip.NewWriter(writer) - defer func() { - _ = zipWriter.Close() - }() - _, err = io.Copy(zipWriter, zipReader) + tarWriter := tar.NewWriter(writer) + defer tarWriter.Close() + err = tarWriter.AddDir(input) return } diff --git a/hack/add/all.sh b/hack/add/all.sh index e4efab48d..d6af4c827 100755 --- a/hack/add/all.sh +++ b/hack/add/all.sh @@ -17,3 +17,4 @@ cd ${dir} ./business-service.sh ./application.sh ./review.sh +./taskgroup.sh diff --git a/hack/add/bucket.sh b/hack/add/bucket.sh index e27340cc4..073f34f89 100755 --- a/hack/add/bucket.sh +++ b/hack/add/bucket.sh @@ -1,29 +1,29 @@ #!/bin/bash host="${HOST:-localhost:8080}" - -path="/etc/hosts" +id="${1:-1}" +path="${2:-/etc/hosts}" echo "______________________ BUCKET ______________________" -curl -F 'file=@/etc/hosts' ${host}/buckets/1/${path} -curl -X GET ${host}/buckets/1/${path} -curl -X DELETE ${host}/buckets/1/${path} -curl -X GET ${host}/buckets/1/${path} +curl -F "file=@${path}" ${host}/buckets/${id}/${path} +curl --output /tmp/get-bucket -X GET ${host}/buckets/${id}/${path} +curl -X DELETE ${host}/buckets/${id}/${path} +curl -X GET ${host}/buckets/${id}/${path} echo "______________________ APP ______________________" -curl -F 'file=@/etc/hosts' ${host}/applications/1/bucket/${path} -curl -X GET ${host}/applications/1/bucket/${path} -curl -X DELETE ${host}/applications/1/bucket/${path} -curl -X GET ${host}/applications/1/bucket/${path} +curl -F "file=@${path}" ${host}/applications/${id}/bucket/${path} +curl --output /tmp/get-bucket-app -X GET ${host}/applications/${id}/bucket/${path} +curl -X DELETE ${host}/applications/${id}/bucket/${path} +curl -X GET ${host}/applications/${id}/bucket/${path} echo "______________________ TASK ______________________" -curl -F 'file=@/etc/hosts' ${host}/tasks/1/bucket/${path} -curl -X GET ${host}/tasks/1/bucket/${path} -curl -X DELETE ${host}/tasks/1/bucket/${path} -curl -X GET ${host}/tasks/1/bucket/${path} +curl -F "file=@${path}" ${host}/tasks/${id}/bucket/${path} +curl --output /tmp/get-bucket-task -X GET ${host}/tasks/${id}/bucket/${path} +curl -X DELETE ${host}/tasks/${id}/bucket/${path} +curl -X GET ${host}/tasks/${id}/bucket/${path} echo "______________________ TASKGROUP ______________________" -curl -F 'file=@/etc/hosts' ${host}/taskgroups/1/bucket/${path} -curl -X GET ${host}/taskgroups/1/bucket/${path} -curl -X DELETE ${host}/taskgroups/1/bucket/${path} -curl -X GET ${host}/taskgroups/1/bucket/${path} +curl -F "file=@${path}" ${host}/taskgroups/${id}/bucket/${path} +curl --output /tmp/get-bucket-tg -X GET ${host}/taskgroups/${id}/bucket/${path} +curl -X DELETE ${host}/taskgroups/${id}/bucket/${path} +curl -X GET ${host}/taskgroups/${id}/bucket/${path} diff --git a/settings/hub.go b/settings/hub.go index 6947afc0b..ab27bde8f 100644 --- a/settings/hub.go +++ b/settings/hub.go @@ -26,6 +26,7 @@ const ( EnvFileTTL = "FILE_TTL" EnvAppName = "APP_NAME" EnvDisconnected = "DISCONNECTED" + EnvReportPath = "REPORT_PATH" ) type Hub struct { @@ -77,6 +78,10 @@ type Hub struct { Product bool // Disconnected indicates no cluster. Disconnected bool + // Report settings. + Report struct { + Path string + } } func (r *Hub) Load() (err error) { @@ -190,6 +195,10 @@ func (r *Hub) Load() (err error) { b, _ := strconv.ParseBool(s) r.Disconnected = b } + r.Report.Path, found = os.LookupEnv(EnvReportPath) + if !found { + r.Report.Path = "/tmp/report" + } return } diff --git a/tar/filter.go b/tar/filter.go new file mode 100644 index 000000000..36b683fe3 --- /dev/null +++ b/tar/filter.go @@ -0,0 +1,32 @@ +package tar + +import ( + pathlib "path" + "path/filepath" +) + +// +// Filter supports glob-style filtering. +type Filter struct { + Root string + Pattern string + cache map[string]bool +} + +// +// Match determines if path matches the filter. +func (r *Filter) Match(path string) (b bool) { + if r.Pattern == "" { + b = true + return + } + if r.cache == nil { + r.cache = map[string]bool{} + matches, _ := filepath.Glob(pathlib.Join(r.Root, r.Pattern)) + for _, p := range matches { + r.cache[p] = true + } + } + _, b = r.cache[path] + return +} diff --git a/tar/reader.go b/tar/reader.go new file mode 100644 index 000000000..91b056b3d --- /dev/null +++ b/tar/reader.go @@ -0,0 +1,79 @@ +package tar + +import ( + "archive/tar" + "compress/gzip" + liberr "github.com/jortel/go-utils/error" + "github.com/konveyor/tackle2-hub/nas" + "io" + "os" + pathlib "path" +) + +// +// NewReader returns a new reader. +func NewReader() (reader *Reader) { + reader = &Reader{} + return +} + +// +// Reader archive reader. +type Reader struct { + Filter Filter +} + +// +// Extract archive content to the destination path. +func (r *Reader) Extract(outDir string, reader io.Reader) (err error) { + zipReader, err := gzip.NewReader(reader) + if err != nil { + err = liberr.Wrap(err) + return + } + defer func() { + _ = zipReader.Close() + }() + err = os.MkdirAll(outDir, 0777) + if err != nil { + err = liberr.Wrap(err) + return + } + tarReader := tar.NewReader(zipReader) + for { + header, nErr := tarReader.Next() + if nErr != nil { + if nErr == io.EOF { + break + } else { + err = liberr.Wrap(nErr) + return + } + } + path := pathlib.Join(outDir, header.Name) + if !r.Filter.Match(path) { + return + } + switch header.Typeflag { + case tar.TypeDir: + err = nas.MkDir(path, os.FileMode(header.Mode)) + if err != nil { + err = liberr.Wrap(err) + return + } + case tar.TypeReg: + file, nErr := os.Create(path) + if nErr != nil { + err = liberr.Wrap(nErr) + return + } + _, err = io.Copy(file, tarReader) + if err != nil { + err = liberr.Wrap(err) + return + } + _ = file.Close() + } + } + return +} diff --git a/tar/writer.go b/tar/writer.go new file mode 100644 index 000000000..8b55708d4 --- /dev/null +++ b/tar/writer.go @@ -0,0 +1,238 @@ +package tar + +import ( + "archive/tar" + "compress/gzip" + liberr "github.com/jortel/go-utils/error" + "io" + "os" + "path/filepath" + "runtime" + "strings" +) + +// +// NewWriter returns a new writer. +func NewWriter(output io.Writer) (writer *Writer) { + writer = &Writer{} + writer.Open(output) + runtime.SetFinalizer( + writer, + func(r *Writer) { + r.Close() + }) + return +} + +// +// Writer is a Zipped TAR streamed writer. +type Writer struct { + Filter Filter + // + drained chan int + tarWriter *tar.Writer + bridge struct { + reader *io.PipeReader + writer *io.PipeWriter + } +} + +// +// Open the writer. +func (r *Writer) Open(output io.Writer) { + if r.tarWriter != nil { + return + } + r.drained = make(chan int) + r.bridge.reader, r.bridge.writer = io.Pipe() + r.tarWriter = tar.NewWriter(r.bridge.writer) + zipWriter := gzip.NewWriter(output) + go func() { + defer func() { + _ = zipWriter.Close() + r.drained <- 0 + }() + _, _ = io.Copy(zipWriter, r.bridge.reader) + }() +} + +// +// AssertDir validates the path is a readable directory. +func (r *Writer) AssertDir(pathIn string) (err error) { + st, err := os.Stat(pathIn) + if err != nil { + err = liberr.Wrap(err) + return + } + if !st.IsDir() { + err = liberr.New("Directory path expected.") + return + } + err = filepath.Walk( + pathIn, + func(path string, info os.FileInfo, nErr error) (err error) { + if nErr != nil { + err = liberr.Wrap(nErr) + return + } + if path == pathIn { + return + } + if !r.Filter.Match(path) { + return + } + header, err := tar.FileInfoHeader(info, "") + if err != nil { + err = liberr.Wrap(err) + return + } + switch header.Typeflag { + case tar.TypeReg: + f, nErr := os.Open(path) + if nErr != nil { + err = liberr.Wrap(nErr) + return + } + _ = f.Close() + } + return + }) + return +} + +// +// AddDir adds a directory. +func (r *Writer) AddDir(pathIn string) (err error) { + if r.tarWriter == nil { + err = liberr.New("Writer not open.") + return + } + err = r.AssertDir(pathIn) + if err != nil { + return + } + err = filepath.Walk( + pathIn, + func(path string, info os.FileInfo, nErr error) (err error) { + if nErr != nil { + err = liberr.Wrap(nErr) + return + } + if path == pathIn { + return + } + if !r.Filter.Match(path) { + return + } + header, err := tar.FileInfoHeader(info, "") + if err != nil { + err = liberr.Wrap(err) + return + } + header.Name = strings.Replace(path, pathIn, "", 1) + switch header.Typeflag { + case tar.TypeDir: + err = r.tarWriter.WriteHeader(header) + if err != nil { + err = liberr.Wrap(err) + return + } + case tar.TypeReg: + err = r.tarWriter.WriteHeader(header) + if err != nil { + err = liberr.Wrap(err) + return + } + file, nErr := os.Open(path) + if err != nil { + err = liberr.Wrap(nErr) + return + } + defer func() { + _ = file.Close() + }() + _, err = io.Copy(r.tarWriter, file) + if err != nil { + err = liberr.Wrap(err) + return + } + } + return + }) + return +} + +// +// AssertFile validates the path is a readable file. +func (r *Writer) AssertFile(pathIn string) (err error) { + st, err := os.Stat(pathIn) + if err != nil { + err = liberr.Wrap(err) + return + } + if st.IsDir() { + err = liberr.New("File path expected.") + return + } + f, nErr := os.Open(pathIn) + if nErr != nil { + err = liberr.Wrap(nErr) + return + } + _ = f.Close() + return +} + +// +// AddFile adds a file. +func (r *Writer) AddFile(pathIn, destPath string) (err error) { + if r.tarWriter == nil { + err = liberr.New("Writer not open.") + return + } + err = r.AssertFile(pathIn) + if err != nil { + return + } + st, err := os.Stat(pathIn) + if err != nil { + err = liberr.Wrap(err) + return + } + header, err := tar.FileInfoHeader(st, "") + if err != nil { + err = liberr.Wrap(err) + return + } + header.Name = destPath + err = r.tarWriter.WriteHeader(header) + if err != nil { + err = liberr.Wrap(err) + return + } + file, nErr := os.Open(pathIn) + if err != nil { + err = liberr.Wrap(nErr) + return + } + defer func() { + _ = file.Close() + }() + _, err = io.Copy(r.tarWriter, file) + if err != nil { + err = liberr.Wrap(err) + return + } + return +} + +// +// Close the writer. +func (r *Writer) Close() { + if r.tarWriter == nil { + return + } + _ = r.tarWriter.Close() + _ = r.bridge.writer.Close() + _ = <-r.drained +} diff --git a/test/tar/data/pet/cat b/test/tar/data/pet/cat new file mode 100644 index 000000000..74b27d208 --- /dev/null +++ b/test/tar/data/pet/cat @@ -0,0 +1,2 @@ +CAT + diff --git a/test/tar/data/pet/dog b/test/tar/data/pet/dog new file mode 100644 index 000000000..83d4b5624 --- /dev/null +++ b/test/tar/data/pet/dog @@ -0,0 +1,2 @@ +DOG + diff --git a/test/tar/data/rabbit b/test/tar/data/rabbit new file mode 100644 index 000000000..fb5f14680 --- /dev/null +++ b/test/tar/data/rabbit @@ -0,0 +1 @@ +RABBIT \ No newline at end of file diff --git a/test/tar/data/wild/lion b/test/tar/data/wild/lion new file mode 100644 index 000000000..0ad0653f5 --- /dev/null +++ b/test/tar/data/wild/lion @@ -0,0 +1,2 @@ +LION + diff --git a/test/tar/data/wild/tiger b/test/tar/data/wild/tiger new file mode 100644 index 000000000..a15cd4695 --- /dev/null +++ b/test/tar/data/wild/tiger @@ -0,0 +1 @@ +TIGER \ No newline at end of file diff --git a/test/tar/tar_test.go b/test/tar/tar_test.go new file mode 100644 index 000000000..815ed237c --- /dev/null +++ b/test/tar/tar_test.go @@ -0,0 +1,65 @@ +package tar + +import ( + uuid2 "github.com/google/uuid" + liberr "github.com/jortel/go-utils/error" + "github.com/konveyor/tackle2-hub/nas" + "github.com/konveyor/tackle2-hub/tar" + "github.com/konveyor/tackle2-hub/test/assert" + "github.com/onsi/gomega" + "os" + "path" + "path/filepath" + "testing" +) + +func TestWriter(t *testing.T) { + g := gomega.NewGomegaWithT(t) + // Setup + uuid, _ := uuid2.NewUUID() + tmpDir := "/tmp/" + uuid.String() + err := nas.MkDir(tmpDir, 0755) + g.Expect(err).To(gomega.BeNil()) + defer func() { + _ = nas.RmDir(tmpDir) + }() + outPath := path.Join(tmpDir, "output.tar.gz") + file, err := os.Create(outPath) + g.Expect(err).To(gomega.BeNil()) + + // Write the ./data tree. + writer := tar.NewWriter(file) + err = writer.AddDir("./data") + g.Expect(err).To(gomega.BeNil()) + + // Write ./data/rabbit => data/pet/rabbit + err = writer.AddFile("./data/rabbit", "data/pet/rabbit") + g.Expect(err).To(gomega.BeNil()) + writer.Close() + _ = file.Close() + + // Read/expand the tarball. + reader := tar.NewReader() + file, err = os.Open(outPath) + g.Expect(err).To(gomega.BeNil()) + err = reader.Extract(tmpDir, file) + g.Expect(err).To(gomega.BeNil()) + + // Validate ./data + err = filepath.Walk( + path.Join("./data"), + func(p string, info os.FileInfo, nErr error) (err error) { + if nErr != nil { + err = liberr.Wrap(nErr) + return + } + if !info.IsDir() { + _ = assert.EqualFileContent(p, path.Join(tmpDir, p)) + } + return + }) + g.Expect(err).To(gomega.BeNil()) + + // Validate ./data/pet/rabbit + _ = assert.EqualFileContent("./data/rabbit", path.Join(tmpDir, "data", "pet", "rabbit")) +} From d8b40c986c049d325d45700e37300e7c08132322 Mon Sep 17 00:00:00 2001 From: Jeff Ortel Date: Fri, 25 Aug 2023 07:39:53 -0700 Subject: [PATCH 2/7] route: /analysis/report Signed-off-by: Jeff Ortel --- api/analysis.go | 116 +++++++++++++++++++++++++++--------------------- 1 file changed, 66 insertions(+), 50 deletions(-) diff --git a/api/analysis.go b/api/analysis.go index 1b00ac6d9..aa618e72b 100644 --- a/api/analysis.go +++ b/api/analysis.go @@ -45,6 +45,7 @@ const ( // AppAnalysesRoot = ApplicationRoot + "/analyses" AppAnalysisRoot = ApplicationRoot + "/analysis" + AppAnalysisReportRoot = AppAnalysisRoot + "/report" AppAnalysisDepsRoot = AppAnalysisRoot + "/dependencies" AppAnalysisIssuesRoot = AppAnalysisRoot + "/issues" ) @@ -84,6 +85,7 @@ func (h AnalysisHandler) AddRoutes(e *gin.Engine) { routeGroup.POST(AppAnalysesRoot, h.AppCreate) routeGroup.GET(AppAnalysesRoot, h.AppList) routeGroup.GET(AppAnalysisRoot, h.AppLatest) + routeGroup.GET(AppAnalysisReportRoot, h.AppLatestReport) routeGroup.GET(AppAnalysisDepsRoot, h.AppDeps) routeGroup.GET(AppAnalysisIssuesRoot, h.AppIssues) } @@ -114,13 +116,43 @@ func (h AnalysisHandler) Get(ctx *gin.Context) { // AppLatest godoc // @summary Get the latest analysis. // @description Get the latest analysis for an application. -// @description When Accept: is application/x-tar, the static report returned. // @tags analyses // @produce octet-stream // @success 200 {object} api.Analysis // @router /applications/{id}/analysis [get] // @param id path string true "Application ID" func (h AnalysisHandler) AppLatest(ctx *gin.Context) { + id := h.pk(ctx) + m := &model.Analysis{} + db := h.DB(ctx) + db = db.Where("ApplicationID", id) + err := db.Last(&m).Error + if err != nil { + _ = ctx.Error(err) + return + } + writer := AnalysisWriter{ctx: ctx} + path, err := writer.Create(id) + if err != nil { + _ = ctx.Error(err) + return + } + defer func() { + _ = os.Remove(path) + }() + h.Status(ctx, http.StatusOK) + ctx.File(path) +} + +// AppLatestReport godoc +// @summary Get the latest analysis (static) report. +// @description Get the latest analysis (static) report. +// @tags analyses +// @produce octet-stream +// @success 200 +// @router /applications/{id}/analysis/report [get] +// @param id path string true "Application ID" +func (h AnalysisHandler) AppLatestReport(ctx *gin.Context) { id := h.pk(ctx) m := &model.Analysis{} db := h.DB(ctx) @@ -133,50 +165,34 @@ func (h AnalysisHandler) AppLatest(ctx *gin.Context) { _ = ctx.Error(err) return } - offered := append(BindMIMEs, TAR) - accepted := ctx.NegotiateFormat(offered...) - if accepted == TAR { - bundleWriter := BundleWriter{} - bundleWriter.ctx = ctx - path, err := bundleWriter.Create(id) - if err != nil { - _ = ctx.Error(err) - return - } - defer func() { - _ = os.Remove(path) - }() - tarWriter := tar.NewWriter(ctx.Writer) - defer func() { - tarWriter.Close() - }() - err = tarWriter.AssertDir(Settings.Report.Path) - if err != nil { - _ = ctx.Error(err) - return - } - err = tarWriter.AssertFile(path) - if err != nil { - _ = ctx.Error(err) - return - } - h.Attachment(ctx, fmt.Sprintf("%s.tar.gz", m.Application.Name)) - ctx.Status(http.StatusOK) - _ = tarWriter.AddDir(Settings.Report.Path) - _ = tarWriter.AddFile(path, "output.js") - } else { - writer := AnalysisWriter{ctx: ctx} - path, err := writer.Create(id) - if err != nil { - _ = ctx.Error(err) - return - } - defer func() { - _ = os.Remove(path) - }() - h.Status(ctx, http.StatusOK) - ctx.File(path) + reportWriter := ReportWriter{} + reportWriter.ctx = ctx + path, err := reportWriter.Create(id) + if err != nil { + _ = ctx.Error(err) + return + } + defer func() { + _ = os.Remove(path) + }() + tarWriter := tar.NewWriter(ctx.Writer) + defer func() { + tarWriter.Close() + }() + err = tarWriter.AssertDir(Settings.Report.Path) + if err != nil { + _ = ctx.Error(err) + return + } + err = tarWriter.AssertFile(path) + if err != nil { + _ = ctx.Error(err) + return } + h.Attachment(ctx, fmt.Sprintf("%s.tar.gz", m.Application.Name)) + ctx.Status(http.StatusOK) + _ = tarWriter.AddDir(Settings.Report.Path) + _ = tarWriter.AddFile(path, "output.js") } // AppList godoc @@ -2192,14 +2208,14 @@ func (r *AnalysisWriter) addDeps(m *model.Analysis) (err error) { } // -// BundleWriter analysis (static) report bundle writer. -type BundleWriter struct { +// ReportWriter analysis (static) report report writer. +type ReportWriter struct { AnalysisWriter } // // Create the output.js file. -func (r *BundleWriter) Create(id uint) (path string, err error) { +func (r *ReportWriter) Create(id uint) (path string, err error) { path = fmt.Sprintf("/tmp/ouput-%d.js", rand.Int()) file, err := os.Create(path) if err != nil { @@ -2213,8 +2229,8 @@ func (r *BundleWriter) Create(id uint) (path string, err error) { } // -// Write the bundle output.js file. -func (r *BundleWriter) Write(id uint, output io.Writer) (err error) { +// Write the report output.js file. +func (r *ReportWriter) Write(id uint, output io.Writer) (err error) { m := &model.Analysis{} db := r.db() db = db.Preload("Application") @@ -2248,7 +2264,7 @@ func (r *BundleWriter) Write(id uint, output io.Writer) (err error) { // // addTags writes tags. -func (r *BundleWriter) addTags(m *model.Analysis) (err error) { +func (r *ReportWriter) addTags(m *model.Analysis) (err error) { r.field("tags") r.beginList() for i := range m.Application.Tags { From 512c28431f86b486eb66eb719c71b4e3fb3b1a0b Mon Sep 17 00:00:00 2001 From: Jeff Ortel Date: Fri, 25 Aug 2023 07:55:56 -0700 Subject: [PATCH 3/7] checkpoint Signed-off-by: Jeff Ortel --- Dockerfile | 2 +- api/analysis.go | 5 +++-- settings/hub.go | 54 +++++++++++++++++++++++++------------------------ 3 files changed, 32 insertions(+), 29 deletions(-) diff --git a/Dockerfile b/Dockerfile index 81486cf83..a29319844 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ COPY --from=builder /opt/app-root/src/bin/hub /usr/local/bin/tackle-hub COPY --from=builder /opt/app-root/src/auth/roles.yaml /tmp/roles.yaml COPY --from=builder /opt/app-root/src/auth/users.yaml /tmp/users.yaml COPY --from=builder /opt/app-root/src/tackle2-seed/resources/ /tmp/seed -COPY --from=report /usr/local/static-report /tmp/report +COPY --from=report /usr/local/static-report /tmp/analysis/report RUN microdnf -y install \ sqlite \ diff --git a/api/analysis.go b/api/analysis.go index aa618e72b..f1d72d92e 100644 --- a/api/analysis.go +++ b/api/analysis.go @@ -175,11 +175,12 @@ func (h AnalysisHandler) AppLatestReport(ctx *gin.Context) { defer func() { _ = os.Remove(path) }() + report := Settings.Analysis.Report.Path tarWriter := tar.NewWriter(ctx.Writer) defer func() { tarWriter.Close() }() - err = tarWriter.AssertDir(Settings.Report.Path) + err = tarWriter.AssertDir(report) if err != nil { _ = ctx.Error(err) return @@ -191,7 +192,7 @@ func (h AnalysisHandler) AppLatestReport(ctx *gin.Context) { } h.Attachment(ctx, fmt.Sprintf("%s.tar.gz", m.Application.Name)) ctx.Status(http.StatusOK) - _ = tarWriter.AddDir(Settings.Report.Path) + _ = tarWriter.AddDir(report) _ = tarWriter.AddFile(path, "output.js") } diff --git a/settings/hub.go b/settings/hub.go index ab27bde8f..bdd2e742d 100644 --- a/settings/hub.go +++ b/settings/hub.go @@ -6,27 +6,27 @@ import ( ) const ( - EnvNamespace = "NAMESPACE" - EnvDbPath = "DB_PATH" - EnvDbSeedPath = "DB_SEED_PATH" - EnvBucketPath = "BUCKET_PATH" - EnvRwxSupported = "RWX_SUPPORTED" - EnvCachePath = "CACHE_PATH" - EnvCachePvc = "CACHE_PVC" - EnvPassphrase = "ENCRYPTION_PASSPHRASE" - EnvTaskReapCreated = "TASK_REAP_CREATED" - EnvTaskReapSucceeded = "TASK_REAP_SUCCEEDED" - EnvTaskReapFailed = "TASK_REAP_FAILED" - EnvTaskSA = "TASK_SA" - EnvTaskRetries = "TASK_RETRIES" - EnvFrequencyTask = "FREQUENCY_TASK" - EnvFrequencyReaper = "FREQUENCY_REAPER" - EnvDevelopment = "DEVELOPMENT" - EnvBucketTTL = "BUCKET_TTL" - EnvFileTTL = "FILE_TTL" - EnvAppName = "APP_NAME" - EnvDisconnected = "DISCONNECTED" - EnvReportPath = "REPORT_PATH" + EnvNamespace = "NAMESPACE" + EnvDbPath = "DB_PATH" + EnvDbSeedPath = "DB_SEED_PATH" + EnvBucketPath = "BUCKET_PATH" + EnvRwxSupported = "RWX_SUPPORTED" + EnvCachePath = "CACHE_PATH" + EnvCachePvc = "CACHE_PVC" + EnvPassphrase = "ENCRYPTION_PASSPHRASE" + EnvTaskReapCreated = "TASK_REAP_CREATED" + EnvTaskReapSucceeded = "TASK_REAP_SUCCEEDED" + EnvTaskReapFailed = "TASK_REAP_FAILED" + EnvTaskSA = "TASK_SA" + EnvTaskRetries = "TASK_RETRIES" + EnvFrequencyTask = "FREQUENCY_TASK" + EnvFrequencyReaper = "FREQUENCY_REAPER" + EnvDevelopment = "DEVELOPMENT" + EnvBucketTTL = "BUCKET_TTL" + EnvFileTTL = "FILE_TTL" + EnvAppName = "APP_NAME" + EnvDisconnected = "DISCONNECTED" + EnvAnalysisReportPath = "ANALYSIS_REPORT_PATH" ) type Hub struct { @@ -78,9 +78,11 @@ type Hub struct { Product bool // Disconnected indicates no cluster. Disconnected bool - // Report settings. - Report struct { - Path string + // Analysis settings. + Analysis struct { + Report struct { + Path string + } } } @@ -195,9 +197,9 @@ func (r *Hub) Load() (err error) { b, _ := strconv.ParseBool(s) r.Disconnected = b } - r.Report.Path, found = os.LookupEnv(EnvReportPath) + r.Analysis.Report.Path, found = os.LookupEnv(EnvAnalysisReportPath) if !found { - r.Report.Path = "/tmp/report" + r.Analysis.Report.Path = "/tmp/analysis/report" } return From e689cec86fd15e0fb3d5d06fd52df3671d99ac56 Mon Sep 17 00:00:00 2001 From: Jeff Ortel Date: Fri, 25 Aug 2023 07:57:32 -0700 Subject: [PATCH 4/7] checkpoint Signed-off-by: Jeff Ortel --- api/analysis.go | 2 +- settings/hub.go | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/api/analysis.go b/api/analysis.go index f1d72d92e..38bb038db 100644 --- a/api/analysis.go +++ b/api/analysis.go @@ -175,7 +175,7 @@ func (h AnalysisHandler) AppLatestReport(ctx *gin.Context) { defer func() { _ = os.Remove(path) }() - report := Settings.Analysis.Report.Path + report := Settings.Analysis.ReportPath tarWriter := tar.NewWriter(ctx.Writer) defer func() { tarWriter.Close() diff --git a/settings/hub.go b/settings/hub.go index bdd2e742d..d7d984e87 100644 --- a/settings/hub.go +++ b/settings/hub.go @@ -80,9 +80,7 @@ type Hub struct { Disconnected bool // Analysis settings. Analysis struct { - Report struct { - Path string - } + ReportPath string } } @@ -197,9 +195,9 @@ func (r *Hub) Load() (err error) { b, _ := strconv.ParseBool(s) r.Disconnected = b } - r.Analysis.Report.Path, found = os.LookupEnv(EnvAnalysisReportPath) + r.Analysis.ReportPath, found = os.LookupEnv(EnvAnalysisReportPath) if !found { - r.Analysis.Report.Path = "/tmp/analysis/report" + r.Analysis.ReportPath = "/tmp/analysis/report" } return From 8c859458424bff4f82bbcb28023d33450e6d81a7 Mon Sep 17 00:00:00 2001 From: Jeff Ortel Date: Fri, 25 Aug 2023 09:28:52 -0700 Subject: [PATCH 5/7] Refactor ReportWriter and AppLatestReport() responsibilities. Signed-off-by: Jeff Ortel --- api/analysis.go | 103 +++++++++++++++++++++++------------------------- 1 file changed, 50 insertions(+), 53 deletions(-) diff --git a/api/analysis.go b/api/analysis.go index 38bb038db..0cd708e25 100644 --- a/api/analysis.go +++ b/api/analysis.go @@ -154,46 +154,8 @@ func (h AnalysisHandler) AppLatest(ctx *gin.Context) { // @param id path string true "Application ID" func (h AnalysisHandler) AppLatestReport(ctx *gin.Context) { id := h.pk(ctx) - m := &model.Analysis{} - db := h.DB(ctx) - db = db.Preload("Application") - db = db.Preload("Application.Tags") - db = db.Preload("Application.Tags.Category") - db = db.Where("ApplicationID", id) - err := db.Last(&m).Error - if err != nil { - _ = ctx.Error(err) - return - } - reportWriter := ReportWriter{} - reportWriter.ctx = ctx - path, err := reportWriter.Create(id) - if err != nil { - _ = ctx.Error(err) - return - } - defer func() { - _ = os.Remove(path) - }() - report := Settings.Analysis.ReportPath - tarWriter := tar.NewWriter(ctx.Writer) - defer func() { - tarWriter.Close() - }() - err = tarWriter.AssertDir(report) - if err != nil { - _ = ctx.Error(err) - return - } - err = tarWriter.AssertFile(path) - if err != nil { - _ = ctx.Error(err) - return - } - h.Attachment(ctx, fmt.Sprintf("%s.tar.gz", m.Application.Name)) - ctx.Status(http.StatusOK) - _ = tarWriter.AddDir(report) - _ = tarWriter.AddFile(path, "output.js") + reportWriter := ReportWriter{ctx: ctx} + reportWriter.Write(id) } // AppList godoc @@ -2209,29 +2171,54 @@ func (r *AnalysisWriter) addDeps(m *model.Analysis) (err error) { } // -// ReportWriter analysis (static) report report writer. +// ReportWriter analysis report writer. type ReportWriter struct { - AnalysisWriter + encoder + ctx *gin.Context } // -// Create the output.js file. -func (r *ReportWriter) Create(id uint) (path string, err error) { - path = fmt.Sprintf("/tmp/ouput-%d.js", rand.Int()) - file, err := os.Create(path) +// db returns a db client. +func (r *ReportWriter) db() (db *gorm.DB) { + rtx := WithContext(r.ctx) + db = rtx.DB.Debug() + return +} + +// +// Write builds and streams the analysis report. +func (r *ReportWriter) Write(id uint) { + path, err := r.buildOutput(id) if err != nil { + _ = r.ctx.Error(err) return } defer func() { - _ = file.Close() + _ = os.Remove(path) }() - err = r.Write(id, file) + tarWriter := tar.NewWriter(r.ctx.Writer) + defer func() { + tarWriter.Close() + }() + err = tarWriter.AssertDir(Settings.Analysis.ReportPath) + if err != nil { + _ = r.ctx.Error(err) + return + } + err = tarWriter.AssertFile(path) + if err != nil { + _ = r.ctx.Error(err) + return + } + r.ctx.Status(http.StatusOK) + _ = tarWriter.AddDir(Settings.Analysis.ReportPath) + _ = tarWriter.AddFile(path, "output.js") return } // -// Write the report output.js file. -func (r *ReportWriter) Write(id uint, output io.Writer) (err error) { +// buildOutput creates the report output.js file. +func (r *ReportWriter) buildOutput(id uint) (path string, err error) { m := &model.Analysis{} db := r.db() db = db.Preload("Application") @@ -2241,16 +2228,26 @@ func (r *ReportWriter) Write(id uint, output io.Writer) (err error) { if err != nil { return } - r.encoder = &jsonEncoder{output: output} + path = fmt.Sprintf("/tmp/ouput-%d.js", rand.Int()) + file, err := os.Create(path) + if err != nil { + return + } + defer func() { + _ = file.Close() + }() + r.encoder = &jsonEncoder{output: file} r.write("window[\"apps\"]=[") r.begin() r.field("id").write(strconv.Itoa(int(m.Application.ID))) r.field("name").writeStr(m.Application.Name) - err = r.addIssues(m) + aWriter := AnalysisWriter{ctx: r.ctx} + aWriter.encoder = r.encoder + err = aWriter.addIssues(m) if err != nil { return } - err = r.addDeps(m) + err = aWriter.addDeps(m) if err != nil { return } From a572584aaf53ef9a56ea452bc094e51b7f7ff44a Mon Sep 17 00:00:00 2001 From: Jeff Ortel Date: Mon, 28 Aug 2023 09:29:47 -0700 Subject: [PATCH 6/7] Using os.CreateTemp(). Signed-off-by: Jeff Ortel --- api/analysis.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/analysis.go b/api/analysis.go index 0cd708e25..bef7c8d86 100644 --- a/api/analysis.go +++ b/api/analysis.go @@ -2228,14 +2228,14 @@ func (r *ReportWriter) buildOutput(id uint) (path string, err error) { if err != nil { return } - path = fmt.Sprintf("/tmp/ouput-%d.js", rand.Int()) - file, err := os.Create(path) + file, err := os.CreateTemp("", "output-*.js") if err != nil { return } defer func() { _ = file.Close() }() + path = file.Name() r.encoder = &jsonEncoder{output: file} r.write("window[\"apps\"]=[") r.begin() From 17368c83bcd48e00cb1aaafb9fe8ce58192c8c41 Mon Sep 17 00:00:00 2001 From: Jeff Ortel Date: Mon, 28 Aug 2023 09:31:51 -0700 Subject: [PATCH 7/7] formatting. Signed-off-by: Jeff Ortel --- api/analysis.go | 1 - 1 file changed, 1 deletion(-) diff --git a/api/analysis.go b/api/analysis.go index bef7c8d86..0ef16ac8b 100644 --- a/api/analysis.go +++ b/api/analysis.go @@ -2091,7 +2091,6 @@ func (r *AnalysisWriter) Write(id uint, output io.Writer) (err error) { return } r.end() - return }