diff --git a/codebase/analytics-gemini/client.go b/codebase/analytics-gemini/client.go deleted file mode 100644 index 045301946f..0000000000 --- a/codebase/analytics-gemini/client.go +++ /dev/null @@ -1,267 +0,0 @@ -package gemini - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "net/http/httputil" - "net/url" - - "github.com/fabric8-services/fabric8-wit/client" - "github.com/fabric8-services/fabric8-wit/errors" - "github.com/fabric8-services/fabric8-wit/goasupport" - "github.com/fabric8-services/fabric8-wit/log" - "github.com/fabric8-services/fabric8-wit/rest" - - goaclient "github.com/goadesign/goa/client" - "github.com/goadesign/goa/middleware" - goajwt "github.com/goadesign/goa/middleware/security/jwt" - errs "github.com/pkg/errors" -) - -// response is the struct which will be used to read the response -// from the analytics gemini service -type response struct { - Error string `json:"error"` -} - -// request is the struct used to form the request to the analytics -// gemini service, which needs RepoURL to be scanned -type Request struct { - GitURL string `json:"git-url"` -} - -// NewScanRepoRequest returns a request object when the URL to -// repository to be scanned and email id of the user is given. -// This request object can be passed while registering or deregistering -// to or from the analytics gemini service respectively. -func NewScanRepoRequest(repoURL string) *Request { - return &Request{ - GitURL: repoURL, - } -} - -// ScanRepoClient struct defines values that can be used to do -// request to the Analytics Gemini service, the Codebase search -// service and also you can mention if the developer mode is enabled -type ScanRepoClient struct { - // Analytics Gemini Service URL and Client - geminiURL string - geminiClient *http.Client - - // Codebase Search service URL and Client - codebaseSearchURL string - codebaseSearchClient *http.Client - - // specify if the dev mode is enabled - devMode bool -} - -// NewScanRepoClient function returns the Analytics Gemini Service -// client that can be used to make request to register or deregister -// a codebase repository URL with the service. It takes in endpoint -// to the Gemini service and Client. Also it takes the endpoint to -// the Codebase search URL and client to it. You can also specify -// if the mode is developer so that these requests can be avoided. -func NewScanRepoClient( - geminiURL string, - geminiClient *http.Client, - codebaseSearchURL string, - codebaseSearchClient *http.Client, - devMode bool, -) *ScanRepoClient { - return &ScanRepoClient{ - // setting gemini resources - geminiURL: geminiURL, - geminiClient: geminiClient, - - // setting codebase search service resources - codebaseSearchURL: codebaseSearchURL, - codebaseSearchClient: codebaseSearchClient, - - // specify if the developer mode is enabled - devMode: devMode, - } -} - -// setHeaders takes in the request object and sets headers like the -// content-type for request and response and the bearer token -func (sr *ScanRepoClient) setHeaders(ctx context.Context, req *http.Request) { - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") - - token := goajwt.ContextJWT(ctx) - if token != nil { - req.Header.Set("Authorization", "Bearer "+token.Raw) - } else { - log.Warn(ctx, map[string]interface{}{ - "err": "context has no JWT token", - }, "") - } - - req.Header.Set(middleware.RequestIDHeader, middleware.ContextRequestID(ctx)) -} - -func logErrors(ctx context.Context, d *Request, err error, bespokeError string) { - log.Error(ctx, map[string]interface{}{ - "repoURL": d.GitURL, - "err": err, - }, bespokeError) -} - -// callGemini is a generic method that makes http call to the gemini -// service, here you can define if the call is to register or deregister -// using the 'path' attribute and check if the request was successful -func (sr *ScanRepoClient) callGemini( - ctx context.Context, - d *Request, - path string, -) error { - dBytes, err := json.Marshal(d) - if err != nil { - logErrors(ctx, d, err, "failed to marshal object into json") - return errs.WithStack(err) - } - data := bytes.NewReader(dBytes) - - geminiURL, err := url.Parse(sr.geminiURL) - if err != nil { - return errs.WithStack(err) - } - geminiURL.Path = path - - req, err := http.NewRequest("POST", geminiURL.String(), data) - if err != nil { - logErrors(ctx, d, err, "failed to create request object") - return errs.WithStack(err) - } - sr.setHeaders(ctx, req) - - if log.IsDebug() { - b, _ := httputil.DumpRequest(req, true) - log.Debug(ctx, map[string]interface{}{ - "request": string(b), - }, "request object") - } - - resp, err := sr.geminiClient.Do(req) - if err != nil { - logErrors(ctx, d, err, "failed to talk to analytics gemini service") - return errs.WithStack(err) - } - defer rest.CloseResponse(resp) - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - logErrors(ctx, d, err, "") - return errs.WithStack(err) - } - - r := response{} - if err := json.Unmarshal(body, &r); err != nil { - logErrors(ctx, d, err, "failed to unmarshal the response") - return errs.WithStack(err) - } - - if resp.StatusCode != http.StatusOK { - switch resp.StatusCode { - case http.StatusUnauthorized: - return fmt.Errorf("unauthorized access: %v", r.Error) - default: - return fmt.Errorf("unknown error, got response: %s", string(body)) - } - } - - return nil -} - -// Register makes call to the analytics Gemini service which registers the -// given codebase URL for scanning of the CVEs in the code. -func (sr *ScanRepoClient) Register(ctx context.Context, d *Request) error { - if sr.devMode { - return nil - } - return sr.callGemini(ctx, d, "/api/v1/user-repo/scan") -} - -// DeRegister unsubscribes you from the analytics Gemini service so that the -// scanning for CVEs will be disabled for this codebase. Call this method when -// the codebase is deleted from database and everywhere else. -func (sr *ScanRepoClient) DeRegister(ctx context.Context, d *Request) error { - if sr.devMode { - return nil - } - // first list all the records with this repoURL in the database - codebases, err := sr.listCodebases(ctx, d) - if err != nil { - return err - } - - // see if we should actually disable the scanning of the codebase - // with the analytics service, if this returns true we don't disable - // if it returns false then we disable - if keepScanningThisCodebase(codebases) { - return nil - } - - // now deregister the repo from gemini server for scanning - return sr.callGemini(ctx, d, "/api/v1/user-repo/drop") -} - -// keepScanningThisCodebase returns true if there is even one codebase -// which has 'cve-scan' set to true. If there are no such codebases -// then it returns false. -func keepScanningThisCodebase(codebases *client.CodebaseList) bool { - var keepScanning bool - - for _, codebase := range codebases.Data { - if *codebase.Attributes.CveScan == true { - keepScanning = true - break - } - } - - return keepScanning -} - -// listCodebases makes a search call to the codebase service given -// the URL of the codebase -func (sr *ScanRepoClient) listCodebases(ctx context.Context, d *Request) (*client.CodebaseList, error) { - u, err := url.Parse(sr.codebaseSearchURL) - if err != nil { - return nil, errors.NewInternalError(ctx, fmt.Errorf("malformed codebase service URL %s: %v", - sr.codebaseSearchURL, err)) - } - - cl := client.New(goaclient.HTTPClientDoer(sr.codebaseSearchClient)) - cl.Host = u.Host - cl.Scheme = u.Scheme - cl.SetJWTSigner(goasupport.NewForwardSigner(goasupport.ForwardContextRequestID(ctx))) - - // search all the codebases associated with the repoURL - path := client.CodebasesSearchPath() - resp, err := cl.CodebasesSearch(ctx, path, d.GitURL, nil, nil) - if err != nil { - return nil, errors.NewInternalError(ctx, fmt.Errorf("could not search codebases: %v", err)) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - formattedErrors, err := cl.DecodeJSONAPIErrors(resp) - if err != nil { - return nil, errors.NewInternalError(ctx, fmt.Errorf("could not decode JSON formatted errors returned while listing codebases: %v", err)) - } - if len(formattedErrors.Errors) > 0 { - return nil, errors.NewInternalError(ctx, errs.Errorf(formattedErrors.Errors[0].Detail)) - } - return nil, errors.NewInternalError(ctx, errs.Errorf("unknown error")) - } - codebases, err := cl.DecodeCodebaseList(resp) - if err != nil { - return nil, errors.NewInternalError(ctx, fmt.Errorf("could not decode the codebase list: %v", err)) - } - return codebases, nil -} diff --git a/codebase/analytics-gemini/client_blackbox_test.go b/codebase/analytics-gemini/client_blackbox_test.go deleted file mode 100644 index 2d88cc95e3..0000000000 --- a/codebase/analytics-gemini/client_blackbox_test.go +++ /dev/null @@ -1,280 +0,0 @@ -package gemini_test - -import ( - "context" - "net/http" - "testing" - - gemini "github.com/fabric8-services/fabric8-wit/codebase/analytics-gemini" - "github.com/fabric8-services/fabric8-wit/configuration" - testjwt "github.com/fabric8-services/fabric8-wit/test/jwt" - testrecorder "github.com/fabric8-services/fabric8-wit/test/recorder" - - "github.com/stretchr/testify/require" -) - -func TestRegister(t *testing.T) { - // given - repoURL := "https://github.com/fabric8-services/fabric8-wit" - - config, err := configuration.New("") - require.NoError(t, err) - url := config.GetAnalyticsGeminiServiceURL() - - t.Run("ok response", func(t *testing.T) { - // given - r, err := testrecorder.New( - "../../test/data/gemini-scan/register-200", - testrecorder.WithJWTMatcher("../../test/jwt/public_key.pem"), - ) - require.NoError(t, err) - defer r.Stop() - - httpClient := &http.Client{ - Transport: r.Transport, - } - - cli := gemini.NewScanRepoClient(url, httpClient, "", nil, false) - - req := gemini.NewScanRepoRequest(repoURL) - ctx, err := testjwt.NewJWTContext("bcdd0b29-123d-11e8-a8bc-b69930b94f5c", "../../test/jwt/private_key.pem") - require.NoError(t, err) - - // then - err = cli.Register(ctx, req) - require.NoError(t, err) - }) - - t.Run("fail on no jwt token", func(t *testing.T) { - r, err := testrecorder.New( - "../../test/data/gemini-scan/register-401", - ) - require.NoError(t, err) - defer r.Stop() - - httpClient := &http.Client{ - Transport: r.Transport, - } - - cli := gemini.NewScanRepoClient(url, httpClient, "", nil, false) - req := gemini.NewScanRepoRequest(repoURL) - - // give a context that has no token - err = cli.Register(context.Background(), req) - require.Error(t, err) - t.Log("successfully errored as: ", err) - }) - - t.Run("fail on parsing url", func(t *testing.T) { - // the url for this request is invalid - url := "%" - cli := gemini.NewScanRepoClient(url, &http.Client{}, "", nil, false) - - req := gemini.NewScanRepoRequest(repoURL) - // then - err = cli.Register(context.Background(), req) - require.Error(t, err) - t.Log("successfully errored as: ", err) - }) - - t.Run("fail on wrong URL", func(t *testing.T) { - // url in this request does not mention what scheme to use - // for e.g. http or https - url := "foo.bar" - cli := gemini.NewScanRepoClient(url, &http.Client{}, "", nil, false) - - req := gemini.NewScanRepoRequest(repoURL) - - ctx, err := testjwt.NewJWTContext("bcdd0b29-123d-11e8-a8bc-b69930b94f5c", "../../test/jwt/private_key.pem") - require.NoError(t, err) - - // then - err = cli.Register(ctx, req) - require.Error(t, err) - t.Log("successfully errored as: ", err) - }) - - t.Run("fail to unmarshal response", func(t *testing.T) { - // server responds 200 but empty body which is not - // json format, so the unmarshaller will complain - r, err := testrecorder.New( - "../../test/data/gemini-scan/register-weird-response", - testrecorder.WithJWTMatcher("../../test/jwt/public_key.pem"), - ) - require.NoError(t, err) - defer r.Stop() - - httpClient := &http.Client{ - Transport: r.Transport, - } - - cli := gemini.NewScanRepoClient(url, httpClient, "", nil, false) - - req := gemini.NewScanRepoRequest(repoURL) - ctx, err := testjwt.NewJWTContext("bcdd0b29-123d-11e8-a8bc-b69930b94f5c", "../../test/jwt/private_key.pem") - require.NoError(t, err) - - // then - err = cli.Register(ctx, req) - require.Error(t, err) - t.Log("successfully errored as: ", err) - }) - - t.Run("unknown response from the server", func(t *testing.T) { - // given - // server responds with 500 here - r, err := testrecorder.New( - "../../test/data/gemini-scan/register-unknown-response", - testrecorder.WithJWTMatcher("../../test/jwt/public_key.pem"), - ) - require.NoError(t, err) - defer r.Stop() - - httpClient := &http.Client{ - Transport: r.Transport, - } - - cli := gemini.NewScanRepoClient(url, httpClient, "", nil, false) - - req := gemini.NewScanRepoRequest(repoURL) - ctx, err := testjwt.NewJWTContext("bcdd0b29-123d-11e8-a8bc-b69930b94f5c", "../../test/jwt/private_key.pem") - require.NoError(t, err) - - // then - err = cli.Register(ctx, req) - require.Error(t, err) - t.Log("successfully errored as: ", err) - }) -} - -func TestDeRegister(t *testing.T) { - // given - repoURL := "https://github.com/fabric8-services/fabric8-wit" - - config, err := configuration.New("") - require.NoError(t, err) - geminiURL := config.GetAnalyticsGeminiServiceURL() - codebaseURL := config.GetCodebaseServiceURL() - - // in this test, the call to deregister is made but there are some codebases - // available with the same URL so further call to the gemini service to deregister - // is not made - t.Run("ok response no call to gemini", func(t *testing.T) { - // given - r, err := testrecorder.New( - "../../test/data/gemini-scan/deregister-200", - ) - require.NoError(t, err) - defer r.Stop() - - httpClient := &http.Client{Transport: r.Transport} - cli := gemini.NewScanRepoClient(geminiURL, httpClient, codebaseURL, httpClient, false) - - req := gemini.NewScanRepoRequest(repoURL) - ctx, err := testjwt.NewJWTContext("bcdd0b29-123d-11e8-a8bc-b69930b94f5c", "../../test/jwt/private_key.pem") - require.NoError(t, err) - - // then - err = cli.DeRegister(ctx, req) - require.NoError(t, err) - }) - - // in this test, call to deregister is made and there are no codebases - // left with same codebase URL so the call to gemini service is made - t.Run("ok response and call to gemini", func(t *testing.T) { - // given for codebases - recordCodebases, err := testrecorder.New( - "../../test/data/gemini-scan/deregister-call-gemini-codebases-200", - ) - require.NoError(t, err) - defer recordCodebases.Stop() - codebaseClient := &http.Client{Transport: recordCodebases.Transport} - - recordGemini, err := testrecorder.New( - "../../test/data/gemini-scan/deregister-call-gemini-200", - testrecorder.WithJWTMatcher("../../test/jwt/public_key.pem"), - ) - require.NoError(t, err) - defer recordGemini.Stop() - geminiClient := &http.Client{Transport: recordGemini.Transport} - - cli := gemini.NewScanRepoClient(geminiURL, geminiClient, codebaseURL, codebaseClient, false) - - req := gemini.NewScanRepoRequest(repoURL) - ctx, err := testjwt.NewJWTContext("bcdd0b29-123d-11e8-a8bc-b69930b94f5c", "../../test/jwt/private_key.pem") - require.NoError(t, err) - - // then - err = cli.DeRegister(ctx, req) - require.NoError(t, err) - }) - - t.Run("malformed url error for the codebase", func(t *testing.T) { - codebaseAlterURL := "%" - cli := gemini.NewScanRepoClient(geminiURL, &http.Client{}, codebaseAlterURL, &http.Client{}, false) - - req := gemini.NewScanRepoRequest(repoURL) - ctx, err := testjwt.NewJWTContext("bcdd0b29-123d-11e8-a8bc-b69930b94f5c", "../../test/jwt/private_key.pem") - require.NoError(t, err) - - // then - err = cli.DeRegister(ctx, req) - require.Error(t, err) - t.Log("successfully errored as: ", err) - }) - - t.Run("should fail on making call to core", func(t *testing.T) { - cli := gemini.NewScanRepoClient(geminiURL, &http.Client{}, codebaseURL, &http.Client{}, false) - - req := gemini.NewScanRepoRequest(repoURL) - ctx, err := testjwt.NewJWTContext("bcdd0b29-123d-11e8-a8bc-b69930b94f5c", "../../test/jwt/private_key.pem") - require.NoError(t, err) - - // then - err = cli.DeRegister(ctx, req) - require.Error(t, err) - t.Log("successfully errored as: ", err) - }) - - t.Run("400 from codebases", func(t *testing.T) { - // given - r, err := testrecorder.New( - "../../test/data/gemini-scan/deregister-400", - ) - require.NoError(t, err) - defer r.Stop() - - httpClient := &http.Client{Transport: r.Transport} - cli := gemini.NewScanRepoClient(geminiURL, httpClient, codebaseURL, httpClient, false) - - req := gemini.NewScanRepoRequest(repoURL) - ctx, err := testjwt.NewJWTContext("bcdd0b29-123d-11e8-a8bc-b69930b94f5c", "../../test/jwt/private_key.pem") - require.NoError(t, err) - - // then - err = cli.DeRegister(ctx, req) - require.Error(t, err) - t.Log("successfully errored as: ", err) - }) - - t.Run("could not decode codebases", func(t *testing.T) { - // given - r, err := testrecorder.New( - "../../test/data/gemini-scan/deregister-bad-output", - ) - require.NoError(t, err) - defer r.Stop() - - httpClient := &http.Client{Transport: r.Transport} - cli := gemini.NewScanRepoClient(geminiURL, httpClient, codebaseURL, httpClient, false) - - req := gemini.NewScanRepoRequest(repoURL) - ctx, err := testjwt.NewJWTContext("bcdd0b29-123d-11e8-a8bc-b69930b94f5c", "../../test/jwt/private_key.pem") - require.NoError(t, err) - - // then - err = cli.DeRegister(ctx, req) - require.Error(t, err) - t.Log("successfully errored as: ", err) - }) -} diff --git a/codebase/analytics-gemini/client_test.go b/codebase/analytics-gemini/client_test.go deleted file mode 100644 index c12f1b054c..0000000000 --- a/codebase/analytics-gemini/client_test.go +++ /dev/null @@ -1,86 +0,0 @@ -package gemini - -import ( - "testing" - - "github.com/fabric8-services/fabric8-wit/client" - "github.com/fabric8-services/fabric8-wit/ptr" -) - -func Test_keepScanningThisCodebase(t *testing.T) { - type args struct { - codebases *client.CodebaseList - } - tests := []struct { - name string - args args - want bool - }{ - { - name: "no codebases at all", - args: args{ - codebases: &client.CodebaseList{ - Data: []*client.Codebase{}, - }, - }, - want: false, - }, - { - name: "one codebase with cve-scan as false", - args: args{ - codebases: &client.CodebaseList{ - Data: []*client.Codebase{ - { - Attributes: &client.CodebaseAttributes{ - CveScan: ptr.Bool(false), - }, - }, - }, - }, - }, - want: false, - }, - { - name: "one codebase with cve-scan as true", - args: args{ - codebases: &client.CodebaseList{ - Data: []*client.Codebase{ - { - Attributes: &client.CodebaseAttributes{ - CveScan: ptr.Bool(true), - }, - }, - }, - }, - }, - want: true, - }, - { - name: "two codebases one with cve-scan as true and other false", - args: args{ - codebases: &client.CodebaseList{ - Data: []*client.Codebase{ - { - Attributes: &client.CodebaseAttributes{ - CveScan: ptr.Bool(false), - }, - }, - { - Attributes: &client.CodebaseAttributes{ - CveScan: ptr.Bool(true), - }, - }, - }, - }, - }, - want: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := keepScanningThisCodebase(tt.args.codebases); got != tt.want { - t.Errorf("keepScanningThisCodebase() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/configuration/configuration.go b/configuration/configuration.go index a455edd1ab..6e0580cbcf 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -99,19 +99,18 @@ const ( varCacheControlQuery = "cachecontrol.query" varCacheControlComment = "cachecontrol.comment" - defaultConfigFile = "config.yaml" - varOpenshiftTenantMasterURL = "openshift.tenant.masterurl" - varCheStarterURL = "chestarterurl" - varValidRedirectURLs = "redirect.valid" - varLogLevel = "log.level" - varLogJSON = "log.json" - varTenantServiceURL = "tenant.serviceurl" - varNotificationServiceURL = "notification.serviceurl" - varTogglesServiceURL = "toggles.serviceurl" - varDeploymentsServiceURL = "deployments.serviceurl" - varCodebaseServiceURL = "codebase.serviceurl" - varAnalyticsGeminiServiceURL = "analytics.gemini.serviceurl" - varDeploymentsHTTPTimeout = "deployments.http.timeout" + defaultConfigFile = "config.yaml" + varOpenshiftTenantMasterURL = "openshift.tenant.masterurl" + varCheStarterURL = "chestarterurl" + varValidRedirectURLs = "redirect.valid" + varLogLevel = "log.level" + varLogJSON = "log.json" + varTenantServiceURL = "tenant.serviceurl" + varNotificationServiceURL = "notification.serviceurl" + varTogglesServiceURL = "toggles.serviceurl" + varDeploymentsServiceURL = "deployments.serviceurl" + varCodebaseServiceURL = "codebase.serviceurl" + varDeploymentsHTTPTimeout = "deployments.http.timeout" ) // Registry encapsulates the Viper configuration registry which stores the @@ -256,7 +255,6 @@ func (c *Registry) setConfigDefaults() { c.v.SetDefault(varDeploymentsServiceURL, defaultDeploymentsServiceURL) c.v.SetDefault(varCodebaseServiceURL, defaultCodebaseServiceURL) c.v.SetDefault(varDeploymentsHTTPTimeout, defaultDeploymentsHTTPTimeout) - c.v.SetDefault(varAnalyticsGeminiServiceURL, defaultAnalyticsGeminiServiceURL) } // GetPostgresHost returns the postgres host as set via default, config file, or environment variable @@ -875,12 +873,6 @@ func (c *Registry) GetCodebaseServiceURL() string { return c.v.GetString(varCodebaseServiceURL) } -// GetAnalyticsGeminiServiceURL returns the URL for the Analytics service -// which is used by the codebase repository to enable or disable scanning -func (c *Registry) GetAnalyticsGeminiServiceURL() string { - return c.v.GetString(varAnalyticsGeminiServiceURL) -} - // GetDeploymentsTimeout returns the amount of seconds until it should timeout. func (c *Registry) GetDeploymentsHTTPTimeoutSeconds() time.Duration { timeout := c.v.GetInt(varDeploymentsHTTPTimeout) @@ -930,9 +922,6 @@ const ( defaultDeploymentsServiceURL = "http://core" defaultCodebaseServiceURL = "http://core" - // Analytics service URLs - defaultAnalyticsGeminiServiceURL = "http://geminiservice.analytics" - // DefaultValidRedirectURLs is a regex to be used to whitelist redirect URL for auth // If the F8_REDIRECT_VALID env var is not set then in Dev Mode all redirects allowed - * // In prod mode the following regex will be used by default: diff --git a/controller/codebase.go b/controller/codebase.go index 9c98ec30dd..05d8be051d 100644 --- a/controller/codebase.go +++ b/controller/codebase.go @@ -10,9 +10,7 @@ import ( "github.com/fabric8-services/fabric8-wit/app" "github.com/fabric8-services/fabric8-wit/application" "github.com/fabric8-services/fabric8-wit/codebase" - gemini "github.com/fabric8-services/fabric8-wit/codebase/analytics-gemini" "github.com/fabric8-services/fabric8-wit/codebase/che" - "github.com/fabric8-services/fabric8-wit/configuration" "github.com/fabric8-services/fabric8-wit/jsonapi" "github.com/fabric8-services/fabric8-wit/log" "github.com/fabric8-services/fabric8-wit/login" @@ -41,17 +39,13 @@ type codebaseConfiguration interface { // CodebaseCheClientProvider the function that provides a `cheClient` type CodebaseCheClientProvider func(ctx context.Context, ns string) (che.Client, error) -// AnalyticsGeminiClientProvider the function that provides a ScanRepoClient -type AnalyticsGeminiClientProvider func() *gemini.ScanRepoClient - // CodebaseController implements the codebase resource. type CodebaseController struct { *goa.Controller - db application.DB - config codebaseConfiguration - ShowTenant account.CodebaseInitTenantProvider - NewCheClient CodebaseCheClientProvider - AnalyticsGeminiClient AnalyticsGeminiClientProvider + db application.DB + config codebaseConfiguration + ShowTenant account.CodebaseInitTenantProvider + NewCheClient CodebaseCheClientProvider } // NewCodebaseController creates a codebase controller. @@ -116,19 +110,6 @@ func (c *CodebaseController) Update(ctx *app.UpdateCodebaseContext) error { return jsonapi.JSONErrorResponse(ctx, err) } - // depending on the update we call the analytics service to enable - // or disable the scan for this codebase, calling only after the - // CveScan attribute is passed with some value - if reqAttributes.CveScan != nil && updatedCb.CVEScan { - if err = c.registerCodebaseToGeminiForScan(ctx, updatedCb.URL); err != nil { - return jsonapi.JSONErrorResponse(ctx, err) - } - } else if reqAttributes.CveScan != nil { - if err = c.deRegisterCodebaseFromGeminiForScan(ctx, updatedCb.URL); err != nil { - return jsonapi.JSONErrorResponse(ctx, err) - } - } - res := &app.CodebaseSingle{ Data: ConvertCodebase(ctx.Request, *updatedCb), } @@ -250,25 +231,6 @@ func (c *CodebaseController) verifyCodebaseOwner(ctx context.Context, codebaseID return cb, nil } -// registerCodebaseToGeminiForScan when given the codebase URL, subscribes this codebase -// to enable code scanning to find CVEs with the analytics gemini service -func (c *CodebaseController) registerCodebaseToGeminiForScan( - ctx context.Context, - repoURL string, -) error { - scanClient := c.AnalyticsGeminiClient() - req := gemini.NewScanRepoRequest(repoURL) - return scanClient.Register(ctx, req) -} - -// deRegisterCodebaseFromGeminiForScan when given the codebase URL, unsubscibes -// this codebase from scanning -func (c *CodebaseController) deRegisterCodebaseFromGeminiForScan(ctx context.Context, repoURL string) error { - scanClient := c.AnalyticsGeminiClient() - req := gemini.NewScanRepoRequest(repoURL) - return scanClient.DeRegister(ctx, req) -} - // Delete deletes the given codebase if the user is authenticated and authorized func (c *CodebaseController) Delete(ctx *app.DeleteCodebaseContext) error { // see if the user is allowed to delete this codebase @@ -319,11 +281,6 @@ func (c *CodebaseController) Delete(ctx *app.DeleteCodebaseContext) error { return jsonapi.JSONErrorResponse(ctx, err) } - // the codebase is deleted only then deregister from the analytics service - if err = c.deRegisterCodebaseFromGeminiForScan(ctx, cb.URL); err != nil { - return jsonapi.JSONErrorResponse(ctx, err) - } - return ctx.NoContent() } @@ -640,22 +597,6 @@ func NewDefaultCheClient(config codebaseConfiguration) CodebaseCheClientProvider } } -// NewDefaultAnalyticsGeminiClient returns the default function to initialize a new Analytics -// Gemini Client. As an attribute it takes in the Registry object. -func NewDefaultAnalyticsGeminiClient(config *configuration.Registry) AnalyticsGeminiClientProvider { - return func() *gemini.ScanRepoClient { - return gemini.NewScanRepoClient( - config.GetAnalyticsGeminiServiceURL(), - &http.Client{}, - config.GetCodebaseServiceURL(), - &http.Client{}, - // this method is used in general to find out if the - // developer mode is enabled - config.IsPostgresDeveloperModeEnabled(), - ) - } -} - // getWorkspaceBranch returns the branch defined in the request payload ('master' is used by default) func getWorkspaceBranch(ctx *app.CreateCodebaseContext) string { branch := "master" diff --git a/controller/codebase_blackbox_test.go b/controller/codebase_blackbox_test.go index 4a1ea72110..33bf02e7d0 100644 --- a/controller/codebase_blackbox_test.go +++ b/controller/codebase_blackbox_test.go @@ -11,7 +11,6 @@ import ( "github.com/fabric8-services/fabric8-wit/account/tenant" "github.com/fabric8-services/fabric8-wit/app" "github.com/fabric8-services/fabric8-wit/app/test" - gemini "github.com/fabric8-services/fabric8-wit/codebase/analytics-gemini" "github.com/fabric8-services/fabric8-wit/codebase/che" "github.com/fabric8-services/fabric8-wit/configuration" . "github.com/fabric8-services/fabric8-wit/controller" @@ -60,16 +59,6 @@ func withShowTenant(f account.CodebaseInitTenantProvider) ConfigureCodebaseContr } } -// withAnalyticsGeminiClient takes in the function that can initialize -// the Analytics Gemini Service Client and it returns a closure which -// takes in the CodebaseController object and initializes this object -// with the Analytics Gemini Client -func withAnalyticsGeminiClient(f AnalyticsGeminiClientProvider) ConfigureCodebaseController { - return func(codebaseCtrl *CodebaseController) { - codebaseCtrl.AnalyticsGeminiClient = f - } -} - func (s *CodebaseControllerTestSuite) UnsecuredController(settings ...ConfigureCodebaseController) (*goa.Service, *CodebaseController) { svc := goa.New("Codebases-service") codebaseCtrl := NewCodebaseController(svc, s.GormDB, s.Configuration) @@ -118,23 +107,6 @@ func MockShowTenant() func(context.Context) (*tenant.TenantSingle, error) { } } -// MockAnalyticsGeminiClient takes in the transport objects for -// Gemini service and the Codebase service and returns function when -// called returns the Gemini Service Client -func MockAnalyticsGeminiClient(geminiTransport, codebaseTransport http.RoundTripper) func() *gemini.ScanRepoClient { - config, _ := configuration.New("") - return func() *gemini.ScanRepoClient { - - return gemini.NewScanRepoClient( - config.GetAnalyticsGeminiServiceURL(), - &http.Client{Transport: geminiTransport}, - config.GetCodebaseServiceURL(), - &http.Client{Transport: codebaseTransport}, - false, - ) - } -} - func (s *CodebaseControllerTestSuite) TestShowCodebase() { s.T().Run("success without stackId", func(t *testing.T) { @@ -185,7 +157,6 @@ func (s *CodebaseControllerTestSuite) TestDeleteCodebase() { testsupport.TestIdentity, withCheClient(NewMockCheClient(m, s.Configuration)), withShowTenant(MockShowTenant()), - withAnalyticsGeminiClient(MockAnalyticsGeminiClient(m, m)), ) // when test.DeleteCodebaseNoContent(t, svc.Context, svc, ctrl, fxt.Codebases[0].ID) @@ -201,11 +172,6 @@ func (s *CodebaseControllerTestSuite) TestDeleteCodebase() { RequestURL: "che-server/workspace/string?masterUrl=https://tsrv.devshift.net:8443&namespace=foo", StatusCode: 200, }, - httpmonitor.Exchange{ - RequestMethod: "GET", - RequestURL: "http://core/api/search/codebases?url=git%40github.com%3Abar%2Ffoo", - StatusCode: 200, - }, ) require.NoError(t, err) @@ -232,7 +198,6 @@ func (s *CodebaseControllerTestSuite) TestDeleteCodebase() { testsupport.TestIdentity, withCheClient(NewMockCheClient(m, s.Configuration)), withShowTenant(MockShowTenant()), - withAnalyticsGeminiClient(MockAnalyticsGeminiClient(m, m)), ) // when test.DeleteCodebaseNoContent(t, svc.Context, svc, ctrl, fxt.Codebases[0].ID) @@ -248,11 +213,6 @@ func (s *CodebaseControllerTestSuite) TestDeleteCodebase() { RequestURL: "che-server/workspace/string?masterUrl=https://tsrv.devshift.net:8443&namespace=foo", StatusCode: 500, }, - httpmonitor.Exchange{ - RequestMethod: "GET", - RequestURL: "http://core/api/search/codebases?url=git%40github.com%3Abar%2Ffoo", - StatusCode: 200, - }, ) require.NoError(t, err) }) @@ -345,17 +305,15 @@ func (s *CodebaseControllerTestSuite) TestUpdateCodebase() { } t.Run("OK", func(t *testing.T) { - r, err := recorder.New("../test/data/gemini-scan/codebase-update-ok") + r, err := recorder.New("../test/data/cve-scan/codebase-update-ok") require.NoError(t, err) defer r.Stop() - m := httpmonitor.NewTransportMonitor(r.Transport) // given fxt := tf.NewTestFixture(t, s.DB, tf.Codebases(1)) codebase := fxt.Codebases[0] svc, ctrl := s.SecuredControllers( *fxt.Identities[0], - withAnalyticsGeminiClient(MockAnalyticsGeminiClient(m, m)), ) // input @@ -372,15 +330,6 @@ func (s *CodebaseControllerTestSuite) TestUpdateCodebase() { require.Equal(t, false, *result.Data.Attributes.CveScan) require.Equal(t, newType, *result.Data.Attributes.Type) require.Equal(t, newStack, *result.Data.Attributes.StackID) - - err = m.ValidateExchanges( - httpmonitor.Exchange{ - RequestMethod: "GET", - RequestURL: "http://core/api/search/codebases?url=git%40github.com%3Afabric8-services%2Ffabric8-wit.git", - StatusCode: 200, - }, - ) - require.NoError(t, err) }) t.Run("forbidden for wrong user", func(t *testing.T) { diff --git a/main.go b/main.go index 7ae927e589..b56ca38bcb 100755 --- a/main.go +++ b/main.go @@ -364,7 +364,6 @@ func main() { codebaseCtrl := controller.NewCodebaseController(service, appDB, config) codebaseCtrl.ShowTenant = account.NewShowTenant(config) codebaseCtrl.NewCheClient = controller.NewDefaultCheClient(config) - codebaseCtrl.AnalyticsGeminiClient = controller.NewDefaultAnalyticsGeminiClient(config) app.MountCodebaseController(service, codebaseCtrl) diff --git a/openshift/core.app.yaml b/openshift/core.app.yaml index 30c3574e9b..1792cefe44 100644 --- a/openshift/core.app.yaml +++ b/openshift/core.app.yaml @@ -124,11 +124,6 @@ objects: configMapKeyRef: name: core key: codebase.serviceurl - - name: F8_ANALYTICS_GEMINI_SERVICEURL - valueFrom: - configMapKeyRef: - name: core - key: analytics.gemini.serviceurl - name: F8_REDIRECT_VALID valueFrom: configMapKeyRef: diff --git a/openshift/core.config.yaml b/openshift/core.config.yaml index 9db4a1a003..4c9d70e41b 100644 --- a/openshift/core.config.yaml +++ b/openshift/core.config.yaml @@ -38,4 +38,3 @@ objects: feature.workitem.remote: "false" deployments.serviceurl: "http://core" codebase.serviceurl: "http://core" - analytics.gemini.serviceurl: "http://f8a-gemini-server.bayesian-production:5000" diff --git a/test/data/gemini-scan/codebase-update-ok.yaml b/test/data/cve-scan/codebase-update-ok.yaml similarity index 100% rename from test/data/gemini-scan/codebase-update-ok.yaml rename to test/data/cve-scan/codebase-update-ok.yaml diff --git a/test/data/gemini-scan/deregister-200.yaml b/test/data/gemini-scan/deregister-200.yaml deleted file mode 100644 index 6121d9cebf..0000000000 --- a/test/data/gemini-scan/deregister-200.yaml +++ /dev/null @@ -1,156 +0,0 @@ ---- -version: 1 -interactions: -- request: - url: http://core/api/search/codebases?url=https%3A%2F%2Fgithub.com%2Ffabric8-services%2Ffabric8-wit - method: GET - body: "" - form: {} - response: - status: 200 OK - code: 200 - body: '{ - "data": [ - { - "attributes": { - "createdAt": "2018-05-08T10:33:50.036144Z", - "cve-scan": true, - "last_used_workspace": "", - "stackId": "vert.x", - "type": "git", - "url": "https://github.com/fabric8-services/fabric8-wit" - }, - "id": "9d16ea1c-cd2a-48f5-af46-86343545baf8", - "links": { - "edit": "https://api.prod-preview.openshift.io/api/codebases/9d16ea1c-cd2a-48f5-af46-86343545baf8/edit", - "related": "https://api.prod-preview.openshift.io/api/codebases/9d16ea1c-cd2a-48f5-af46-86343545baf8", - "self": "https://api.prod-preview.openshift.io/api/codebases/9d16ea1c-cd2a-48f5-af46-86343545baf8" - }, - "relationships": { - "space": { - "data": { - "id": "dca3f0b7-816d-4cf4-9119-581f06d4c2a6", - "type": "spaces" - }, - "links": { - "related": "https://api.prod-preview.openshift.io/api/spaces/dca3f0b7-816d-4cf4-9119-581f06d4c2a6", - "self": "https://api.prod-preview.openshift.io/api/spaces/dca3f0b7-816d-4cf4-9119-581f06d4c2a6" - } - }, - "workspaces": { - "links": { - "related": "https://api.prod-preview.openshift.io/api/codebases/9d16ea1c-cd2a-48f5-af46-86343545baf8/workspaces", - "self": "https://api.prod-preview.openshift.io/api/codebases/9d16ea1c-cd2a-48f5-af46-86343545baf8/workspaces" - } - } - }, - "type": "codebases" - } - ], - "included": [ - { - "attributes": { - "created-at": "2018-05-08T10:32:56.034534Z", - "description": "", - "name": "aagamshah", - "updated-at": "2018-05-08T10:32:56.034534Z", - "version": 0 - }, - "id": "dca3f0b7-816d-4cf4-9119-581f06d4c2a6", - "links": { - "backlog": { - "self": "https://api.prod-preview.openshift.io/api/spaces/dca3f0b7-816d-4cf4-9119-581f06d4c2a6/backlog" - }, - "filters": "https://api.prod-preview.openshift.io/api/filters", - "related": "https://api.prod-preview.openshift.io/api/spaces/dca3f0b7-816d-4cf4-9119-581f06d4c2a6", - "self": "https://api.prod-preview.openshift.io/api/spaces/dca3f0b7-816d-4cf4-9119-581f06d4c2a6", - "workitemlinktypes": "https://api.prod-preview.openshift.io/api/spacetemplates/929c963a-174c-4c37-b487-272067e88bd4/workitemlinktypes", - "workitemtypes": "https://api.prod-preview.openshift.io/api/spacetemplates/929c963a-174c-4c37-b487-272067e88bd4/workitemtypes" - }, - "relationships": { - "areas": { - "links": { - "related": "https://api.prod-preview.openshift.io/api/spaces/dca3f0b7-816d-4cf4-9119-581f06d4c2a6/areas" - } - }, - "backlog": { - "links": { - "related": "https://api.prod-preview.openshift.io/api/spaces/dca3f0b7-816d-4cf4-9119-581f06d4c2a6/backlog" - } - }, - "codebases": { - "links": { - "related": "https://api.prod-preview.openshift.io/api/spaces/dca3f0b7-816d-4cf4-9119-581f06d4c2a6/codebases" - } - }, - "collaborators": { - "links": { - "related": "https://api.prod-preview.openshift.io/api/spaces/dca3f0b7-816d-4cf4-9119-581f06d4c2a6/collaborators" - } - }, - "filters": { - "links": { - "related": "https://api.prod-preview.openshift.io/api/filters" - } - }, - "iterations": { - "links": { - "related": "https://api.prod-preview.openshift.io/api/spaces/dca3f0b7-816d-4cf4-9119-581f06d4c2a6/iterations" - } - }, - "labels": { - "links": { - "related": "https://api.prod-preview.openshift.io/api/spaces/dca3f0b7-816d-4cf4-9119-581f06d4c2a6/labels" - } - }, - "owned-by": { - "data": { - "id": "137ac7b8-a954-4fb3-a2b8-acee00ac6c13", - "type": "identities" - }, - "links": { - "related": "https://api.prod-preview.openshift.io/api/users/137ac7b8-a954-4fb3-a2b8-acee00ac6c13" - } - }, - "space-template": { - "data": { - "id": "929c963a-174c-4c37-b487-272067e88bd4", - "type": "spacetemplates" - }, - "links": { - "related": "https://api.prod-preview.openshift.io/api/spacetemplates/929c963a-174c-4c37-b487-272067e88bd4", - "self": "https://api.prod-preview.openshift.io/api/spacetemplates/929c963a-174c-4c37-b487-272067e88bd4" - } - }, - "workitemlinktypes": { - "links": { - "related": "https://api.prod-preview.openshift.io/api/spacetemplates/929c963a-174c-4c37-b487-272067e88bd4/workitemlinktypes" - } - }, - "workitems": { - "links": { - "related": "https://api.prod-preview.openshift.io/api/spaces/dca3f0b7-816d-4cf4-9119-581f06d4c2a6/workitems" - } - }, - "workitemtypegroups": { - "links": { - "related": "https://api.prod-preview.openshift.io/api/spacetemplates/929c963a-174c-4c37-b487-272067e88bd4/workitemtypegroups" - } - }, - "workitemtypes": { - "links": { - "related": "https://api.prod-preview.openshift.io/api/spacetemplates/929c963a-174c-4c37-b487-272067e88bd4/workitemtypes" - } - } - }, - "type": "spaces" - } - ], - "links": { - "first": "https://api.prod-preview.openshift.io/api/search/codebases?page[offset]=0&page[limit]=20&url=https://github.com/fabric8-services/fabric8-wit", - "last": "https://api.prod-preview.openshift.io/api/search/codebases?page[offset]=0&page[limit]=20&url=https://github.com/fabric8-services/fabric8-wit" - }, - "meta": { - "totalCount": 1 - } -}' diff --git a/test/data/gemini-scan/deregister-400.yaml b/test/data/gemini-scan/deregister-400.yaml deleted file mode 100644 index 0745f254f6..0000000000 --- a/test/data/gemini-scan/deregister-400.yaml +++ /dev/null @@ -1,22 +0,0 @@ ---- -version: 1 -interactions: -- request: - url: http://core/api/search/codebases?url=https%3A%2F%2Fgithub.com%2Ffabric8-services%2Ffabric8-wit - method: GET - body: "" - form: {} - response: - status: 400 Bad Request - code: 400 - body: '{ - "errors": [ - { - "code": "invalid_request", - "detail": "missing required parameter \"url\"", - "id": "Lm4JBUF0", - "status": "400", - "title": "Bad Request" - } - ] -}' diff --git a/test/data/gemini-scan/deregister-bad-output.yaml b/test/data/gemini-scan/deregister-bad-output.yaml deleted file mode 100644 index cf3a0486ef..0000000000 --- a/test/data/gemini-scan/deregister-bad-output.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -version: 1 -interactions: -- request: - url: http://core/api/search/codebases?url=https%3A%2F%2Fgithub.com%2Ffabric8-services%2Ffabric8-wit - method: GET - body: "" - form: {} - response: - status: 200 OK - code: 200 - body: '' diff --git a/test/data/gemini-scan/deregister-call-gemini-200.yaml b/test/data/gemini-scan/deregister-call-gemini-200.yaml deleted file mode 100644 index ee45a2bb2e..0000000000 --- a/test/data/gemini-scan/deregister-call-gemini-200.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -version: 1 -interactions: -- request: - headers: - sub: ["bcdd0b29-123d-11e8-a8bc-b69930b94f5c"] - url: http://geminiservice.analytics/api/v1/user-repo/drop - method: POST - body: '{ - "git-url": "https://github.com/fabric8-services/fabric8-wit", - "email-ids": [ - "sdeshmuk@redhat.com" - ] -}' - response: - status: 200 OK - code: 200 - body: '{"summary":"Repository scan unsubscribed"}' diff --git a/test/data/gemini-scan/deregister-call-gemini-codebases-200.yaml b/test/data/gemini-scan/deregister-call-gemini-codebases-200.yaml deleted file mode 100644 index 856e9cb829..0000000000 --- a/test/data/gemini-scan/deregister-call-gemini-codebases-200.yaml +++ /dev/null @@ -1,21 +0,0 @@ ---- -version: 1 -interactions: -- request: - url: http://core/api/search/codebases?url=https%3A%2F%2Fgithub.com%2Ffabric8-services%2Ffabric8-wit - method: GET - body: "" - form: {} - response: - status: 200 OK - code: 200 - body: '{ - "data": [], - "links": { - "first": "https://api.prod-preview.openshift.io/api/search/codebases?page[offset]=0&page[limit]=20&url=https://github.com/surajssd/ahapp-tesagat-1amsh", - "last": "https://api.prod-preview.openshift.io/api/search/codebases?page[offset]=0&page[limit]=0&url=https://github.com/surajssd/ahapp-tesagat-1amsh" - }, - "meta": { - "totalCount": 0 - } -}' diff --git a/test/data/gemini-scan/register-200.yaml b/test/data/gemini-scan/register-200.yaml deleted file mode 100644 index 31a5110414..0000000000 --- a/test/data/gemini-scan/register-200.yaml +++ /dev/null @@ -1,19 +0,0 @@ ---- -version: 1 -interactions: -- request: - headers: - sub: ["bcdd0b29-123d-11e8-a8bc-b69930b94f5c"] - url: http://geminiservice.analytics/api/v1/user-repo/scan - method: POST - body: '{ - "git-url": "https://github.com/fabric8-services/fabric8-wit", - "email-ids": [ - "sdeshmuk@redhat.com" - ] -}' - response: - status: 200 OK - code: 200 - body: '{"summary":"Repository scan initiated"}' - diff --git a/test/data/gemini-scan/register-401.yaml b/test/data/gemini-scan/register-401.yaml deleted file mode 100644 index a7cbfea0ff..0000000000 --- a/test/data/gemini-scan/register-401.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -version: 1 -interactions: -- request: - url: http://geminiservice.analytics/api/v1/user-repo/scan - method: POST - body: '{ - "git-url": "https://github.com/fabric8-services/fabric8-wit", - "email-ids": [ - "sdeshmuk@redhat.com" - ] -}' - response: - status: 401 Unauthorized - code: 401 - body: '{"error":"Authentication failed - could not decode JWT token"}' diff --git a/test/data/gemini-scan/register-unknown-response.yaml b/test/data/gemini-scan/register-unknown-response.yaml deleted file mode 100644 index 94242dc80e..0000000000 --- a/test/data/gemini-scan/register-unknown-response.yaml +++ /dev/null @@ -1,19 +0,0 @@ ---- -version: 1 -interactions: -- request: - headers: - sub: ["bcdd0b29-123d-11e8-a8bc-b69930b94f5c"] - url: http://geminiservice.analytics/api/v1/user-repo/scan - method: POST - body: '{ - "git-url": "https://github.com/fabric8-services/fabric8-wit", - "email-ids": [ - "sdeshmuk@redhat.com" - ] -}' - response: - status: 500 Internal Server Error - code: 500 - body: '{"foo":"bar"}' - diff --git a/test/data/gemini-scan/register-weird-response.yaml b/test/data/gemini-scan/register-weird-response.yaml deleted file mode 100644 index f5e013b329..0000000000 --- a/test/data/gemini-scan/register-weird-response.yaml +++ /dev/null @@ -1,19 +0,0 @@ ---- -version: 1 -interactions: -- request: - headers: - sub: ["bcdd0b29-123d-11e8-a8bc-b69930b94f5c"] - url: http://geminiservice.analytics/api/v1/user-repo/scan - method: POST - body: '{ - "git-url": "https://github.com/fabric8-services/fabric8-wit", - "email-ids": [ - "sdeshmuk@redhat.com" - ] -}' - response: - status: 200 OK - code: 200 - body: '' -