From bf588bee5f0bb3ab6a11443bdccedd78716687d0 Mon Sep 17 00:00:00 2001 From: Antonio Musolino Date: Thu, 7 Mar 2024 10:24:36 +0100 Subject: [PATCH] feat: added gitlab user crawling (#6424) * feat(gitlab): add all users fetch * chore: archived model should not be modified. --- backend/plugins/gitlab/e2e/account_test.go | 77 +++++++++++++++++++ .../_raw_gitlab_api_users_direct_api.csv | 7 ++ .../_tool_gitlab_accounts_direct_api.csv | 7 ++ .../snapshot_tables/accounts_direct_api.csv | 7 ++ backend/plugins/gitlab/models/account.go | 21 ++--- ...add_created_user_at_tool_gitlab_account.go | 48 ++++++++++++ .../models/migrationscripts/register.go | 1 + .../plugins/gitlab/tasks/account_collector.go | 6 ++ .../plugins/gitlab/tasks/account_convertor.go | 1 + .../plugins/gitlab/tasks/account_extractor.go | 40 +++++++--- 10 files changed, 197 insertions(+), 18 deletions(-) create mode 100644 backend/plugins/gitlab/e2e/raw_tables/_raw_gitlab_api_users_direct_api.csv create mode 100644 backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_accounts_direct_api.csv create mode 100644 backend/plugins/gitlab/e2e/snapshot_tables/accounts_direct_api.csv create mode 100644 backend/plugins/gitlab/models/migrationscripts/20231220_add_created_user_at_tool_gitlab_account.go diff --git a/backend/plugins/gitlab/e2e/account_test.go b/backend/plugins/gitlab/e2e/account_test.go index 8b1f48b7921..26f20c0d7bd 100644 --- a/backend/plugins/gitlab/e2e/account_test.go +++ b/backend/plugins/gitlab/e2e/account_test.go @@ -19,9 +19,11 @@ package e2e import ( "testing" + "time" "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/impl" "github.com/apache/incubator-devlake/plugins/gitlab/models" "github.com/apache/incubator-devlake/plugins/gitlab/tasks" @@ -31,6 +33,12 @@ func TestGitlabAccountDataFlow(t *testing.T) { var gitlab impl.Gitlab dataflowTester := e2ehelper.NewDataFlowTester(t, "gitlab", gitlab) + apiClient := &api.ApiClient{} + apiClient.Setup( + "https://gitlab.com", + make(map[string]string), + time.Hour, + ) taskData := &tasks.GitlabTaskData{ Options: &tasks.GitlabOptions{ @@ -38,6 +46,10 @@ func TestGitlabAccountDataFlow(t *testing.T) { ProjectId: 12345678, ScopeConfig: new(models.GitlabScopeConfig), }, + + ApiClient: &api.ApiAsyncClient{ + ApiClient: apiClient, + }, } // import raw data table @@ -81,3 +93,68 @@ func TestGitlabAccountDataFlow(t *testing.T) { ), ) } + +func TestGitlabAccountDataFlowUsersApi(t *testing.T) { + + var gitlab impl.Gitlab + dataflowTester := e2ehelper.NewDataFlowTester(t, "gitlab", gitlab) + apiClient := &api.ApiClient{} + apiClient.Setup( + "https://custom.gitlab.com", + make(map[string]string), + time.Hour, + ) + + taskData := &tasks.GitlabTaskData{ + Options: &tasks.GitlabOptions{ + ConnectionId: 1, + ProjectId: 12345678, + ScopeConfig: new(models.GitlabScopeConfig), + }, + + ApiClient: &api.ApiAsyncClient{ + ApiClient: apiClient, + }, + } + + // import raw data table + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_gitlab_api_users_direct_api.csv", + "_raw_gitlab_api_users") + + // verify extraction + dataflowTester.FlushTabler(&models.GitlabAccount{}) + dataflowTester.Subtask(tasks.ExtractAccountsMeta, taskData) + dataflowTester.VerifyTable( + models.GitlabAccount{}, + "./snapshot_tables/_tool_gitlab_accounts_direct_api.csv", + e2ehelper.ColumnWithRawData( + "connection_id", + "gitlab_id", + "username", + "email", + "name", + "state", + "avatar_url", + "web_url", + "created_user_at", + ), + ) + + // verify conversion + dataflowTester.FlushTabler(&crossdomain.Account{}) + dataflowTester.Subtask(tasks.ConvertAccountsMeta, taskData) + dataflowTester.VerifyTable( + crossdomain.Account{}, + "./snapshot_tables/accounts_direct_api.csv", + e2ehelper.ColumnWithRawData( + "id", + "email", + "full_name", + "user_name", + "avatar_url", + "organization", + "created_date", + "status", + ), + ) +} diff --git a/backend/plugins/gitlab/e2e/raw_tables/_raw_gitlab_api_users_direct_api.csv b/backend/plugins/gitlab/e2e/raw_tables/_raw_gitlab_api_users_direct_api.csv new file mode 100644 index 00000000000..d8bf76f555a --- /dev/null +++ b/backend/plugins/gitlab/e2e/raw_tables/_raw_gitlab_api_users_direct_api.csv @@ -0,0 +1,7 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":2994198,""username"":""abc1"",""name"":""abc1"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc1"",""created_at"":""2018-03-25T09:43:17.048+01:00"",""bio"":"""",""location"":"""",""public_email"":"""",""skype"":"""",""linkedin"":"""",""twitter"":"""",""discord"":"""",""website_url"":"""",""organization"":"""",""job_title"":"""",""pronouns"":null,""bot"":false,""work_information"":null,""followers"":0,""following"":1,""is_followed"":false,""local_time"":null,""last_sign_in_at"":""2024-01-04T09:43:11.817+01:00"",""confirmed_at"":""2021-03-25T09:43:16.650+01:00"",""last_activity_on"":""2024-01-09"",""email"":""abc1@dev.com"",""theme_id"":11,""color_scheme_id"":2,""projects_limit"":10,""current_sign_in_at"":""2024-01-08T16:16:41.386+01:00"",""identities"":[],""can_create_group"":false,""can_create_project"":true,""two_factor_enabled"":false,""external"":false,""private_profile"":false,""commit_email"":""abc1@dev.com"",""shared_runners_minutes_limit"":null,""extra_shared_runners_minutes_limit"":null,""scim_identities"":[],""is_admin"":false,""note"":null,""namespace_id"":351,""created_by"":{""id"":139,""username"":""abc2"",""name"":""abc2"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc2""},""using_license_seat"":true,""is_auditor"":false,""provisioned_by_group_id"":null,""enterprise_group_id"":null,""enterprise_group_associated_at"":null}",https://custom.gitlab.com/api/v4/users,null,2022-07-13 13:15:05.261 +2,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":2994199,""username"":""abc2"",""name"":""abc2"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc2"",""created_at"":""2019-03-25T09:43:17.048+01:00"",""bio"":"""",""location"":"""",""public_email"":"""",""skype"":"""",""linkedin"":"""",""twitter"":"""",""discord"":"""",""website_url"":"""",""organization"":"""",""job_title"":"""",""pronouns"":null,""bot"":false,""work_information"":null,""followers"":0,""following"":1,""is_followed"":false,""local_time"":null,""last_sign_in_at"":""2024-01-04T09:43:11.817+01:00"",""confirmed_at"":""2021-03-25T09:43:16.650+01:00"",""last_activity_on"":""2024-01-09"",""email"":""abc2@dev.com"",""theme_id"":11,""color_scheme_id"":2,""projects_limit"":10,""current_sign_in_at"":""2024-01-08T16:16:41.386+01:00"",""identities"":[],""can_create_group"":false,""can_create_project"":true,""two_factor_enabled"":false,""external"":false,""private_profile"":false,""commit_email"":""abc2@dev.com"",""shared_runners_minutes_limit"":null,""extra_shared_runners_minutes_limit"":null,""scim_identities"":[],""is_admin"":false,""note"":null,""namespace_id"":351,""created_by"":{""id"":139,""username"":""abc2"",""name"":""abc2"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc2""},""using_license_seat"":true,""is_auditor"":false,""provisioned_by_group_id"":null,""enterprise_group_id"":null,""enterprise_group_associated_at"":null}",https://custom.gitlab.com/api/v4/users,null,2022-07-13 13:15:05.261 +3,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":2994200,""username"":""abc3"",""name"":""abc3"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc3"",""created_at"":""2020-03-25T09:43:17.048+01:00"",""bio"":"""",""location"":"""",""public_email"":"""",""skype"":"""",""linkedin"":"""",""twitter"":"""",""discord"":"""",""website_url"":"""",""organization"":"""",""job_title"":"""",""pronouns"":null,""bot"":false,""work_information"":null,""followers"":0,""following"":1,""is_followed"":false,""local_time"":null,""last_sign_in_at"":""2024-01-04T09:43:11.817+01:00"",""confirmed_at"":""2021-03-25T09:43:16.650+01:00"",""last_activity_on"":""2024-01-09"",""email"":""abc3@dev.com"",""theme_id"":11,""color_scheme_id"":2,""projects_limit"":10,""current_sign_in_at"":""2024-01-08T16:16:41.386+01:00"",""identities"":[],""can_create_group"":false,""can_create_project"":true,""two_factor_enabled"":false,""external"":false,""private_profile"":false,""commit_email"":""abc2@dev.com"",""shared_runners_minutes_limit"":null,""extra_shared_runners_minutes_limit"":null,""scim_identities"":[],""is_admin"":false,""note"":null,""namespace_id"":351,""created_by"":{""id"":139,""username"":""abc2"",""name"":""abc2"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc2""},""using_license_seat"":true,""is_auditor"":false,""provisioned_by_group_id"":null,""enterprise_group_id"":null,""enterprise_group_associated_at"":null}",https://custom.gitlab.com/api/v4/users,null,2022-07-13 13:15:05.261 +4,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":2944200,""username"":""abc4"",""name"":""abc4"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc4"",""created_at"":""2021-03-25T09:43:17.048+01:00"",""bio"":"""",""location"":"""",""public_email"":"""",""skype"":"""",""linkedin"":"""",""twitter"":"""",""discord"":"""",""website_url"":"""",""organization"":"""",""job_title"":"""",""pronouns"":null,""bot"":false,""work_information"":null,""followers"":0,""following"":1,""is_followed"":false,""local_time"":null,""last_sign_in_at"":""2024-01-04T09:43:11.817+01:00"",""confirmed_at"":""2021-03-25T09:43:16.650+01:00"",""last_activity_on"":""2024-01-09"",""email"":""abc4@dev.com"",""theme_id"":11,""color_scheme_id"":2,""projects_limit"":10,""current_sign_in_at"":""2024-01-08T16:16:41.386+01:00"",""identities"":[],""can_create_group"":false,""can_create_project"":true,""two_factor_enabled"":false,""external"":false,""private_profile"":false,""commit_email"":""abc2@dev.com"",""shared_runners_minutes_limit"":null,""extra_shared_runners_minutes_limit"":null,""scim_identities"":[],""is_admin"":false,""note"":null,""namespace_id"":351,""created_by"":{""id"":139,""username"":""abc2"",""name"":""abc2"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc2""},""using_license_seat"":true,""is_auditor"":false,""provisioned_by_group_id"":null,""enterprise_group_id"":null,""enterprise_group_associated_at"":null}",https://custom.gitlab.com/api/v4/users,null,2022-07-13 13:15:05.261 +5,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":2944220,""username"":""abc5"",""name"":""abc5"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc5"",""created_at"":""2022-03-25T09:43:17.048+01:00"",""bio"":"""",""location"":"""",""public_email"":"""",""skype"":"""",""linkedin"":"""",""twitter"":"""",""discord"":"""",""website_url"":"""",""organization"":"""",""job_title"":"""",""pronouns"":null,""bot"":false,""work_information"":null,""followers"":0,""following"":1,""is_followed"":false,""local_time"":null,""last_sign_in_at"":""2024-01-04T09:43:11.817+01:00"",""confirmed_at"":""2021-03-25T09:43:16.650+01:00"",""last_activity_on"":""2024-01-09"",""email"":""abc5@dev.com"",""theme_id"":11,""color_scheme_id"":2,""projects_limit"":10,""current_sign_in_at"":""2024-01-08T16:16:41.386+01:00"",""identities"":[],""can_create_group"":false,""can_create_project"":true,""two_factor_enabled"":false,""external"":false,""private_profile"":false,""commit_email"":""abc2@dev.com"",""shared_runners_minutes_limit"":null,""extra_shared_runners_minutes_limit"":null,""scim_identities"":[],""is_admin"":false,""note"":null,""namespace_id"":351,""created_by"":{""id"":139,""username"":""abc2"",""name"":""abc2"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc2""},""using_license_seat"":true,""is_auditor"":false,""provisioned_by_group_id"":null,""enterprise_group_id"":null,""enterprise_group_associated_at"":null}",https://custom.gitlab.com/api/v4/users,null,2022-07-13 13:15:05.261 +6,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":2944221,""username"":""abc6"",""name"":""abc6"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc6"",""created_at"":""2023-03-25T09:43:17.048+01:00"",""bio"":"""",""location"":"""",""public_email"":"""",""skype"":"""",""linkedin"":"""",""twitter"":"""",""discord"":"""",""website_url"":"""",""organization"":"""",""job_title"":"""",""pronouns"":null,""bot"":false,""work_information"":null,""followers"":0,""following"":1,""is_followed"":false,""local_time"":null,""last_sign_in_at"":""2024-01-04T09:43:11.817+01:00"",""confirmed_at"":""2021-03-25T09:43:16.650+01:00"",""last_activity_on"":""2024-01-09"",""email"":""abc6@dev.com"",""theme_id"":11,""color_scheme_id"":2,""projects_limit"":10,""current_sign_in_at"":""2024-01-08T16:16:41.386+01:00"",""identities"":[],""can_create_group"":false,""can_create_project"":true,""two_factor_enabled"":false,""external"":false,""private_profile"":false,""commit_email"":""abc2@dev.com"",""shared_runners_minutes_limit"":null,""extra_shared_runners_minutes_limit"":null,""scim_identities"":[],""is_admin"":false,""note"":null,""namespace_id"":351,""created_by"":{""id"":139,""username"":""abc2"",""name"":""abc2"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc2""},""using_license_seat"":true,""is_auditor"":false,""provisioned_by_group_id"":null,""enterprise_group_id"":null,""enterprise_group_associated_at"":null}",https://custom.gitlab.com/api/v4/users,null,2022-07-13 13:15:05.261 diff --git a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_accounts_direct_api.csv b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_accounts_direct_api.csv new file mode 100644 index 00000000000..83522143908 --- /dev/null +++ b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_accounts_direct_api.csv @@ -0,0 +1,7 @@ +connection_id,gitlab_id,username,email,name,state,membership_state,avatar_url,web_url,created_user_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +1,2994198,abc1,abc1@dev.com,abc1,active,active,,https://custom.gitlab.com/abc1,2018-03-25T08:43:17.048+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,1, +1,2994199,abc2,abc2@dev.com,abc2,active,active,,https://custom.gitlab.com/abc2,2019-03-25T08:43:17.048+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,2, +1,2994200,abc3,abc3@dev.com,abc3,active,active,,https://custom.gitlab.com/abc3,2020-03-25T08:43:17.048+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,3, +1,2944200,abc4,abc4@dev.com,abc4,active,active,,https://custom.gitlab.com/abc4,2021-03-25T08:43:17.048+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,4, +1,2944220,abc5,abc5@dev.com,abc5,active,active,,https://custom.gitlab.com/abc5,2022-03-25T08:43:17.048+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,5, +1,2944221,abc6,abc6@dev.com,abc6,active,active,,https://custom.gitlab.com/abc6,2023-03-25T08:43:17.048+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,6, diff --git a/backend/plugins/gitlab/e2e/snapshot_tables/accounts_direct_api.csv b/backend/plugins/gitlab/e2e/snapshot_tables/accounts_direct_api.csv new file mode 100644 index 00000000000..0e9377b11d4 --- /dev/null +++ b/backend/plugins/gitlab/e2e/snapshot_tables/accounts_direct_api.csv @@ -0,0 +1,7 @@ +id,email,full_name,user_name,avatar_url,organization,created_date,status,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +gitlab:GitlabAccount:1:2994198,abc1@dev.com,abc1,abc1,,,2018-03-25T08:43:17.048+00:00,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,1, +gitlab:GitlabAccount:1:2994199,abc2@dev.com,abc2,abc2,,,2019-03-25T08:43:17.048+00:00,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,2, +gitlab:GitlabAccount:1:2994200,abc3@dev.com,abc3,abc3,,,2020-03-25T08:43:17.048+00:00,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,3, +gitlab:GitlabAccount:1:2944200,abc4@dev.com,abc4,abc4,,,2021-03-25T08:43:17.048+00:00,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,4, +gitlab:GitlabAccount:1:2944220,abc5@dev.com,abc5,abc5,,,2022-03-25T08:43:17.048+00:00,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,5, +gitlab:GitlabAccount:1:2944221,abc6@dev.com,abc6,abc6,,,2023-03-25T08:43:17.048+00:00,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,6, diff --git a/backend/plugins/gitlab/models/account.go b/backend/plugins/gitlab/models/account.go index f9f8f1e3346..d646583ed9c 100644 --- a/backend/plugins/gitlab/models/account.go +++ b/backend/plugins/gitlab/models/account.go @@ -18,19 +18,22 @@ limitations under the License. package models import ( + "time" + "github.com/apache/incubator-devlake/core/models/common" ) type GitlabAccount struct { - ConnectionId uint64 `gorm:"primaryKey"` - GitlabId int `gorm:"primaryKey" json:"id"` - Username string `gorm:"type:varchar(255)"` - Email string `gorm:"type:varchar(255)"` - Name string `gorm:"type:varchar(255)"` - State string `gorm:"type:varchar(255)"` - MembershipState string `json:"membership_state" gorm:"type:varchar(255)"` - AvatarUrl string `json:"avatar_url" gorm:"type:varchar(255)"` - WebUrl string `json:"web_url" gorm:"type:varchar(255)"` + ConnectionId uint64 `gorm:"primaryKey"` + GitlabId int `gorm:"primaryKey" json:"id"` + Username string `gorm:"type:varchar(255)"` + Email string `gorm:"type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + State string `gorm:"type:varchar(255)"` + MembershipState string `json:"membership_state" gorm:"type:varchar(255)"` + AvatarUrl string `json:"avatar_url" gorm:"type:varchar(255)"` + WebUrl string `json:"web_url" gorm:"type:varchar(255)"` + CreatedUserAt *time.Time `json:"created_at"` common.NoPKModel } diff --git a/backend/plugins/gitlab/models/migrationscripts/20231220_add_created_user_at_tool_gitlab_account.go b/backend/plugins/gitlab/models/migrationscripts/20231220_add_created_user_at_tool_gitlab_account.go new file mode 100644 index 00000000000..f46c0006c24 --- /dev/null +++ b/backend/plugins/gitlab/models/migrationscripts/20231220_add_created_user_at_tool_gitlab_account.go @@ -0,0 +1,48 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type gitlabAccount20231120 struct { + CreatedUserAt *time.Time +} + +func (gitlabAccount20231120) TableName() string { + return "_tool_gitlab_accounts" +} + +type addUserCreationAt20231120 struct{} + +func (*addUserCreationAt20231120) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(basicRes, &gitlabAccount20231120{}) +} + +func (*addUserCreationAt20231120) Version() uint64 { + return 20231226140000 +} + +func (*addUserCreationAt20231120) Name() string { + return "add created_user_at column to _tool_gitlab_accounts" +} diff --git a/backend/plugins/gitlab/models/migrationscripts/register.go b/backend/plugins/gitlab/models/migrationscripts/register.go index 0ec89372777..c6d97dd2015 100644 --- a/backend/plugins/gitlab/models/migrationscripts/register.go +++ b/backend/plugins/gitlab/models/migrationscripts/register.go @@ -41,6 +41,7 @@ func All() []plugin.MigrationScript { new(addRawParamTableForScope), new(addProjectArchived), new(addDeployment), + new(addUserCreationAt20231120), new(addEnvNamePattern), new(addQueuedDuration20231129), new(modifyDeploymentMessageType), diff --git a/backend/plugins/gitlab/tasks/account_collector.go b/backend/plugins/gitlab/tasks/account_collector.go index fb21f2b9e93..4d234056c0a 100644 --- a/backend/plugins/gitlab/tasks/account_collector.go +++ b/backend/plugins/gitlab/tasks/account_collector.go @@ -22,6 +22,7 @@ import ( "fmt" "net/http" "net/url" + "strings" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" @@ -56,6 +57,11 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error { urlTemplate = "/projects/{{ .Params.ProjectId }}/members/" } + // Collect all users if endpoint is private gitlab instance + if !strings.HasPrefix(data.ApiClient.GetEndpoint(), "https://gitlab.com") { + urlTemplate = "/users" + } + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ RawDataSubTaskArgs: *rawDataSubTaskArgs, ApiClient: data.ApiClient, diff --git a/backend/plugins/gitlab/tasks/account_convertor.go b/backend/plugins/gitlab/tasks/account_convertor.go index ec18679bedc..9db00334c96 100644 --- a/backend/plugins/gitlab/tasks/account_convertor.go +++ b/backend/plugins/gitlab/tasks/account_convertor.go @@ -73,6 +73,7 @@ func ConvertAccounts(taskCtx plugin.SubTaskContext) errors.Error { FullName: GitlabAccount.Name, Email: GitlabAccount.Email, AvatarUrl: GitlabAccount.AvatarUrl, + CreatedDate: GitlabAccount.CreatedUserAt, } return []interface{}{ diff --git a/backend/plugins/gitlab/tasks/account_extractor.go b/backend/plugins/gitlab/tasks/account_extractor.go index 1b57a7b2ac4..1ab39dcb7bd 100644 --- a/backend/plugins/gitlab/tasks/account_extractor.go +++ b/backend/plugins/gitlab/tasks/account_extractor.go @@ -19,6 +19,7 @@ package tasks import ( "encoding/json" + "strings" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" @@ -42,6 +43,9 @@ var ExtractAccountsMeta = plugin.SubTaskMeta{ func ExtractAccounts(taskCtx plugin.SubTaskContext) errors.Error { rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_USER_TABLE) + // Do not extract createdUserAt if we are not using /users API + var skipCreatedUserAt = strings.HasPrefix(data.ApiClient.GetEndpoint(), "https://gitlab.com") + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ RawDataSubTaskArgs: *rawDataSubTaskArgs, Extract: func(row *api.RawData) ([]interface{}, errors.Error) { @@ -52,16 +56,34 @@ func ExtractAccounts(taskCtx plugin.SubTaskContext) errors.Error { } results := make([]interface{}, 0) - GitlabAccount := &models.GitlabAccount{ - ConnectionId: data.Options.ConnectionId, - GitlabId: userRes.GitlabId, - Username: userRes.Username, - Name: userRes.Name, - State: userRes.State, - MembershipState: userRes.MembershipState, - AvatarUrl: userRes.AvatarUrl, - WebUrl: userRes.WebUrl, + var GitlabAccount *models.GitlabAccount + if skipCreatedUserAt { + GitlabAccount = &models.GitlabAccount{ + ConnectionId: data.Options.ConnectionId, + GitlabId: userRes.GitlabId, + Username: userRes.Username, + Name: userRes.Name, + State: userRes.State, + MembershipState: userRes.MembershipState, + AvatarUrl: userRes.AvatarUrl, + WebUrl: userRes.WebUrl, + Email: userRes.Email, + } + } else { + GitlabAccount = &models.GitlabAccount{ + ConnectionId: data.Options.ConnectionId, + GitlabId: userRes.GitlabId, + Username: userRes.Username, + Name: userRes.Name, + State: userRes.State, + MembershipState: userRes.MembershipState, + AvatarUrl: userRes.AvatarUrl, + WebUrl: userRes.WebUrl, + Email: userRes.Email, + CreatedUserAt: userRes.CreatedUserAt, + } } + results = append(results, GitlabAccount) return results, nil