Skip to content

Commit

Permalink
feat: added gitlab user crawling (#6424)
Browse files Browse the repository at this point in the history
* feat(gitlab): add all users fetch

* chore: archived model should not be modified.
  • Loading branch information
antoniomuso authored Mar 7, 2024
1 parent 5750d88 commit bf588be
Show file tree
Hide file tree
Showing 10 changed files with 197 additions and 18 deletions.
77 changes: 77 additions & 0 deletions backend/plugins/gitlab/e2e/account_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,11 @@ package e2e

import (
"testing"
"time"

"github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/impl"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
"github.com/apache/incubator-devlake/plugins/gitlab/tasks"
Expand All @@ -31,13 +33,23 @@ func TestGitlabAccountDataFlow(t *testing.T) {

var gitlab impl.Gitlab
dataflowTester := e2ehelper.NewDataFlowTester(t, "gitlab", gitlab)
apiClient := &api.ApiClient{}
apiClient.Setup(
"https://gitlab.com",
make(map[string]string),
time.Hour,
)

taskData := &tasks.GitlabTaskData{
Options: &tasks.GitlabOptions{
ConnectionId: 1,
ProjectId: 12345678,
ScopeConfig: new(models.GitlabScopeConfig),
},

ApiClient: &api.ApiAsyncClient{
ApiClient: apiClient,
},
}

// import raw data table
Expand Down Expand Up @@ -81,3 +93,68 @@ func TestGitlabAccountDataFlow(t *testing.T) {
),
)
}

func TestGitlabAccountDataFlowUsersApi(t *testing.T) {

var gitlab impl.Gitlab
dataflowTester := e2ehelper.NewDataFlowTester(t, "gitlab", gitlab)
apiClient := &api.ApiClient{}
apiClient.Setup(
"https://custom.gitlab.com",
make(map[string]string),
time.Hour,
)

taskData := &tasks.GitlabTaskData{
Options: &tasks.GitlabOptions{
ConnectionId: 1,
ProjectId: 12345678,
ScopeConfig: new(models.GitlabScopeConfig),
},

ApiClient: &api.ApiAsyncClient{
ApiClient: apiClient,
},
}

// import raw data table
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_gitlab_api_users_direct_api.csv",
"_raw_gitlab_api_users")

// verify extraction
dataflowTester.FlushTabler(&models.GitlabAccount{})
dataflowTester.Subtask(tasks.ExtractAccountsMeta, taskData)
dataflowTester.VerifyTable(
models.GitlabAccount{},
"./snapshot_tables/_tool_gitlab_accounts_direct_api.csv",
e2ehelper.ColumnWithRawData(
"connection_id",
"gitlab_id",
"username",
"email",
"name",
"state",
"avatar_url",
"web_url",
"created_user_at",
),
)

// verify conversion
dataflowTester.FlushTabler(&crossdomain.Account{})
dataflowTester.Subtask(tasks.ConvertAccountsMeta, taskData)
dataflowTester.VerifyTable(
crossdomain.Account{},
"./snapshot_tables/accounts_direct_api.csv",
e2ehelper.ColumnWithRawData(
"id",
"email",
"full_name",
"user_name",
"avatar_url",
"organization",
"created_date",
"status",
),
)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
id,params,data,url,input,created_at
1,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":2994198,""username"":""abc1"",""name"":""abc1"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc1"",""created_at"":""2018-03-25T09:43:17.048+01:00"",""bio"":"""",""location"":"""",""public_email"":"""",""skype"":"""",""linkedin"":"""",""twitter"":"""",""discord"":"""",""website_url"":"""",""organization"":"""",""job_title"":"""",""pronouns"":null,""bot"":false,""work_information"":null,""followers"":0,""following"":1,""is_followed"":false,""local_time"":null,""last_sign_in_at"":""2024-01-04T09:43:11.817+01:00"",""confirmed_at"":""2021-03-25T09:43:16.650+01:00"",""last_activity_on"":""2024-01-09"",""email"":""abc1@dev.com"",""theme_id"":11,""color_scheme_id"":2,""projects_limit"":10,""current_sign_in_at"":""2024-01-08T16:16:41.386+01:00"",""identities"":[],""can_create_group"":false,""can_create_project"":true,""two_factor_enabled"":false,""external"":false,""private_profile"":false,""commit_email"":""abc1@dev.com"",""shared_runners_minutes_limit"":null,""extra_shared_runners_minutes_limit"":null,""scim_identities"":[],""is_admin"":false,""note"":null,""namespace_id"":351,""created_by"":{""id"":139,""username"":""abc2"",""name"":""abc2"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc2""},""using_license_seat"":true,""is_auditor"":false,""provisioned_by_group_id"":null,""enterprise_group_id"":null,""enterprise_group_associated_at"":null}",https://custom.gitlab.com/api/v4/users,null,2022-07-13 13:15:05.261
2,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":2994199,""username"":""abc2"",""name"":""abc2"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc2"",""created_at"":""2019-03-25T09:43:17.048+01:00"",""bio"":"""",""location"":"""",""public_email"":"""",""skype"":"""",""linkedin"":"""",""twitter"":"""",""discord"":"""",""website_url"":"""",""organization"":"""",""job_title"":"""",""pronouns"":null,""bot"":false,""work_information"":null,""followers"":0,""following"":1,""is_followed"":false,""local_time"":null,""last_sign_in_at"":""2024-01-04T09:43:11.817+01:00"",""confirmed_at"":""2021-03-25T09:43:16.650+01:00"",""last_activity_on"":""2024-01-09"",""email"":""abc2@dev.com"",""theme_id"":11,""color_scheme_id"":2,""projects_limit"":10,""current_sign_in_at"":""2024-01-08T16:16:41.386+01:00"",""identities"":[],""can_create_group"":false,""can_create_project"":true,""two_factor_enabled"":false,""external"":false,""private_profile"":false,""commit_email"":""abc2@dev.com"",""shared_runners_minutes_limit"":null,""extra_shared_runners_minutes_limit"":null,""scim_identities"":[],""is_admin"":false,""note"":null,""namespace_id"":351,""created_by"":{""id"":139,""username"":""abc2"",""name"":""abc2"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc2""},""using_license_seat"":true,""is_auditor"":false,""provisioned_by_group_id"":null,""enterprise_group_id"":null,""enterprise_group_associated_at"":null}",https://custom.gitlab.com/api/v4/users,null,2022-07-13 13:15:05.261
3,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":2994200,""username"":""abc3"",""name"":""abc3"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc3"",""created_at"":""2020-03-25T09:43:17.048+01:00"",""bio"":"""",""location"":"""",""public_email"":"""",""skype"":"""",""linkedin"":"""",""twitter"":"""",""discord"":"""",""website_url"":"""",""organization"":"""",""job_title"":"""",""pronouns"":null,""bot"":false,""work_information"":null,""followers"":0,""following"":1,""is_followed"":false,""local_time"":null,""last_sign_in_at"":""2024-01-04T09:43:11.817+01:00"",""confirmed_at"":""2021-03-25T09:43:16.650+01:00"",""last_activity_on"":""2024-01-09"",""email"":""abc3@dev.com"",""theme_id"":11,""color_scheme_id"":2,""projects_limit"":10,""current_sign_in_at"":""2024-01-08T16:16:41.386+01:00"",""identities"":[],""can_create_group"":false,""can_create_project"":true,""two_factor_enabled"":false,""external"":false,""private_profile"":false,""commit_email"":""abc2@dev.com"",""shared_runners_minutes_limit"":null,""extra_shared_runners_minutes_limit"":null,""scim_identities"":[],""is_admin"":false,""note"":null,""namespace_id"":351,""created_by"":{""id"":139,""username"":""abc2"",""name"":""abc2"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc2""},""using_license_seat"":true,""is_auditor"":false,""provisioned_by_group_id"":null,""enterprise_group_id"":null,""enterprise_group_associated_at"":null}",https://custom.gitlab.com/api/v4/users,null,2022-07-13 13:15:05.261
4,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":2944200,""username"":""abc4"",""name"":""abc4"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc4"",""created_at"":""2021-03-25T09:43:17.048+01:00"",""bio"":"""",""location"":"""",""public_email"":"""",""skype"":"""",""linkedin"":"""",""twitter"":"""",""discord"":"""",""website_url"":"""",""organization"":"""",""job_title"":"""",""pronouns"":null,""bot"":false,""work_information"":null,""followers"":0,""following"":1,""is_followed"":false,""local_time"":null,""last_sign_in_at"":""2024-01-04T09:43:11.817+01:00"",""confirmed_at"":""2021-03-25T09:43:16.650+01:00"",""last_activity_on"":""2024-01-09"",""email"":""abc4@dev.com"",""theme_id"":11,""color_scheme_id"":2,""projects_limit"":10,""current_sign_in_at"":""2024-01-08T16:16:41.386+01:00"",""identities"":[],""can_create_group"":false,""can_create_project"":true,""two_factor_enabled"":false,""external"":false,""private_profile"":false,""commit_email"":""abc2@dev.com"",""shared_runners_minutes_limit"":null,""extra_shared_runners_minutes_limit"":null,""scim_identities"":[],""is_admin"":false,""note"":null,""namespace_id"":351,""created_by"":{""id"":139,""username"":""abc2"",""name"":""abc2"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc2""},""using_license_seat"":true,""is_auditor"":false,""provisioned_by_group_id"":null,""enterprise_group_id"":null,""enterprise_group_associated_at"":null}",https://custom.gitlab.com/api/v4/users,null,2022-07-13 13:15:05.261
5,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":2944220,""username"":""abc5"",""name"":""abc5"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc5"",""created_at"":""2022-03-25T09:43:17.048+01:00"",""bio"":"""",""location"":"""",""public_email"":"""",""skype"":"""",""linkedin"":"""",""twitter"":"""",""discord"":"""",""website_url"":"""",""organization"":"""",""job_title"":"""",""pronouns"":null,""bot"":false,""work_information"":null,""followers"":0,""following"":1,""is_followed"":false,""local_time"":null,""last_sign_in_at"":""2024-01-04T09:43:11.817+01:00"",""confirmed_at"":""2021-03-25T09:43:16.650+01:00"",""last_activity_on"":""2024-01-09"",""email"":""abc5@dev.com"",""theme_id"":11,""color_scheme_id"":2,""projects_limit"":10,""current_sign_in_at"":""2024-01-08T16:16:41.386+01:00"",""identities"":[],""can_create_group"":false,""can_create_project"":true,""two_factor_enabled"":false,""external"":false,""private_profile"":false,""commit_email"":""abc2@dev.com"",""shared_runners_minutes_limit"":null,""extra_shared_runners_minutes_limit"":null,""scim_identities"":[],""is_admin"":false,""note"":null,""namespace_id"":351,""created_by"":{""id"":139,""username"":""abc2"",""name"":""abc2"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc2""},""using_license_seat"":true,""is_auditor"":false,""provisioned_by_group_id"":null,""enterprise_group_id"":null,""enterprise_group_associated_at"":null}",https://custom.gitlab.com/api/v4/users,null,2022-07-13 13:15:05.261
6,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":2944221,""username"":""abc6"",""name"":""abc6"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc6"",""created_at"":""2023-03-25T09:43:17.048+01:00"",""bio"":"""",""location"":"""",""public_email"":"""",""skype"":"""",""linkedin"":"""",""twitter"":"""",""discord"":"""",""website_url"":"""",""organization"":"""",""job_title"":"""",""pronouns"":null,""bot"":false,""work_information"":null,""followers"":0,""following"":1,""is_followed"":false,""local_time"":null,""last_sign_in_at"":""2024-01-04T09:43:11.817+01:00"",""confirmed_at"":""2021-03-25T09:43:16.650+01:00"",""last_activity_on"":""2024-01-09"",""email"":""abc6@dev.com"",""theme_id"":11,""color_scheme_id"":2,""projects_limit"":10,""current_sign_in_at"":""2024-01-08T16:16:41.386+01:00"",""identities"":[],""can_create_group"":false,""can_create_project"":true,""two_factor_enabled"":false,""external"":false,""private_profile"":false,""commit_email"":""abc2@dev.com"",""shared_runners_minutes_limit"":null,""extra_shared_runners_minutes_limit"":null,""scim_identities"":[],""is_admin"":false,""note"":null,""namespace_id"":351,""created_by"":{""id"":139,""username"":""abc2"",""name"":""abc2"",""state"":""active"",""locked"":false,""avatar_url"":"""",""web_url"":""https://custom.gitlab.com/abc2""},""using_license_seat"":true,""is_auditor"":false,""provisioned_by_group_id"":null,""enterprise_group_id"":null,""enterprise_group_associated_at"":null}",https://custom.gitlab.com/api/v4/users,null,2022-07-13 13:15:05.261
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
connection_id,gitlab_id,username,email,name,state,membership_state,avatar_url,web_url,created_user_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
1,2994198,abc1,abc1@dev.com,abc1,active,active,,https://custom.gitlab.com/abc1,2018-03-25T08:43:17.048+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,1,
1,2994199,abc2,abc2@dev.com,abc2,active,active,,https://custom.gitlab.com/abc2,2019-03-25T08:43:17.048+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,2,
1,2994200,abc3,abc3@dev.com,abc3,active,active,,https://custom.gitlab.com/abc3,2020-03-25T08:43:17.048+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,3,
1,2944200,abc4,abc4@dev.com,abc4,active,active,,https://custom.gitlab.com/abc4,2021-03-25T08:43:17.048+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,4,
1,2944220,abc5,abc5@dev.com,abc5,active,active,,https://custom.gitlab.com/abc5,2022-03-25T08:43:17.048+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,5,
1,2944221,abc6,abc6@dev.com,abc6,active,active,,https://custom.gitlab.com/abc6,2023-03-25T08:43:17.048+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,6,
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
id,email,full_name,user_name,avatar_url,organization,created_date,status,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
gitlab:GitlabAccount:1:2994198,abc1@dev.com,abc1,abc1,,,2018-03-25T08:43:17.048+00:00,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,1,
gitlab:GitlabAccount:1:2994199,abc2@dev.com,abc2,abc2,,,2019-03-25T08:43:17.048+00:00,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,2,
gitlab:GitlabAccount:1:2994200,abc3@dev.com,abc3,abc3,,,2020-03-25T08:43:17.048+00:00,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,3,
gitlab:GitlabAccount:1:2944200,abc4@dev.com,abc4,abc4,,,2021-03-25T08:43:17.048+00:00,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,4,
gitlab:GitlabAccount:1:2944220,abc5@dev.com,abc5,abc5,,,2022-03-25T08:43:17.048+00:00,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,5,
gitlab:GitlabAccount:1:2944221,abc6@dev.com,abc6,abc6,,,2023-03-25T08:43:17.048+00:00,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_users,6,
21 changes: 12 additions & 9 deletions backend/plugins/gitlab/models/account.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,19 +18,22 @@ limitations under the License.
package models

import (
"time"

"github.com/apache/incubator-devlake/core/models/common"
)

type GitlabAccount struct {
ConnectionId uint64 `gorm:"primaryKey"`
GitlabId int `gorm:"primaryKey" json:"id"`
Username string `gorm:"type:varchar(255)"`
Email string `gorm:"type:varchar(255)"`
Name string `gorm:"type:varchar(255)"`
State string `gorm:"type:varchar(255)"`
MembershipState string `json:"membership_state" gorm:"type:varchar(255)"`
AvatarUrl string `json:"avatar_url" gorm:"type:varchar(255)"`
WebUrl string `json:"web_url" gorm:"type:varchar(255)"`
ConnectionId uint64 `gorm:"primaryKey"`
GitlabId int `gorm:"primaryKey" json:"id"`
Username string `gorm:"type:varchar(255)"`
Email string `gorm:"type:varchar(255)"`
Name string `gorm:"type:varchar(255)"`
State string `gorm:"type:varchar(255)"`
MembershipState string `json:"membership_state" gorm:"type:varchar(255)"`
AvatarUrl string `json:"avatar_url" gorm:"type:varchar(255)"`
WebUrl string `json:"web_url" gorm:"type:varchar(255)"`
CreatedUserAt *time.Time `json:"created_at"`

common.NoPKModel
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/*
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package migrationscripts

import (
"time"

"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/helpers/migrationhelper"
)

type gitlabAccount20231120 struct {
CreatedUserAt *time.Time
}

func (gitlabAccount20231120) TableName() string {
return "_tool_gitlab_accounts"
}

type addUserCreationAt20231120 struct{}

func (*addUserCreationAt20231120) Up(basicRes context.BasicRes) errors.Error {
return migrationhelper.AutoMigrateTables(basicRes, &gitlabAccount20231120{})
}

func (*addUserCreationAt20231120) Version() uint64 {
return 20231226140000
}

func (*addUserCreationAt20231120) Name() string {
return "add created_user_at column to _tool_gitlab_accounts"
}
1 change: 1 addition & 0 deletions backend/plugins/gitlab/models/migrationscripts/register.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ func All() []plugin.MigrationScript {
new(addRawParamTableForScope),
new(addProjectArchived),
new(addDeployment),
new(addUserCreationAt20231120),
new(addEnvNamePattern),
new(addQueuedDuration20231129),
new(modifyDeploymentMessageType),
Expand Down
6 changes: 6 additions & 0 deletions backend/plugins/gitlab/tasks/account_collector.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"fmt"
"net/http"
"net/url"
"strings"

"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
Expand Down Expand Up @@ -56,6 +57,11 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error {
urlTemplate = "/projects/{{ .Params.ProjectId }}/members/"
}

// Collect all users if endpoint is private gitlab instance
if !strings.HasPrefix(data.ApiClient.GetEndpoint(), "https://gitlab.com") {
urlTemplate = "/users"
}

collector, err := api.NewApiCollector(api.ApiCollectorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
ApiClient: data.ApiClient,
Expand Down
1 change: 1 addition & 0 deletions backend/plugins/gitlab/tasks/account_convertor.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ func ConvertAccounts(taskCtx plugin.SubTaskContext) errors.Error {
FullName: GitlabAccount.Name,
Email: GitlabAccount.Email,
AvatarUrl: GitlabAccount.AvatarUrl,
CreatedDate: GitlabAccount.CreatedUserAt,
}

return []interface{}{
Expand Down
Loading

0 comments on commit bf588be

Please sign in to comment.