Skip to content

Commit

Permalink
Transfer config - Precheck for Docker repo names
Browse files Browse the repository at this point in the history
  • Loading branch information
yahavi committed Nov 5, 2023
1 parent 095debb commit 30235f3
Show file tree
Hide file tree
Showing 15 changed files with 174 additions and 62 deletions.
20 changes: 11 additions & 9 deletions artifactory/commands/transferconfig/transferconfig.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/generic"
"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferconfig/configxmlutils"
commandsUtils "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/utils"
"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/utils/precheckrunner"
"github.com/jfrog/jfrog-cli-core/v2/artifactory/utils"
"github.com/jfrog/jfrog-cli-core/v2/common/commands"
"github.com/jfrog/jfrog-cli-core/v2/utils/config"
Expand Down Expand Up @@ -212,7 +213,7 @@ func (tcc *TransferConfigCommand) runPreChecks() error {
return err
}

return tcc.NewPreChecksRunner(remoteRepositories).Run(context.Background(), tcc.TargetServerDetails)
return tcc.NewPreChecksRunner(selectedRepos, remoteRepositories).Run(context.Background(), tcc.TargetServerDetails)
}

func (tcc *TransferConfigCommand) printWarnings() (err error) {
Expand Down Expand Up @@ -296,16 +297,17 @@ func (tcc *TransferConfigCommand) verifyConfigImportPlugin() error {
}

// Creates the Pre-checks runner for the config import command
func (tcc *TransferConfigCommand) NewPreChecksRunner(remoteRepositories []interface{}) (runner *commandsUtils.PreCheckRunner) {
runner = commandsUtils.NewPreChecksRunner()
func (tcc *TransferConfigCommand) NewPreChecksRunner(selectedRepos map[utils.RepoType][]services.RepositoryDetails, remoteRepositories []interface{}) (runner *precheckrunner.PreCheckRunner) {
runner = precheckrunner.NewPreChecksRunner()

// Add pre-checks here
runner.AddCheck(commandsUtils.NewRemoteRepositoryCheck(&tcc.TargetArtifactoryManager, remoteRepositories))
runner.AddCheck(precheckrunner.NewRepositoryNamingCheck(selectedRepos))
runner.AddCheck(precheckrunner.NewRemoteRepositoryCheck(&tcc.TargetArtifactoryManager, remoteRepositories))

return
}

func (tcc *TransferConfigCommand) getEncryptedItems(selectedSourceRepos map[utils.RepoType][]string) (configXml string, remoteRepositories []interface{}, err error) {
func (tcc *TransferConfigCommand) getEncryptedItems(selectedSourceRepos map[utils.RepoType][]services.RepositoryDetails) (configXml string, remoteRepositories []interface{}, err error) {
reactivateKeyEncryption, err := tcc.DeactivateKeyEncryption()
if err != nil {
return "", nil, err
Expand All @@ -324,10 +326,10 @@ func (tcc *TransferConfigCommand) getEncryptedItems(selectedSourceRepos map[util
}

// Get all remote repositories from the source Artifactory server.
if remoteRepositoryNames, ok := selectedSourceRepos[utils.Remote]; ok && len(remoteRepositoryNames) > 0 {
remoteRepositories = make([]interface{}, len(remoteRepositoryNames))
for i, repoName := range remoteRepositoryNames {
if err = tcc.SourceArtifactoryManager.GetRepository(repoName, &remoteRepositories[i]); err != nil {
if remoteRepositoriesDetails, ok := selectedSourceRepos[utils.Remote]; ok && len(remoteRepositoriesDetails) > 0 {
remoteRepositories = make([]interface{}, len(remoteRepositoriesDetails))
for i, remoteRepositoryDetails := range remoteRepositoriesDetails {
if err = tcc.SourceArtifactoryManager.GetRepository(remoteRepositoryDetails.Key, &remoteRepositories[i]); err != nil {
return
}
}
Expand Down
2 changes: 1 addition & 1 deletion artifactory/commands/transferconfig/transferconfig_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ func TestGetConfigXml(t *testing.T) {

// Test get config xml
transferConfigCmd := createTransferConfigCommand(t, serverDetails, nil)
configXml, _, err := transferConfigCmd.getEncryptedItems(make(map[utils.RepoType][]string))
configXml, _, err := transferConfigCmd.getEncryptedItems(make(map[utils.RepoType][]services.RepositoryDetails))
assert.NoError(t, err)
assert.Equal(t, "<config></config>", configXml)
}
Expand Down
16 changes: 10 additions & 6 deletions artifactory/commands/transferconfigmerge/transferconfigmerge.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ func (tcmc *TransferConfigMergeCommand) SetExcludeProjectsPatterns(excludeProjec

type mergeEntities struct {
projectsToTransfer []accessServices.Project
reposToTransfer map[utils.RepoType][]string
reposToTransfer map[utils.RepoType][]services.RepositoryDetails
}

type Conflict struct {
Expand Down Expand Up @@ -243,8 +243,8 @@ func compareProjects(sourceProject, targetProject accessServices.Project) (*Conf
}, nil
}

func (tcmc *TransferConfigMergeCommand) mergeRepositories(conflicts *[]Conflict) (reposToTransfer map[utils.RepoType][]string, err error) {
reposToTransfer = make(map[utils.RepoType][]string)
func (tcmc *TransferConfigMergeCommand) mergeRepositories(conflicts *[]Conflict) (reposToTransfer map[utils.RepoType][]services.RepositoryDetails, err error) {
reposToTransfer = make(map[utils.RepoType][]services.RepositoryDetails)
sourceRepos, err := tcmc.SourceArtifactoryManager.GetAllRepositories()
if err != nil {
return
Expand Down Expand Up @@ -286,7 +286,7 @@ func (tcmc *TransferConfigMergeCommand) mergeRepositories(conflicts *[]Conflict)
}
} else {
repoType := utils.RepoTypeFromString(sourceRepo.Type)
reposToTransfer[repoType] = append(reposToTransfer[repoType], sourceRepo.Key)
reposToTransfer[repoType] = append(reposToTransfer[repoType], sourceRepo)
}
}
return
Expand Down Expand Up @@ -351,7 +351,7 @@ func (tcmc *TransferConfigMergeCommand) transferProjectsToTarget(reposToTransfer
return
}

func (tcmc *TransferConfigMergeCommand) decryptAndGetAllRemoteRepositories(remoteRepositoryNames []string) (remoteRepositories []interface{}, err error) {
func (tcmc *TransferConfigMergeCommand) decryptAndGetAllRemoteRepositories(remoteRepositoriesDetails []services.RepositoryDetails) (remoteRepositories []interface{}, err error) {
// Decrypt source Artifactory to get remote repositories with raw text passwords
reactivateKeyEncryption, err := tcmc.DeactivateKeyEncryption()
if err != nil {
Expand All @@ -362,7 +362,11 @@ func (tcmc *TransferConfigMergeCommand) decryptAndGetAllRemoteRepositories(remot
err = reactivationErr
}
}()
return tcmc.GetAllRemoteRepositories(remoteRepositoryNames)
var remoteRepositoryKeys []string
for _, remoteRepositoryDetails := range remoteRepositoriesDetails {
remoteRepositoryKeys = append(remoteRepositoryKeys, remoteRepositoryDetails.Key)
}
return tcmc.GetAllRemoteRepositories(remoteRepositoryKeys)
}

type projectsMapper struct {
Expand Down
7 changes: 4 additions & 3 deletions artifactory/commands/transferfiles/longpropertycheck.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/jfrog/gofrog/parallel"
"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api"
cmdutils "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/utils"
"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/utils/precheckrunner"
"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
"github.com/jfrog/jfrog-cli-core/v2/utils/progressbar"
"github.com/jfrog/jfrog-client-go/artifactory"
Expand Down Expand Up @@ -67,7 +68,7 @@ func (lpc *LongPropertyCheck) Name() string {
return longPropertyCheckName
}

func (lpc *LongPropertyCheck) ExecuteCheck(args cmdutils.RunArguments) (passed bool, err error) {
func (lpc *LongPropertyCheck) ExecuteCheck(args precheckrunner.RunArguments) (passed bool, err error) {
// Init producer consumer
lpc.producerConsumer = parallel.NewRunner(threadCount, maxThreadCapacity, false)
lpc.filesChan = make(chan FileWithLongProperty, threadCount)
Expand Down Expand Up @@ -111,7 +112,7 @@ func (lpc *LongPropertyCheck) ExecuteCheck(args cmdutils.RunArguments) (passed b

// Search for long properties in the server and create a search task to find the files that contains them
// Returns the number of long properties found
func (lpc *LongPropertyCheck) longPropertiesTaskProducer(progress *progressbar.TasksProgressBar, args cmdutils.RunArguments) int {
func (lpc *LongPropertyCheck) longPropertiesTaskProducer(progress *progressbar.TasksProgressBar, args precheckrunner.RunArguments) int {
// Init
serviceManager, err := createTransferServiceManager(args.Context, args.ServerDetails)
if err != nil {
Expand Down Expand Up @@ -173,7 +174,7 @@ func getSearchAllPropertiesQuery(pageNumber int) string {

// Create a task that fetch from the server the files with the given property.
// We keep only the files that are at the requested repos and pass them at the files channel
func createSearchPropertyTask(property Property, repos []string, args cmdutils.RunArguments, filesChan chan FileWithLongProperty, progress *progressbar.TasksProgressBar) parallel.TaskFunc {
func createSearchPropertyTask(property Property, repos []string, args precheckrunner.RunArguments, filesChan chan FileWithLongProperty, progress *progressbar.TasksProgressBar) parallel.TaskFunc {
return func(threadId int) (err error) {
serviceManager, err := createTransferServiceManager(args.Context, args.ServerDetails)
if err != nil {
Expand Down
23 changes: 12 additions & 11 deletions artifactory/commands/transferfiles/longpropertycheck_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,22 @@ package transferfiles

import (
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"

"github.com/jfrog/gofrog/parallel"
"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api"
"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/utils"
"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/utils/precheckrunner"
commonTests "github.com/jfrog/jfrog-cli-core/v2/common/tests"
"github.com/jfrog/jfrog-cli-core/v2/utils/config"
"github.com/jfrog/jfrog-client-go/artifactory"
servicesUtils "github.com/jfrog/jfrog-client-go/artifactory/services/utils"
clientutils "github.com/jfrog/jfrog-client-go/utils"
"github.com/stretchr/testify/assert"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
)

var (
Expand Down Expand Up @@ -82,7 +83,7 @@ func testGetLongProperties(t *testing.T, serverProperties, expectedLongPropertie
longPropertyCheck := NewLongPropertyCheck([]string{})
longPropertyCheck.filesChan = make(chan FileWithLongProperty, threadCount)

count := longPropertyCheck.longPropertiesTaskProducer(nil, utils.RunArguments{Context: nil, ServerDetails: serverDetails})
count := longPropertyCheck.longPropertiesTaskProducer(nil, precheckrunner.RunArguments{Context: nil, ServerDetails: serverDetails})
assert.Len(t, expectedLongProperties, count)
}

Expand Down Expand Up @@ -119,7 +120,7 @@ func testSearchPropertyInFilesTask(t *testing.T, prop Property, specificRepos []
wait.Done()
}()

task := createSearchPropertyTask(prop, specificRepos, utils.RunArguments{Context: nil, ServerDetails: serverDetails}, filesChan, nil)
task := createSearchPropertyTask(prop, specificRepos, precheckrunner.RunArguments{Context: nil, ServerDetails: serverDetails}, filesChan, nil)
assert.NoError(t, task(0))
close(filesChan)
wait.Wait()
Expand Down Expand Up @@ -170,7 +171,7 @@ func testSearchPropertiesInFiles(t *testing.T, properties []Property, specificRe
waitCollection.Done()
}()

longPropertyCheck.longPropertiesTaskProducer(nil, utils.RunArguments{Context: nil, ServerDetails: serverDetails})
longPropertyCheck.longPropertiesTaskProducer(nil, precheckrunner.RunArguments{Context: nil, ServerDetails: serverDetails})
longPropertyCheck.producerConsumer.Done()
longPropertyCheck.producerConsumer.Run()
close(longPropertyCheck.filesChan)
Expand Down Expand Up @@ -201,7 +202,7 @@ func testLongPropertyCheckWithStubServer(t *testing.T, serverProperties []Proper
defer testServer.Close()

longPropertyCheck := NewLongPropertyCheck(specificRepos)
passed, err := longPropertyCheck.ExecuteCheck(utils.RunArguments{Context: nil, ServerDetails: serverDetails})
passed, err := longPropertyCheck.ExecuteCheck(precheckrunner.RunArguments{Context: nil, ServerDetails: serverDetails})
assert.NoError(t, err)
assert.Equal(t, shouldPass, passed)
}
Expand Down
6 changes: 3 additions & 3 deletions artifactory/commands/transferfiles/transfer.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import (
"time"

"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/state"
commandsUtils "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/utils"
"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/utils/precheckrunner"
"github.com/jfrog/jfrog-cli-core/v2/artifactory/utils"
"github.com/jfrog/jfrog-cli-core/v2/utils/config"
"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
Expand Down Expand Up @@ -292,7 +292,7 @@ func (tdc *TransferFilesCommand) initStorageInfoManagers() error {
}

// Creates the Pre-checks runner for the data transfer command
func (tdc *TransferFilesCommand) NewTransferDataPreChecksRunner() (runner *commandsUtils.PreCheckRunner, err error) {
func (tdc *TransferFilesCommand) NewTransferDataPreChecksRunner() (runner *precheckrunner.PreCheckRunner, err error) {
// Get relevant repos
serviceManager, err := createTransferServiceManager(tdc.context, tdc.sourceServerDetails)
if err != nil {
Expand All @@ -307,7 +307,7 @@ func (tdc *TransferFilesCommand) NewTransferDataPreChecksRunner() (runner *comma
return
}

runner = commandsUtils.NewPreChecksRunner()
runner = precheckrunner.NewPreChecksRunner()

// Add pre checks here
runner.AddCheck(NewLongPropertyCheck(append(localRepos, federatedRepos...)))
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package utils
package precheckrunner

import (
"context"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package utils
package precheckrunner

import (
"context"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
package utils
package precheckrunner

import (
"encoding/json"
"fmt"
"net/http"
"time"

"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/utils"
"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
"github.com/jfrog/jfrog-cli-core/v2/utils/progressbar"
"github.com/jfrog/jfrog-client-go/artifactory"
Expand Down Expand Up @@ -114,7 +115,7 @@ func (rrc *RemoteRepositoryCheck) doCheckRemoteRepositories(args RunArguments, r
}

// Create rtDetails
rtDetails, err := CreateArtifactoryClientDetails(*rrc.targetServicesManager)
rtDetails, err := utils.CreateArtifactoryClientDetails(*rrc.targetServicesManager)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -145,7 +146,7 @@ func (rrc *RemoteRepositoryCheck) startCheckRemoteRepositories(rtDetails *httput
LogMsgPrefix: "[Config import]",
ExecutionHandler: func() (shouldRetry bool, err error) {
// Start the remote repositories check process
resp, responseBody, err := (*rrc.targetServicesManager).Client().SendPost(artifactoryUrl+PluginsExecuteRestApi+"remoteRepositoriesCheck", requestBody, rtDetails)
resp, responseBody, err := (*rrc.targetServicesManager).Client().SendPost(artifactoryUrl+utils.PluginsExecuteRestApi+"remoteRepositoriesCheck", requestBody, rtDetails)
if err != nil {
return false, err
}
Expand Down Expand Up @@ -190,7 +191,7 @@ func (rrc *RemoteRepositoryCheck) waitForRemoteReposCheckCompletion(rtDetails *h
func (rrc *RemoteRepositoryCheck) createImportPollingAction(rtDetails *httputils.HttpClientDetails, artifactoryUrl string, progressBar *progressbar.TasksProgressBar) httputils.PollingAction {
return func() (shouldStop bool, responseBody []byte, err error) {
// Get config import status
resp, body, _, err := (*rrc.targetServicesManager).Client().SendGet(artifactoryUrl+PluginsExecuteRestApi+"remoteRepositoriesCheckStatus", true, rtDetails)
resp, body, _, err := (*rrc.targetServicesManager).Client().SendGet(artifactoryUrl+utils.PluginsExecuteRestApi+"remoteRepositoriesCheckStatus", true, rtDetails)
if err != nil {
return true, nil, err
}
Expand Down Expand Up @@ -227,7 +228,7 @@ func unmarshalRemoteUrlResponse(body []byte) (*remoteUrlResponse, error) {
// Create csv summary of all the files with inaccessible remote repositories and log the result
func handleFailureRun(inaccessibleRepositories []inaccessibleRepository) (err error) {
// Create summary
csvPath, err := CreateCSVFile("inaccessible-repositories", inaccessibleRepositories, time.Now())
csvPath, err := utils.CreateCSVFile("inaccessible-repositories", inaccessibleRepositories, time.Now())
if err != nil {
log.Error("Couldn't create the inaccessible remote repository URLs CSV file", err)
return
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package utils
package precheckrunner

import (
"encoding/json"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
package precheckrunner

import (
"fmt"
"strings"
"time"

commandUtils "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/utils"

"github.com/jfrog/jfrog-cli-core/v2/artifactory/utils"
"github.com/jfrog/jfrog-client-go/artifactory/services"
"github.com/jfrog/jfrog-client-go/utils/log"
)

const (
repositoryNamingCheckName = "Repositories naming"
illegalDockerRepositoryKeyReason = "Docker repository keys in the SasS are not allowed to include '.' or '_' characters."
)

type illegalRepositoryKeys struct {
RepoKey string `json:"repo_key,omitempty"`
Reason string `json:"reason,omitempty"`
}

// Run repository naming check before transferring configuration from one Artifactory to another
type RepositoryNamingCheck struct {
selectedRepos map[utils.RepoType][]services.RepositoryDetails
}

func NewRepositoryNamingCheck(selectedRepos map[utils.RepoType][]services.RepositoryDetails) *RepositoryNamingCheck {
return &RepositoryNamingCheck{selectedRepos}
}

func (drc *RepositoryNamingCheck) Name() string {
return repositoryNamingCheckName
}

func (drc *RepositoryNamingCheck) ExecuteCheck(args RunArguments) (passed bool, err error) {
results := drc.getIllegalRepositoryKeys()
if len(results) == 0 {
return true, nil
}

return false, handleFailuresInRepositoryKeysRun(results)
}

func (drc *RepositoryNamingCheck) getIllegalRepositoryKeys() []illegalRepositoryKeys {
var results []illegalRepositoryKeys
for _, repositoriesOfType := range drc.selectedRepos {
for _, repository := range repositoriesOfType {
if strings.ToLower(repository.PackageType) == "docker" && strings.ContainsAny(repository.Key, "_.") {
log.Debug("Found Docker repository with illegal characters:", repository.Key)
results = append(results, illegalRepositoryKeys{
RepoKey: repository.Key,
Reason: illegalDockerRepositoryKeyReason,
})
}
}
}
return results
}

// Create CSV summary of all the files with illegal repository keys and log the result
func handleFailuresInRepositoryKeysRun(illegalDockerRepositoryKeys []illegalRepositoryKeys) (err error) {
// Create summary
csvPath, err := commandUtils.CreateCSVFile("illegal-repository-keys", illegalDockerRepositoryKeys, time.Now())
if err != nil {
log.Error("Couldn't create the illegal repository keys CSV file", err)
return
}
// Log result
log.Info(fmt.Sprintf("Found %d illegal repository keys. Check the summary CSV file in: %s", len(illegalDockerRepositoryKeys), csvPath))
return
}
Loading

0 comments on commit 30235f3

Please sign in to comment.