diff --git a/download.go b/download.go index 7689413..e111097 100644 --- a/download.go +++ b/download.go @@ -134,3 +134,26 @@ func (d *ProviderDownloader) MirrorProviderInstanceToDest(pi ProviderSpecificIns return psib, nil } + +// removes versions from psibs where we don't have every OS+arch combination downloaded +// this is only to remove the version from index.json, we still keep the existing providers around so that we don't need to redownload everything later +func FilterVersionsWithFailedPSIBs(psibs []ProviderSpecificInstanceBinary, failedPvis []ProviderSpecificInstance) []ProviderSpecificInstanceBinary { + filteredPsibs := []ProviderSpecificInstanceBinary{} + + for _, p := range psibs { + foundInFailed := false + for _, fp := range failedPvis { + if p.Name == fp.Name && p.Owner == fp.Owner && p.Version == fp.Version { + foundInFailed = true + break + } + } + if foundInFailed { + sugar.Warnf("filtering %s out due to only having some of the providers for version %s", p.Provider.String(), p.Version) + } else { + filteredPsibs = append(filteredPsibs, p) + } + } + + return filteredPsibs +} diff --git a/main.go b/main.go index cbc1c4e..d8461e4 100644 --- a/main.go +++ b/main.go @@ -212,16 +212,23 @@ func MirrorProvidersWithConfig(config Configuration, logger *zap.Logger) error { var psibs []ProviderSpecificInstanceBinary psibs = append(psibs, valid...) + // we need to record failed downloads as well so that we can exclude that entire version from the catalog, + // in instances where some particular OS+arch combo of a provider fails to download for some reason + failedPvis := []ProviderSpecificInstance{} + for _, pvi := range pvisToDownload { psib, err := d.MirrorProviderInstanceToDest(pvi) if err != nil { sugar.Errorf("error mirroring provider instance %s: %w", pvi, err) + failedPvis = append(failedPvis, pvi) continue } psibs = append(psibs, *psib) } - err = d.Storage.StoreCatalog(psibs) + finalPsibs := FilterVersionsWithFailedPSIBs(psibs, failedPvis) + + err = d.Storage.StoreCatalog(finalPsibs) if err != nil { sugar.Errorf("error writing catalog for provider %s: %w", provider, err) continue