Merge pull request #4 from erhudy/fix-partial-downloads

Don't add partially mirrored providers to the index

download.go

@@ -134,3 +134,26 @@ func (d *ProviderDownloader) MirrorProviderInstanceToDest(pi ProviderSpecificIns
 
 	return psib, nil
 }
+
+// removes versions from psibs where we don't have every OS+arch combination downloaded
+// this is only to remove the version from index.json, we still keep the existing providers around so that we don't need to redownload everything later
+func FilterVersionsWithFailedPSIBs(psibs []ProviderSpecificInstanceBinary, failedPvis []ProviderSpecificInstance) []ProviderSpecificInstanceBinary {
+	filteredPsibs := []ProviderSpecificInstanceBinary{}
+
+	for _, p := range psibs {
+		foundInFailed := false
+		for _, fp := range failedPvis {
+			if p.Name == fp.Name && p.Owner == fp.Owner && p.Version == fp.Version {
+				foundInFailed = true
+				break
+			}
+		}
+		if foundInFailed {
+			sugar.Warnf("filtering %s out due to only having some of the providers for version %s", p.Provider.String(), p.Version)
+		} else {
+			filteredPsibs = append(filteredPsibs, p)
+		}
+	}
+
+	return filteredPsibs
+}

main.go

@@ -212,16 +212,23 @@ func MirrorProvidersWithConfig(config Configuration, logger *zap.Logger) error {
 		var psibs []ProviderSpecificInstanceBinary
 		psibs = append(psibs, valid...)
 
+		// we need to record failed downloads as well so that we can exclude that entire version from the catalog,
+		// in instances where some particular OS+arch combo of a provider fails to download for some reason
+		failedPvis := []ProviderSpecificInstance{}
+
 		for _, pvi := range pvisToDownload {
 			psib, err := d.MirrorProviderInstanceToDest(pvi)
 			if err != nil {
 				sugar.Errorf("error mirroring provider instance %s: %w", pvi, err)
+				failedPvis = append(failedPvis, pvi)
 				continue
 			}
 			psibs = append(psibs, *psib)
 		}
 
-		err = d.Storage.StoreCatalog(psibs)
+		finalPsibs := FilterVersionsWithFailedPSIBs(psibs, failedPvis)
+
+		err = d.Storage.StoreCatalog(finalPsibs)
 		if err != nil {
 			sugar.Errorf("error writing catalog for provider %s: %w", provider, err)
 			continue