Persist unused DDEX fields in discovery (#7862)

AudiusProject · Mar 18, 2024 · f6bc750 · f6bc750
1 parent e892314
commit f6bc750
Show file tree

Hide file tree

Showing 28 changed files with 715 additions and 133 deletions.
diff --git a/packages/common/src/models/Collection.ts b/packages/common/src/models/Collection.ts
@@ -4,7 +4,7 @@ import { Repost } from '../models/Repost'
 import { Nullable } from '../utils/typeUtils'
 
 import { Favorite } from './Favorite'
-import { UserTrackMetadata } from './Track'
+import { UserTrackMetadata, ResourceContributor, Copyright } from './Track'
 import { User, UserMetadata } from './User'
 
 export enum Variant {
@@ -60,6 +60,10 @@ export type CollectionMetadata = {
   release_date?: string
   ddex_app?: string | null
   ddex_release_ids?: any | null
+  artists?: [ResourceContributor] | null
+  copyright_line?: Copyright | null
+  producer_copyright_line?: Copyright | null
+  parental_warning_type?: string | null
 }
 
 export type CollectionDownloadReason = { is_from_favorites: boolean }

diff --git a/packages/common/src/models/Track.ts b/packages/common/src/models/Track.ts
@@ -164,6 +164,23 @@ export type SolCollectionMap = {
 
 export type GatedTrackStatus = null | 'UNLOCKING' | 'UNLOCKED' | 'LOCKED'
 
+export type ResourceContributor = {
+  name: string
+  roles: [string]
+  sequence_number: number
+}
+
+export type RightsController = {
+  name: string
+  roles: [string]
+  rights_share_unknown?: string
+}
+
+export type Copyright = {
+  year: string
+  text: string
+}
+
 export type TrackMetadata = {
   ai_attribution_user_id?: Nullable<number>
   blocknumber: number
@@ -210,6 +227,14 @@ export type TrackMetadata = {
   is_downloadable: boolean
   is_original_available: boolean
   ddex_app?: Nullable<string>
+  ddex_release_ids?: any | null
+  artists?: [ResourceContributor] | null
+  resource_contributors?: [ResourceContributor] | null
+  indirect_resource_contributors?: [ResourceContributor] | null
+  rights_controller?: RightsController | null
+  copyright_line?: Copyright | null
+  producer_copyright_line?: Copyright | null
+  parental_warning_type?: string | null
 
   // Optional Fields
   is_playlist_upload?: boolean

diff --git a/packages/common/src/schemas/metadata.ts b/packages/common/src/schemas/metadata.ts
@@ -55,7 +55,14 @@ const trackMetadataSchema = {
   is_playlist_upload: false,
   ai_attribution_user_id: null,
   ddex_release_ids: null,
-  ddex_app: null
+  ddex_app: null,
+  artists: null,
+  resource_contributors: null,
+  indirect_resource_contributors: null,
+  rights_controller: null,
+  copyright_line: null,
+  producer_copyright_line: null,
+  parental_warning_type: null
 }
 
 export const newTrackMetadata = (fields, validate = false): TrackMetadata => {
@@ -95,7 +102,11 @@ const collectionMetadataSchema = {
   upc: null,
   description: null,
   ddex_release_ids: null,
-  ddex_app: null
+  ddex_app: null,
+  artists: null,
+  copyright_line: null,
+  producer_copyright_line: null,
+  parental_warning_type: null
 }
 
 export const newCollectionMetadata = (fields?: any, validate = false) => {

diff --git a/packages/ddex/ingester/artistutils/artistutils.go b/packages/ddex/ingester/artistutils/artistutils.go
@@ -28,7 +28,7 @@ func CreateArtistNameIndex(usersColl *mongo.Collection, ctx context.Context) err
 
 // GetFirstArtistID searches Mongo OAuthed artists for the first match on the track's display artists' names
 // and returns that artist's ID
-func GetFirstArtistID(artists []common.Artist, usersColl *mongo.Collection, ctx context.Context) (string, string, []string, error) {
+func GetFirstArtistID(artists []common.ResourceContributor, usersColl *mongo.Collection, ctx context.Context) (string, string, []string, error) {
 	// Sort artists by SequenceNumber (asc)
 	sort.Slice(artists, func(i, j int) bool {
 		return artists[i].SequenceNumber < artists[j].SequenceNumber

diff --git a/packages/ddex/ingester/common/sdk_types.go b/packages/ddex/ingester/common/sdk_types.go
@@ -174,13 +174,20 @@ type ReleaseIDs struct {
 }
 
 type TrackMetadata struct {
-	Title               string         `bson:"title"`
-	ReleaseDate         time.Time      `bson:"release_date"`
-	Genre               Genre          `bson:"genre"`
-	Duration            int            `bson:"duration"`
-	PreviewStartSeconds NullableInt    `bson:"preview_start_seconds,omitempty"`
-	ISRC                NullableString `bson:"isrc,omitempty"`
-	DDEXReleaseIDs      ReleaseIDs     `bson:"ddex_release_ids"`
+	Title                        string                `bson:"title"`
+	ReleaseDate                  time.Time             `bson:"release_date"`
+	Genre                        Genre                 `bson:"genre"`
+	Duration                     int                   `bson:"duration"`
+	PreviewStartSeconds          NullableInt           `bson:"preview_start_seconds,omitempty"`
+	ISRC                         NullableString        `bson:"isrc,omitempty"`
+	DDEXReleaseIDs               ReleaseIDs            `bson:"ddex_release_ids"`
+	Artists                      []ResourceContributor `bson:"artists"`
+	ResourceContributors         []ResourceContributor `bson:"resource_contributors,omitempty"`
+	IndirectResourceContributors []ResourceContributor `bson:"indirect_resource_contributors,omitempty"`
+	RightsController             RightsController      `bson:"rights_controller,omitempty"`
+	CopyrightLine                Copyright             `bson:"copyright_line,omitempty"`
+	ProducerCopyrightLine        Copyright             `bson:"producer_copyright_line,omitempty"`
+	ParentalWarningType          string                `bson:"parental_warning_type,omitempty"`
 
 	// TODO: Handle License from PLineText?
 	License NullableString `bson:"license,omitempty"`
@@ -191,23 +198,8 @@ type TrackMetadata struct {
 	Tags        NullableString `bson:"tags,omitempty"`
 
 	// Extra fields (not in SDK)
-
-	// michelle add to sdk ==
-	Artists []Artist `bson:"artists"`
-	// michelle add to sdk ==
-
-	ArtistID   string `bson:"artist_id"`
-	ArtistName string `bson:"artist_name"`
-
-	// michelle add to sdk ==
-	ResourceContributors         []ResourceContributor `bson:"resource_contributor,omitempty"`
-	IndirectResourceContributors []ResourceContributor `bson:"indirect_resource_contributor,omitempty"`
-	RightsController             RightsController      `bson:"rights_controller,omitempty"`
-	CopyrightLine                Copyright             `bson:"copyright_line,omitempty"`
-	ProducerCopyrightLine        Copyright             `bson:"producer_copyright_line,omitempty"`
-	ParentalWarningType          string                `bson:"parental_warning_type,omitempty"`
-	// michelle add to sdk ==
-
+	ArtistID                    string `bson:"artist_id"`
+	ArtistName                  string `bson:"artist_name"`
 	PreviewAudioFileURL         string `bson:"preview_audio_file_url"`
 	PreviewAudioFileURLHash     string `bson:"preview_audio_file_url_hash"`
 	PreviewAudioFileURLHashAlgo string `bson:"preview_audio_file_url_hash_algo"`
@@ -220,56 +212,47 @@ type TrackMetadata struct {
 }
 
 // Not part of SDK
-type Artist struct {
-	Name           string   `bson:"name"`
-	Roles          []string `bson:"roles"`
-	SequenceNumber int      `bson:"sequence_number"`
-}
 
 type Copyright struct {
-	Year string
-	Text string
+	Year string `bson:"year"`
+	Text string `bson:"text"`
 }
 
 // ResourceContributor represents a contributor to the sound recording
 type ResourceContributor struct {
-	Name           string
-	Roles          []string
-	SequenceNumber int `bson:"sequence_number"`
+	Name           string   `bson:"name"`
+	Roles          []string `bson:"roles"`
+	SequenceNumber int      `bson:"sequence_number"`
 }
 
 type RightsController struct {
-	Name               string
-	Roles              []string
-	RightsShareUnknown string
+	Name               string   `bson:"name"`
+	Roles              []string `bson:"roles"`
+	RightsShareUnknown string   `bson:"rights_share_unknown,omitempty"`
 }
 
 type CollectionMetadata struct {
-	PlaylistName      string         `bson:"playlist_name"`
-	PlaylistOwnerID   string         `bson:"playlist_owner_id"`
-	PlaylistOwnerName string         `bson:"playlist_owner_name"`
-	Description       NullableString `bson:"description,omitempty"`
-	IsAlbum           bool           `bson:"is_album"`
-	IsPrivate         bool           `bson:"is_private"`
-	Tags              NullableString `bson:"tags,omitempty"`
-	Genre             Genre          `bson:"genre"`
-	Mood              Mood           `bson:"mood,omitempty"`
-	ReleaseDate       time.Time      `bson:"release_date"`
-	DDEXReleaseIDs    ReleaseIDs     `bson:"ddex_release_ids"`
+	PlaylistName          string                `bson:"playlist_name"`
+	PlaylistOwnerID       string                `bson:"playlist_owner_id"`
+	PlaylistOwnerName     string                `bson:"playlist_owner_name"`
+	Description           NullableString        `bson:"description,omitempty"`
+	IsAlbum               bool                  `bson:"is_album"`
+	IsPrivate             bool                  `bson:"is_private"`
+	Tags                  NullableString        `bson:"tags,omitempty"`
+	Genre                 Genre                 `bson:"genre"`
+	Mood                  Mood                  `bson:"mood,omitempty"`
+	ReleaseDate           time.Time             `bson:"release_date"`
+	DDEXReleaseIDs        ReleaseIDs            `bson:"ddex_release_ids"`
+	Artists               []ResourceContributor `bson:"artists"`
+	CopyrightLine         Copyright             `bson:"copyright_line,omitempty"`
+	ProducerCopyrightLine Copyright             `bson:"producer_copyright_line,omitempty"`
+	ParentalWarningType   string                `bson:"parental_warning_type,omitempty"`
 
 	// TODO: Handle these fields
 	License NullableString `bson:"license,omitempty"`
 	UPC     NullableString `bson:"upc,omitempty"`
 
 	// Extra fields (not in SDK)
-
-	// michelle add to sdk ==
-	Artists               []Artist  `bson:"artists"`
-	CopyrightLine         Copyright `bson:"copyright_line,omitempty"`
-	ProducerCopyrightLine Copyright `bson:"producer_copyright_line,omitempty"`
-	ParentalWarningType   string    `bson:"parental_warning_type,omitempty"`
-	// michelle add to sdk ==
-
 	CoverArtURL         string `bson:"cover_art_url"`
 	CoverArtURLHash     string `bson:"cover_art_url_hash"`
 	CoverArtURLHashAlgo string `bson:"cover_art_url_hash_algo"`

diff --git a/packages/ddex/ingester/e2e_test/e2e_test.go b/packages/ddex/ingester/e2e_test/e2e_test.go
@@ -145,7 +145,7 @@ func (e *e2eTest) runERN382Batched(t *testing.T) {
 				ICPN: "721620118165",
 			},
 			CoverArtURL: "s3://audius-test-crawled/721620118165/resources/721620118165_T7_007.jpg",
-			Artists: []common.Artist{{
+			Artists: []common.ResourceContributor{{
 				Name:           "Monkey Claw",
 				Roles:          []string{"MainArtist"},
 				SequenceNumber: 1,
@@ -172,7 +172,7 @@ func (e *e2eTest) runERN382Batched(t *testing.T) {
 				Duration:            811,
 				PreviewStartSeconds: intPtr(0),
 				ISRC:                stringPtr("CASE00000001"),
-				Artists: []common.Artist{
+				Artists: []common.ResourceContributor{
 					{
 						Name:  "Monkey Claw",
 						Roles: []string{"MainArtist"},
@@ -209,7 +209,7 @@ func (e *e2eTest) runERN382Batched(t *testing.T) {
 				Duration:            366,
 				PreviewStartSeconds: intPtr(0),
 				ISRC:                stringPtr("CASE00000002"),
-				Artists: []common.Artist{
+				Artists: []common.ResourceContributor{
 					{
 						Name:  "Monkey Claw",
 						Roles: []string{"MainArtist"},
@@ -246,7 +246,7 @@ func (e *e2eTest) runERN382Batched(t *testing.T) {
 				Duration:            1269,
 				PreviewStartSeconds: intPtr(0),
 				ISRC:                stringPtr("CASE00000003"),
-				Artists: []common.Artist{
+				Artists: []common.ResourceContributor{
 					{
 						Name:  "Monkey Claw",
 						Roles: []string{"MainArtist"},
@@ -283,7 +283,7 @@ func (e *e2eTest) runERN382Batched(t *testing.T) {
 				Duration:            165,
 				PreviewStartSeconds: intPtr(0),
 				ISRC:                stringPtr("CASE00000004"),
-				Artists: []common.Artist{
+				Artists: []common.ResourceContributor{
 					{
 						Name:  "Monkey Claw",
 						Roles: []string{"MainArtist"},
@@ -320,7 +320,7 @@ func (e *e2eTest) runERN382Batched(t *testing.T) {
 				Duration:            294,
 				PreviewStartSeconds: intPtr(0),
 				ISRC:                stringPtr("CASE00000005"),
-				Artists: []common.Artist{
+				Artists: []common.ResourceContributor{
 					{
 						Name:  "Monkey Claw",
 						Roles: []string{"MainArtist"},
@@ -357,7 +357,7 @@ func (e *e2eTest) runERN382Batched(t *testing.T) {
 				Duration:            741,
 				PreviewStartSeconds: intPtr(0),
 				ISRC:                stringPtr("CASE00000006"),
-				Artists: []common.Artist{
+				Artists: []common.ResourceContributor{
 					{
 						Name:  "Monkey Claw",
 						Roles: []string{"MainArtist"},
@@ -450,7 +450,7 @@ func (e *e2eTest) runERN381ReleaseByRelease(t *testing.T) {
 			CoverArtURL:         fmt.Sprintf("s3://audius-test-crawled/%s/resources/A10301A0005108088N_T-1027024165547_Image.jpg", pendingRelease.ReleaseID),
 			CoverArtURLHash:     "582fb410615167205e8741580cf77e71",
 			CoverArtURLHashAlgo: "MD5",
-			Artists: []common.Artist{{
+			Artists: []common.ResourceContributor{{
 				Name:           "Theo Random",
 				Roles:          []string{"MainArtist"},
 				SequenceNumber: 1,
@@ -467,7 +467,7 @@ func (e *e2eTest) runERN381ReleaseByRelease(t *testing.T) {
 				ReleaseDate: time.Time{},
 				Genre:       common.HipHopRap,
 				Duration:    279,
-				Artists: []common.Artist{{
+				Artists: []common.ResourceContributor{{
 					Name:           "Theo Random",
 					Roles:          []string{"AssociatedPerformer", "MainArtist"},
 					SequenceNumber: 1,
@@ -529,7 +529,7 @@ func (e *e2eTest) runERN381ReleaseByRelease(t *testing.T) {
 				Duration:    142,
 				ArtistID:    "",
 				ArtistName:  "",
-				Artists: []common.Artist{
+				Artists: []common.ResourceContributor{
 					{
 						Name:           "Theo Random",
 						Roles:          []string{"AssociatedPerformer", "MainArtist"},
@@ -543,7 +543,7 @@ func (e *e2eTest) runERN381ReleaseByRelease(t *testing.T) {
 				},
 				ResourceContributors: []common.ResourceContributor{
 					{
-						Name:           "Theo Random &amp; Thato Saul",
+						Name:           "Theo Random & Thato Saul",
 						Roles:          []string{"AssociatedPerformer"},
 						SequenceNumber: 1,
 					},