Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Parse and store all DDEX standard identitiers #7818

Merged
merged 8 commits into from
Mar 14, 2024
Merged

Parse and store all DDEX standard identitiers #7818

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
da1ba70
add additional_ids jsonb col to tracks
michellebrier Mar 12, 2024
c7b72a6
add additionalIds to sdk types
michellebrier Mar 12, 2024
589d23c
add to ern parser. todo e2e test, rename to ddex_release_ids, and add…
michellebrier Mar 13, 2024
10b6720
rename + add to playlists table. todo e2e test
michellebrier Mar 13, 2024
4078dc4
e2e test
michellebrier Mar 13, 2024
0622629
fix release IDs on tracks on albums
michellebrier Mar 13, 2024
32750a6
fix builds
michellebrier Mar 14, 2024
a939cf1
missed a metadata
michellebrier Mar 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/common/src/models/Collection.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ export type CollectionMetadata = {
local?: boolean
release_date?: string
ddex_app?: string | null
ddex_release_ids?: any | null
}

export type CollectionDownloadReason = { is_from_favorites: boolean }
Expand Down
8 changes: 6 additions & 2 deletions packages/common/src/schemas/metadata.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,9 @@ const trackMetadataSchema = {
iswc: null,
download: null,
is_playlist_upload: false,
ai_attribution_user_id: null
ai_attribution_user_id: null,
ddex_release_ids: null,
ddex_app: null
}

export const newTrackMetadata = (fields, validate = false): TrackMetadata => {
Expand Down Expand Up @@ -91,7 +93,9 @@ const collectionMetadataSchema = {
save_count: null,
license: null,
upc: null,
description: null
description: null,
ddex_release_ids: null,
ddex_app: null
}

export const newCollectionMetadata = (fields?: any, validate = false) => {
Expand Down
19 changes: 19 additions & 0 deletions packages/ddex/ingester/common/sdk_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,13 +156,31 @@ type NullableBool = *bool
type NullableString = *string
type NullableInt = *int

type ReleaseIDs struct {
PartyID string `bson:"party_id,omitempty"`
CatalogNumber string `bson:"catalog_number,omitempty"`
ICPN string `bson:"icpn,omitempty"`
GRid string `bson:"grid,omitempty"`
ISAN string `bson:"isan,omitempty"`
ISBN string `bson:"isbn,omitempty"`
ISMN string `bson:"ismn,omitempty"`
ISRC string `bson:"isrc,omitempty"`
ISSN string `bson:"issn,omitempty"`
ISTC string `bson:"istc,omitempty"`
ISWC string `bson:"iswc,omitempty"`
MWLI string `bson:"mwli,omitempty"`
SICI string `bson:"sici,omitempty"`
ProprietaryID string `bson:"proprietary_id,omitempty"`
}

type TrackMetadata struct {
Title string `bson:"title"`
ReleaseDate time.Time `bson:"release_date"`
Genre Genre `bson:"genre"`
Duration int `bson:"duration"`
PreviewStartSeconds NullableInt `bson:"preview_start_seconds,omitempty"`
ISRC NullableString `bson:"isrc,omitempty"`
DDEXReleaseIDs ReleaseIDs `bson:"ddex_release_ids"`

// TODO: Handle License from PLineText?
License NullableString `bson:"license,omitempty"`
Expand Down Expand Up @@ -205,6 +223,7 @@ type CollectionMetadata struct {
Genre Genre `bson:"genre"`
Mood Mood `bson:"mood,omitempty"`
ReleaseDate time.Time `bson:"release_date"`
DDEXReleaseIDs ReleaseIDs `bson:"ddex_release_ids"`

// TODO: Handle these fields
License NullableString `bson:"license,omitempty"`
Expand Down
100 changes: 67 additions & 33 deletions packages/ddex/ingester/e2e_test/e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,13 +141,19 @@ func (e *e2eTest) runERN382Batched(t *testing.T) {
IsPrivate: false,
Genre: "Metal",
ReleaseDate: publishDate,
CoverArtURL: "s3://audius-test-crawled/721620118165/resources/721620118165_T7_007.jpg",
DDEXReleaseIDs: common.ReleaseIDs{
ICPN: "721620118165",
},
CoverArtURL: "s3://audius-test-crawled/721620118165/resources/721620118165_T7_007.jpg",
},
DDEXReleaseRef: "R0",
Tracks: []common.TrackMetadata{
{
Title: "Can you feel ...the Monkey Claw!",
ReleaseDate: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC),
Title: "Can you feel ...the Monkey Claw!",
ReleaseDate: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC),
DDEXReleaseIDs: common.ReleaseIDs{
ISRC: "CASE00000001",
},
Genre: "Metal",
Duration: 811,
PreviewStartSeconds: intPtr(0),
Expand All @@ -161,8 +167,11 @@ func (e *e2eTest) runERN382Batched(t *testing.T) {
AudioFileURL: "s3://audius-test-crawled/721620118165/resources/721620118165_T1_001.wav",
},
{
Title: "Red top mountain, blown sky high",
ReleaseDate: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC),
Title: "Red top mountain, blown sky high",
ReleaseDate: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC),
DDEXReleaseIDs: common.ReleaseIDs{
ISRC: "CASE00000002",
},
Genre: "Metal",
Duration: 366,
PreviewStartSeconds: intPtr(0),
Expand All @@ -176,8 +185,11 @@ func (e *e2eTest) runERN382Batched(t *testing.T) {
AudioFileURL: "s3://audius-test-crawled/721620118165/resources/721620118165_T2_002.wav",
},
{
Title: "Seige of Antioch",
ReleaseDate: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC),
Title: "Seige of Antioch",
ReleaseDate: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC),
DDEXReleaseIDs: common.ReleaseIDs{
ISRC: "CASE00000003",
},
Genre: "Metal",
Duration: 1269,
PreviewStartSeconds: intPtr(0),
Expand All @@ -191,8 +203,11 @@ func (e *e2eTest) runERN382Batched(t *testing.T) {
AudioFileURL: "s3://audius-test-crawled/721620118165/resources/721620118165_T3_003.wav",
},
{
Title: "Warhammer",
ReleaseDate: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC),
Title: "Warhammer",
ReleaseDate: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC),
DDEXReleaseIDs: common.ReleaseIDs{
ISRC: "CASE00000004",
},
Genre: "Metal",
Duration: 165,
PreviewStartSeconds: intPtr(0),
Expand All @@ -206,8 +221,11 @@ func (e *e2eTest) runERN382Batched(t *testing.T) {
AudioFileURL: "s3://audius-test-crawled/721620118165/resources/721620118165_T4_004.wav",
},
{
Title: "Iron Horse",
ReleaseDate: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC),
Title: "Iron Horse",
ReleaseDate: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC),
DDEXReleaseIDs: common.ReleaseIDs{
ISRC: "CASE00000005",
},
Genre: "Metal",
Duration: 294,
PreviewStartSeconds: intPtr(0),
Expand All @@ -221,8 +239,11 @@ func (e *e2eTest) runERN382Batched(t *testing.T) {
AudioFileURL: "s3://audius-test-crawled/721620118165/resources/721620118165_T5_005.wav",
},
{
Title: "Yes... I can feel the Monkey Claw!",
ReleaseDate: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC),
Title: "Yes... I can feel the Monkey Claw!",
ReleaseDate: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC),
DDEXReleaseIDs: common.ReleaseIDs{
ISRC: "CASE00000006",
},
Genre: "Metal",
Duration: 741,
PreviewStartSeconds: intPtr(0),
Expand Down Expand Up @@ -286,10 +307,15 @@ func (e *e2eTest) runERN381ReleaseByRelease(t *testing.T) {
assert.Equal(t, common.CreateAlbumRelease{
DDEXReleaseRef: "R0",
Metadata: common.CollectionMetadata{
PlaylistName: "Present.",
PlaylistOwnerID: "Bmv3bJ",
PlaylistOwnerName: "Theo Random",
ReleaseDate: publishDate,
PlaylistName: "Present.",
PlaylistOwnerID: "Bmv3bJ",
PlaylistOwnerName: "Theo Random",
ReleaseDate: publishDate,
DDEXReleaseIDs: common.ReleaseIDs{
CatalogNumber: "G010005108088N",
GRid: "A10301A0005108088N",
ICPN: "196871335584",
},
IsAlbum: true,
IsPrivate: false,
Genre: common.HipHopRap,
Expand All @@ -299,29 +325,37 @@ func (e *e2eTest) runERN381ReleaseByRelease(t *testing.T) {
},
Tracks: []common.TrackMetadata{
{
Title: "Playing With Fire.",
ReleaseDate: time.Time{},
Genre: common.HipHopRap,
Duration: 279,
Artists: []common.Artist{{Name: "Theo Random", Roles: []string{"AssociatedPerformer", "MainArtist"}}},
ArtistID: "",
ArtistName: "",
ISRC: stringPtr("ZAA012300131"),
Title: "Playing With Fire.",
ReleaseDate: time.Time{},
Genre: common.HipHopRap,
Duration: 279,
Artists: []common.Artist{{Name: "Theo Random", Roles: []string{"AssociatedPerformer", "MainArtist"}}},
ArtistID: "",
ArtistName: "",
ISRC: stringPtr("ZAA012300131"),
DDEXReleaseIDs: common.ReleaseIDs{
GRid: "A10328E0010879163O",
ISRC: "ZAA012300131",
},
PreviewStartSeconds: intPtr(48),
PreviewAudioFileURL: fmt.Sprintf("s3://audius-test-crawled/%s/", pendingRelease.ReleaseID),
AudioFileURL: fmt.Sprintf("s3://audius-test-crawled/%s/resources/A10301A0005108088N_T-1096524256352_SoundRecording_001-001.m4a", pendingRelease.ReleaseID),
AudioFileURLHash: "8bb2ce119257314a8fcb215a49f14b33",
AudioFileURLHashAlgo: "MD5",
},
{
Title: "No Comment.",
ReleaseDate: time.Time{},
Genre: common.HipHopRap,
Duration: 142,
ArtistID: "",
ArtistName: "",
Artists: []common.Artist{{Name: "Theo Random", Roles: []string{"AssociatedPerformer", "MainArtist"}}, {Name: "Thato Saul", Roles: []string{"AssociatedPerformer", "MainArtist"}}},
ISRC: stringPtr("ZAA012300128"),
Title: "No Comment.",
ReleaseDate: time.Time{},
Genre: common.HipHopRap,
Duration: 142,
ArtistID: "",
ArtistName: "",
Artists: []common.Artist{{Name: "Theo Random", Roles: []string{"AssociatedPerformer", "MainArtist"}}, {Name: "Thato Saul", Roles: []string{"AssociatedPerformer", "MainArtist"}}},
ISRC: stringPtr("ZAA012300128"),
DDEXReleaseIDs: common.ReleaseIDs{
GRid: "A10328E0010879164M",
ISRC: "ZAA012300128",
},
PreviewStartSeconds: intPtr(48),
PreviewAudioFileURL: fmt.Sprintf("s3://audius-test-crawled/%s/", pendingRelease.ReleaseID),
AudioFileURL: fmt.Sprintf("s3://audius-test-crawled/%s/resources/A10301A0005108088N_T-1096524142976_SoundRecording_001-002.m4a", pendingRelease.ReleaseID),
Expand Down
27 changes: 26 additions & 1 deletion packages/ddex/ingester/parser/ern38x.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,24 @@ func processReleaseNode(rNode *xmlquery.Node, soundRecordings *[]SoundRecording,
isrc := safeInnerText(rNode.SelectElement("ReleaseId/ISRC"))
releaseType := safeInnerText(rNode.SelectElement("ReleaseType"))

// Release IDs
ddexReleaseIDs := &common.ReleaseIDs{
PartyID: safeInnerText(rNode.SelectElement("ReleaseId/PartyId")),
CatalogNumber: safeInnerText(rNode.SelectElement("ReleaseId/CatalogNumber")),
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i'm wondering if there would ever be a case where a track transfers rights and we want to know the DPID attribute of the CatalogNumber / ProprietaryId. i'm thinking probably not but just writing this down in case it comes up later. we should be able to use the ddex app's key as a way to identify which distributor the track comes from anyway

ICPN: safeInnerText(rNode.SelectElement("ReleaseId/ICPN")),
GRid: safeInnerText(rNode.SelectElement("ReleaseId/GRid")),
ISAN: safeInnerText(rNode.SelectElement("ReleaseId/ISAN")),
ISBN: safeInnerText(rNode.SelectElement("ReleaseId/ISBN")),
ISMN: safeInnerText(rNode.SelectElement("ReleaseId/ISMN")),
ISRC: isrc,
ISSN: safeInnerText(rNode.SelectElement("ReleaseId/ISSN")),
ISTC: safeInnerText(rNode.SelectElement("ReleaseId/ISTC")),
ISWC: safeInnerText(rNode.SelectElement("ReleaseId/ISWC")),
MWLI: safeInnerText(rNode.SelectElement("ReleaseId/MWLI")),
SICI: safeInnerText(rNode.SelectElement("ReleaseId/SICI")),
ProprietaryID: safeInnerText(rNode.SelectElement("ReleaseId/ProprietaryId")),
}

// Convert releaseDate from string of format YYYY-MM-DD to time.Time
if releaseDateStr == "" {
err = fmt.Errorf("missing release date for <ReleaseReference>%s</ReleaseReference>", releaseRef)
Expand Down Expand Up @@ -286,6 +304,7 @@ func processReleaseNode(rNode *xmlquery.Node, soundRecordings *[]SoundRecording,
PlaylistName: title,
PlaylistOwnerName: artistName,
ReleaseDate: releaseDate,
DDEXReleaseIDs: *ddexReleaseIDs,
Genre: genre,
IsAlbum: true,
IsPrivate: false, // TODO: Use DealList to determine this. Same with releaseDate because I think the XML element it's reading is deprecated
Expand Down Expand Up @@ -340,12 +359,17 @@ func processReleaseNode(rNode *xmlquery.Node, soundRecordings *[]SoundRecording,
trackMetadata.Title = title
}

if *trackMetadata.ISRC == "" {
if trackMetadata.ISRC == nil || *trackMetadata.ISRC == "" {
if isrc == "" {
err = fmt.Errorf("missing isrc for <ReleaseReference>%s</ReleaseReference>", releaseRef)
return
}
*trackMetadata.ISRC = isrc
} else {
if *trackMetadata.ISRC != isrc {
// Use the ISRC from the SoundRecording if it differs from the Release ISRC
(*ddexReleaseIDs).ISRC = *trackMetadata.ISRC
}
}

if trackMetadata.Genre == "" {
Expand All @@ -368,6 +392,7 @@ func processReleaseNode(rNode *xmlquery.Node, soundRecordings *[]SoundRecording,

trackMetadata.ArtistName = artistName
trackMetadata.ReleaseDate = releaseDate
trackMetadata.DDEXReleaseIDs = *ddexReleaseIDs
trackMetadata.Copyright = copyright
trackMetadata.CoverArtURL = coverArtURL
trackMetadata.CoverArtURLHash = coverArtURLHash
Expand Down
22 changes: 20 additions & 2 deletions packages/ddex/ingester/parser/parser.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,23 @@ func (p *Parser) parseRelease(release *common.UnprocessedRelease, deliveryZipFil

// If there's an album release, the tracks we parsed out are actually part of the album release
if len(createAlbumRelease) > 0 {
// Copy release IDs from individual track releases to the album's tracks
isrcToReleaseIDsMap := make(map[string]common.ReleaseIDs)
for _, trackRelease := range createTrackRelease {
if trackRelease.Metadata.ISRC != nil {
isrcToReleaseIDsMap[*trackRelease.Metadata.ISRC] = trackRelease.Metadata.DDEXReleaseIDs
}
}
for i, album := range createAlbumRelease {
for j, trackMetadata := range album.Tracks {
if trackMetadata.ISRC != nil {
if releaseIDs, exists := isrcToReleaseIDsMap[*trackMetadata.ISRC]; exists {
createAlbumRelease[i].Tracks[j].DDEXReleaseIDs = releaseIDs
}
}
}
}
// Clear the individual track releases
createTrackRelease = []common.CreateTrackRelease{}
}

Expand Down Expand Up @@ -302,10 +319,11 @@ func (p *Parser) parseBatch(batch *common.UnprocessedBatch, deliveryZipFileETag
// TODO: Support more ID types (GRid is preferred) as we get more examples
var releaseID string
releaseICPN := safeInnerText(messageInBatch.SelectElement("IncludedReleaseId/ICPN"))
releaseGRid := safeInnerText(messageInBatch.SelectElement("IncludedReleaseId/GRid"))
if releaseICPN != "" {
releaseID = releaseICPN
// } else if releaseGRid != "" {
// releaseID = releaseGRid
} else if releaseGRid != "" {
releaseID = releaseGRid
} else {
err := fmt.Errorf("no valid IncludedReleaseId found")
batch.ValidationErrors = append(batch.ValidationErrors, err.Error())
Expand Down
2 changes: 2 additions & 0 deletions packages/ddex/publisher/src/models/pendingReleases.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ const artistSchema = new mongoose.Schema({
const trackMetadataSchema = new mongoose.Schema({
title: { type: String, required: true },
release_date: { type: Date, required: true },
ddex_release_ids: mongoose.Schema.Types.Mixed,
genre: { type: String, enum: genres, required: true },
duration: { type: Number, required: true },
preview_start_seconds: Number,
Expand Down Expand Up @@ -120,6 +121,7 @@ const collectionMetadataSchema = new mongoose.Schema({
playlist_owner_id: { type: String, required: true },
genre: { type: String, enum: genres, required: true },
release_date: { type: Date, required: true },
ddex_release_ids: mongoose.Schema.Types.Mixed,
description: String,
is_album: Boolean,
is_private: Boolean,
Expand Down
2 changes: 2 additions & 0 deletions packages/ddex/publisher/src/services/publisherService.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ const formatTrackMetadata = (
isrc: metadata.isrc,
license: metadata.license,
releaseDate: new Date(metadata.release_date),
ddexReleaseIds: metadata.ddex_release_ids,
previewStartSeconds: metadata.preview_start_seconds ?? undefined,
// isUnlisted: // TODO: set visibility
// iswc:
Expand All @@ -56,6 +57,7 @@ const formatAlbumMetadata = (
license: metadata.license || '',
mood: (metadata.mood || 'Other') as Mood, // TODO: SDK requires mood, but XML doesn't provide one
releaseDate: new Date(metadata.release_date),
ddexReleaseIds: metadata.ddex_release_ids,
tags: metadata.tags || '',
upc: metadata.upc || '',
}
Expand Down
9 changes: 9 additions & 0 deletions packages/discovery-provider/ddl/migrations/0060_add_ddex_release_ids.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
begin;

alter table tracks add column if not exists ddex_release_ids jsonb;
create index if not exists idx_ddex_release_ids on tracks using GIN (ddex_release_ids);

alter table playlists add column if not exists ddex_release_ids jsonb;
create index if not exists idx_ddex_release_ids on playlists using GIN (ddex_release_ids);

commit;
1 change: 1 addition & 0 deletions packages/discovery-provider/integration_tests/tasks/entity_manager/test_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@ def test_valid_parse_metadata(app):
"license": "All rights reserved",
"isrc": None,
"iswc": None,
"ddex_release_ids": None,
"track_segments": [
{
"duration": 6.016,
Expand Down
Loading