Skip to content

Commit

Permalink
Parse more ddex fields (#7854)
Browse files Browse the repository at this point in the history
  • Loading branch information
@michellebrier
michellebrier authored Mar 15, 2024
1 parent 77078ae commit 9610aa5
Show file tree
Hide file tree
Showing 5 changed files with 495 additions and 100 deletions.
92 changes: 64 additions & 28 deletions packages/ddex/ingester/artistutils/artistutils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@ import (
"encoding/json"
"errors"
"fmt"
"ingester/common"
"io"
"net/http"
"sort"
"strings"

"go.mongodb.org/mongo-driver/bson"
Expand All @@ -24,42 +26,76 @@ func CreateArtistNameIndex(usersColl *mongo.Collection, ctx context.Context) err
return nil
}

// GetArtistID searches Mongo OAuthed artists for an exact match on artistName (display name) and returns the artist's ID
func GetArtistID(artistName string, usersColl *mongo.Collection, ctx context.Context) (string, error) {
filter := bson.M{"name": artistName}
cursor, err := usersColl.Find(ctx, filter)
if err != nil {
return "", err
}
defer cursor.Close(ctx)
// GetFirstArtistID searches Mongo OAuthed artists for the first match on the track's display artists' names
// and returns that artist's ID
func GetFirstArtistID(artists []common.Artist, usersColl *mongo.Collection, ctx context.Context) (string, string, []string, error) {
// Sort artists by SequenceNumber (asc)
sort.Slice(artists, func(i, j int) bool {
return artists[i].SequenceNumber < artists[j].SequenceNumber
})

var results []bson.M
if err = cursor.All(ctx, &results); err != nil {
return "", err
}
artistID := ""
artistName := ""
var err error
var warnings []string
for _, artist := range artists {
filter := bson.M{"name": artist.Name}
cursor, err := usersColl.Find(ctx, filter)
if err != nil {
return "", "", warnings, err
}
defer cursor.Close(ctx)

// Fail if multiple artists have their display name set to the same string
if len(results) > 1 {
var handles []string
for _, result := range results {
if handle, ok := result["handle"].(string); ok {
handles = append(handles, "@"+handle)
var results []bson.M
if err = cursor.All(ctx, &results); err != nil {
return "", "", warnings, err
}

// Continue to the next display artist if multiple artists have their display name set to this artist.Name
if len(results) > 1 {
var handles []string
for _, result := range results {
if handle, ok := result["handle"].(string); ok {
handles = append(handles, "@"+handle)
}
}
idsStr := strings.Join(handles, ", ")
warnings = append(warnings, fmt.Sprintf("error: more than one artist found with the same display name %s: %s", artist.Name, idsStr))
continue
}

// Continue to the next display artist if no artist is found, and use /v1/users/search to display potential
// matches in the warnings
if len(results) == 0 {
id, err := searchArtistOnAudius(artist.Name)
if err != nil {
warnings = append(warnings, err.Error())
continue
}
// Unreachable for now
artistID = id
artistName = artist.Name
err = nil
break
}

id, ok := results[0]["_id"].(string)
if !ok {
warnings = append(warnings, "error: unable to parse _id from the users collection")
continue
}
idsStr := strings.Join(handles, ", ")
return "", fmt.Errorf("error: more than one artist found with the same display name: %s", idsStr)
}

// Fail if no artist is found, and use /v1/users/search to display potential matches in the error message
if len(results) == 0 {
return searchArtistOnAudius(artistName)
// Found first OAuthed artist ID
artistID = id
artistName = artist.Name
err = nil
break
}

artistID, ok := results[0]["_id"].(string)
if !ok {
return "", errors.New("error: unable to parse artist ID")
if artistID != "" {
return artistID, artistName, warnings, nil
}
return artistID, nil
return "", "", warnings, err
}

// searchArtistOnAudius searches Audius (public /v1/users/search endpoint) for an artist by name and returns potential matches.
Expand Down
70 changes: 55 additions & 15 deletions packages/ddex/ingester/common/sdk_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,25 +191,57 @@ type TrackMetadata struct {
Tags NullableString `bson:"tags,omitempty"`

// Extra fields (not in SDK)
Artists []Artist `bson:"artists"`
ArtistID string `bson:"artist_id"`
ArtistName string `bson:"artist_name"`
Copyright string `bson:"copyright"`
PreviewAudioFileURL string `bson:"preview_audio_file_url"`
PreviewAudioFileURLHash string `bson:"preview_audio_file_url_hash"`
PreviewAudioFileURLHashAlgo string `bson:"preview_audio_file_url_hash_algo"`
AudioFileURL string `bson:"audio_file_url"`
AudioFileURLHash string `bson:"audio_file_url_hash"`
AudioFileURLHashAlgo string `bson:"audio_file_url_hash_algo"`
CoverArtURL string `bson:"cover_art_url"`
CoverArtURLHash string `bson:"cover_art_url_hash"`
CoverArtURLHashAlgo string `bson:"cover_art_url_hash_algo"`

// michelle add to sdk ==
Artists []Artist `bson:"artists"`
// michelle add to sdk ==

ArtistID string `bson:"artist_id"`
ArtistName string `bson:"artist_name"`

// michelle add to sdk ==
ResourceContributors []ResourceContributor `bson:"resource_contributor,omitempty"`
IndirectResourceContributors []ResourceContributor `bson:"indirect_resource_contributor,omitempty"`
RightsController RightsController `bson:"rights_controller,omitempty"`
CopyrightLine Copyright `bson:"copyright_line,omitempty"`
ProducerCopyrightLine Copyright `bson:"producer_copyright_line,omitempty"`
ParentalWarningType string `bson:"parental_warning_type,omitempty"`
// michelle add to sdk ==

PreviewAudioFileURL string `bson:"preview_audio_file_url"`
PreviewAudioFileURLHash string `bson:"preview_audio_file_url_hash"`
PreviewAudioFileURLHashAlgo string `bson:"preview_audio_file_url_hash_algo"`
AudioFileURL string `bson:"audio_file_url"`
AudioFileURLHash string `bson:"audio_file_url_hash"`
AudioFileURLHashAlgo string `bson:"audio_file_url_hash_algo"`
CoverArtURL string `bson:"cover_art_url"`
CoverArtURLHash string `bson:"cover_art_url_hash"`
CoverArtURLHashAlgo string `bson:"cover_art_url_hash_algo"`
}

// Not part of SDK
type Artist struct {
Name string `bson:"name"`
Roles []string `bson:"roles"`
Name string `bson:"name"`
Roles []string `bson:"roles"`
SequenceNumber int `bson:"sequence_number"`
}

type Copyright struct {
Year string
Text string
}

// ResourceContributor represents a contributor to the sound recording
type ResourceContributor struct {
Name string
Roles []string
SequenceNumber int `bson:"sequence_number"`
}

type RightsController struct {
Name string
Roles []string
RightsShareUnknown string
}

type CollectionMetadata struct {
Expand All @@ -230,6 +262,14 @@ type CollectionMetadata struct {
UPC NullableString `bson:"upc,omitempty"`

// Extra fields (not in SDK)

// michelle add to sdk ==
Artists []Artist `bson:"artists"`
CopyrightLine Copyright `bson:"copyright_line,omitempty"`
ProducerCopyrightLine Copyright `bson:"producer_copyright_line,omitempty"`
ParentalWarningType string `bson:"parental_warning_type,omitempty"`
// michelle add to sdk ==

CoverArtURL string `bson:"cover_art_url"`
CoverArtURLHash string `bson:"cover_art_url_hash"`
CoverArtURLHashAlgo string `bson:"cover_art_url_hash_algo"`
Expand Down
Loading

0 comments on commit 9610aa5

Please sign in to comment.