Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Parse more ddex fields #7854

Merged
merged 2 commits into from
Mar 15, 2024
Merged

Parse more ddex fields #7854

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c23e9ee
parse out fields in ingester
michellebrier Mar 15, 2024
5d065da
e2e test
michellebrier Mar 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 64 additions & 28 deletions packages/ddex/ingester/artistutils/artistutils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@ import (
"encoding/json"
"errors"
"fmt"
"ingester/common"
"io"
"net/http"
"sort"
"strings"

"go.mongodb.org/mongo-driver/bson"
Expand All @@ -24,42 +26,76 @@ func CreateArtistNameIndex(usersColl *mongo.Collection, ctx context.Context) err
return nil
}

// GetArtistID searches Mongo OAuthed artists for an exact match on artistName (display name) and returns the artist's ID
func GetArtistID(artistName string, usersColl *mongo.Collection, ctx context.Context) (string, error) {
filter := bson.M{"name": artistName}
cursor, err := usersColl.Find(ctx, filter)
if err != nil {
return "", err
}
defer cursor.Close(ctx)
// GetFirstArtistID searches Mongo OAuthed artists for the first match on the track's display artists' names
// and returns that artist's ID
func GetFirstArtistID(artists []common.Artist, usersColl *mongo.Collection, ctx context.Context) (string, string, []string, error) {
// Sort artists by SequenceNumber (asc)
sort.Slice(artists, func(i, j int) bool {
return artists[i].SequenceNumber < artists[j].SequenceNumber
})

var results []bson.M
if err = cursor.All(ctx, &results); err != nil {
return "", err
}
artistID := ""
artistName := ""
var err error
var warnings []string
for _, artist := range artists {
filter := bson.M{"name": artist.Name}
cursor, err := usersColl.Find(ctx, filter)
if err != nil {
return "", "", warnings, err
}
defer cursor.Close(ctx)

// Fail if multiple artists have their display name set to the same string
if len(results) > 1 {
var handles []string
for _, result := range results {
if handle, ok := result["handle"].(string); ok {
handles = append(handles, "@"+handle)
var results []bson.M
if err = cursor.All(ctx, &results); err != nil {
return "", "", warnings, err
}

// Continue to the next display artist if multiple artists have their display name set to this artist.Name
if len(results) > 1 {
var handles []string
for _, result := range results {
if handle, ok := result["handle"].(string); ok {
handles = append(handles, "@"+handle)
}
}
idsStr := strings.Join(handles, ", ")
warnings = append(warnings, fmt.Sprintf("error: more than one artist found with the same display name %s: %s", artist.Name, idsStr))
continue
}

// Continue to the next display artist if no artist is found, and use /v1/users/search to display potential
// matches in the warnings
if len(results) == 0 {
id, err := searchArtistOnAudius(artist.Name)
if err != nil {
warnings = append(warnings, err.Error())
continue
}
// Unreachable for now
artistID = id
artistName = artist.Name
err = nil
break
}

id, ok := results[0]["_id"].(string)
if !ok {
warnings = append(warnings, "error: unable to parse _id from the users collection")
continue
}
idsStr := strings.Join(handles, ", ")
return "", fmt.Errorf("error: more than one artist found with the same display name: %s", idsStr)
}

// Fail if no artist is found, and use /v1/users/search to display potential matches in the error message
if len(results) == 0 {
return searchArtistOnAudius(artistName)
// Found first OAuthed artist ID
artistID = id
artistName = artist.Name
err = nil
break
}

artistID, ok := results[0]["_id"].(string)
if !ok {
return "", errors.New("error: unable to parse artist ID")
if artistID != "" {
return artistID, artistName, warnings, nil
}
return artistID, nil
return "", "", warnings, err
}

// searchArtistOnAudius searches Audius (public /v1/users/search endpoint) for an artist by name and returns potential matches.
Expand Down
70 changes: 55 additions & 15 deletions packages/ddex/ingester/common/sdk_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,25 +191,57 @@ type TrackMetadata struct {
Tags NullableString `bson:"tags,omitempty"`

// Extra fields (not in SDK)
Artists []Artist `bson:"artists"`
ArtistID string `bson:"artist_id"`
ArtistName string `bson:"artist_name"`
Copyright string `bson:"copyright"`
PreviewAudioFileURL string `bson:"preview_audio_file_url"`
PreviewAudioFileURLHash string `bson:"preview_audio_file_url_hash"`
PreviewAudioFileURLHashAlgo string `bson:"preview_audio_file_url_hash_algo"`
AudioFileURL string `bson:"audio_file_url"`
AudioFileURLHash string `bson:"audio_file_url_hash"`
AudioFileURLHashAlgo string `bson:"audio_file_url_hash_algo"`
CoverArtURL string `bson:"cover_art_url"`
CoverArtURLHash string `bson:"cover_art_url_hash"`
CoverArtURLHashAlgo string `bson:"cover_art_url_hash_algo"`

// michelle add to sdk ==
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is like a personal todo?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah

Artists []Artist `bson:"artists"`
// michelle add to sdk ==

ArtistID string `bson:"artist_id"`
ArtistName string `bson:"artist_name"`

// michelle add to sdk ==
ResourceContributors []ResourceContributor `bson:"resource_contributor,omitempty"`
IndirectResourceContributors []ResourceContributor `bson:"indirect_resource_contributor,omitempty"`
RightsController RightsController `bson:"rights_controller,omitempty"`
CopyrightLine Copyright `bson:"copyright_line,omitempty"`
ProducerCopyrightLine Copyright `bson:"producer_copyright_line,omitempty"`
ParentalWarningType string `bson:"parental_warning_type,omitempty"`
// michelle add to sdk ==

PreviewAudioFileURL string `bson:"preview_audio_file_url"`
PreviewAudioFileURLHash string `bson:"preview_audio_file_url_hash"`
PreviewAudioFileURLHashAlgo string `bson:"preview_audio_file_url_hash_algo"`
AudioFileURL string `bson:"audio_file_url"`
AudioFileURLHash string `bson:"audio_file_url_hash"`
AudioFileURLHashAlgo string `bson:"audio_file_url_hash_algo"`
CoverArtURL string `bson:"cover_art_url"`
CoverArtURLHash string `bson:"cover_art_url_hash"`
CoverArtURLHashAlgo string `bson:"cover_art_url_hash_algo"`
}

// Not part of SDK
type Artist struct {
Name string `bson:"name"`
Roles []string `bson:"roles"`
Name string `bson:"name"`
Roles []string `bson:"roles"`
SequenceNumber int `bson:"sequence_number"`
}

type Copyright struct {
Year string
Text string
}

// ResourceContributor represents a contributor to the sound recording
type ResourceContributor struct {
Name string
Roles []string
SequenceNumber int `bson:"sequence_number"`
}

type RightsController struct {
Name string
Roles []string
RightsShareUnknown string
}

type CollectionMetadata struct {
Expand All @@ -230,6 +262,14 @@ type CollectionMetadata struct {
UPC NullableString `bson:"upc,omitempty"`

// Extra fields (not in SDK)

// michelle add to sdk ==
Artists []Artist `bson:"artists"`
CopyrightLine Copyright `bson:"copyright_line,omitempty"`
ProducerCopyrightLine Copyright `bson:"producer_copyright_line,omitempty"`
ParentalWarningType string `bson:"parental_warning_type,omitempty"`
// michelle add to sdk ==

CoverArtURL string `bson:"cover_art_url"`
CoverArtURLHash string `bson:"cover_art_url_hash"`
CoverArtURLHashAlgo string `bson:"cover_art_url_hash_algo"`
Expand Down
Loading