diff --git a/troi/__init__.py b/troi/__init__.py index beab05e..bc52839 100644 --- a/troi/__init__.py +++ b/troi/__init__.py @@ -311,6 +311,7 @@ def __init__(self, ranking=None, year=None, spotify_id=None, + soundcloud_id=None, apple_music_id=None, musicbrainz=None, listenbrainz=None, @@ -325,6 +326,7 @@ def __init__(self, self.year = year self.spotify_id = spotify_id self.apple_music_id = apple_music_id + self.soundcloud_id = soundcloud_id def __str__(self): return "" % (self.name, self.mbid, self.msid) diff --git a/troi/cli.py b/troi/cli.py index 623f65d..a247c12 100755 --- a/troi/cli.py +++ b/troi/cli.py @@ -74,9 +74,20 @@ def cli(): type=str, required=False, multiple=True) +@click.option('--soundcloud-user-id', help="The soundcloud user name to upload the playlist to", type=str, required=False) +@click.option('--soundcloud-token', + help="The soundcloud token with the correct permissions required to upload playlists", + type=str, + required=False) +@click.option('--soundcloud-url', + help="instead of creating a new soundcloud playlist, update the existing playlist at this url", + type=str, + required=False, + multiple=True) @click.argument('args', nargs=-1, type=click.UNPROCESSED) def playlist(patch, quiet, save, token, upload, args, created_for, name, desc, min_recordings, spotify_user_id, spotify_token, - spotify_url, apple_music_developer_token, apple_music_user_token, apple_music_url): + spotify_url, soundcloud_user_id, soundcloud_token, soundcloud_url, + apple_music_developer_token, apple_music_user_token, apple_music_url): """ Generate a global MBID based playlist using a patch """ @@ -106,7 +117,6 @@ def playlist(patch, quiet, save, token, upload, args, created_for, name, desc, m "is_collaborative": False, "existing_urls": spotify_url } - if apple_music_developer_token: patch_args["apple_music"] = { "developer_token": apple_music_developer_token, @@ -115,6 +125,14 @@ def playlist(patch, quiet, save, token, upload, args, created_for, name, desc, m "is_collaborative": False, "existing_urls": apple_music_url } + if soundcloud_token: + patch_args["soundcloud"] = { + "user_id": soundcloud_user_id, + "token": soundcloud_token, + "is_public": True, + "is_collaborative": False, + "existing_urls": soundcloud_url + } if args is None: args = [] diff --git a/troi/patch.py b/troi/patch.py index b9d7ac8..2a89208 100755 --- a/troi/patch.py +++ b/troi/patch.py @@ -19,6 +19,7 @@ min_recordings=10, spotify=None, apple_music=None, + soundcloud=None, quiet=False) @@ -147,6 +148,7 @@ def generate_playlist(self): * min-recordings: The minimum number of recordings that must be present in a playlist to consider it complete. If it doesn't have sufficient numbers of tracks, ignore the playlist and don't submit it. Default: Off, a playlist with at least one track will be considere complete. * spotify: if present, attempt to submit the playlist to spotify as well. should be a dict and contain the spotify user id, spotify auth token with appropriate permissions, whether the playlist should be public, private or collaborative. it can also optionally have the existing urls to update playlists instead of creating new ones. * apple_music: if present, attempt to submit the playlist to Apple Music as well. should be a dict and contain the apple developer token, user music token, whether the playlist should be public, private. it can also optionally have the existing urls to update playlists instead of creating new ones. + * soundcloud: if present, attempt to submit the palylist to soundcloud. should contain soundcloud auth token, whether the playlist should be public or private """ try: @@ -172,7 +174,8 @@ def generate_playlist(self): token = self.patch_args["token"] spotify = self.patch_args["spotify"] apple_music = self.patch_args["apple_music"] - if upload and not token and not spotify and not apple_music: + soundcloud = self.patch_args["soundcloud"] + if upload and not token and not spotify and not apple_music and not soundcloud: raise RuntimeError("In order to upload a playlist, you must provide an auth token. Use option --token.") min_recordings = self.patch_args["min_recordings"] @@ -186,6 +189,11 @@ def generate_playlist(self): spotify["is_collaborative"], spotify.get("existing_urls", [])): logger.info("Submitted playlist to spotify: %s" % url) + if result is not None and soundcloud and upload: + for url, _ in playlist.submit_to_soundcloud(soundcloud["user_id"], soundcloud["token"], soundcloud["is_public"], + soundcloud.get("existing_urls", [])): + logger.info("Submitted playlist to soundcloud: %s" % url) + if result is not None and apple_music and upload: for url, _ in playlist.submit_to_apple_music(apple_music["music_user_token"], apple_music["developer_token"], apple_music["is_public"], apple_music.get("existing_urls", [])): logger.info("Submitted playlist to apple music: %s" % url) diff --git a/troi/patches/playlist_from_ms.py b/troi/patches/playlist_from_ms.py index 155371f..cc4c45b 100644 --- a/troi/patches/playlist_from_ms.py +++ b/troi/patches/playlist_from_ms.py @@ -20,7 +20,7 @@ def inputs(): MS_TOKEN is the music service token from which the playlist is retrieved. For now, only Spotify tokens are accepted. PLAYLIST_ID is the playlist id to retrieve the tracks from it. MUSIC_SERVICE is the music service from which the playlist is retrieved - APPLE_USER_TOKEN is the apple user token. Optional, if music services is not Apple Music + APPLE_USER_TOKEN is the apple user token. Optional, if music service is not Apple Music """ return [ {"type": "argument", "args": ["ms_token"], "kwargs": {"required": False}}, diff --git a/troi/playlist.py b/troi/playlist.py index ba25781..ce69863 100755 --- a/troi/playlist.py +++ b/troi/playlist.py @@ -13,6 +13,8 @@ from troi.tools.common_lookup import music_service_tracks_to_mbid from troi.tools.spotify_lookup import submit_to_spotify from troi.tools.apple_lookup import submit_to_apple_music +from troi.tools.soundcloud_lookup import submit_to_soundcloud +from troi.tools.utils import SoundcloudAPI logger = logging.getLogger(__name__) @@ -333,7 +335,7 @@ def submit_to_apple_music(self, for idx, playlist in enumerate(self.playlists): if len(playlist.recordings) == 0: continue - + existing_url = None if existing_urls and idx < len(existing_urls) and existing_urls[idx]: existing_url = existing_urls[idx] @@ -343,9 +345,34 @@ def submit_to_apple_music(self, return submitted + def submit_to_soundcloud(self, + user_id: str, + access_token: str, + is_public: bool = True, + existing_urls: str = None): + """ Given soundcloud user id, soundcloud auth token, upload the playlists generated in the current element to Soundcloud and return the + urls of submitted playlists. + + """ + sd = SoundcloudAPI(access_token=access_token) + submitted = [] + + for idx, playlist in enumerate(self.playlists): + if len(playlist.recordings) == 0: + continue + + existing_url = None + if existing_urls and idx < len(existing_urls) and existing_urls[idx]: + existing_url = existing_urls[idx] + + playlist_url, playlist_id = submit_to_soundcloud(sd, playlist, is_public, existing_url) + submitted.append((playlist_url, playlist_id)) + + return submitted + class PlaylistRedundancyReducerElement(Element): - + ''' This element takes a larger playlist and whittles it down to a smaller playlist by removing some tracks in order to reduce the number of times a single artist appears diff --git a/troi/tools/soundcloud_lookup.py b/troi/tools/soundcloud_lookup.py index 9e49bf8..0856df1 100644 --- a/troi/tools/soundcloud_lookup.py +++ b/troi/tools/soundcloud_lookup.py @@ -1,22 +1,141 @@ -from .utils import create_http_session +import requests +import logging + +from collections import defaultdict +from more_itertools import chunked +from .utils import SoundcloudAPI, SoundCloudException + +logger = logging.getLogger(__name__) + +SOUNDCLOUD_IDS_LOOKUP_URL = "https://labs.api.listenbrainz.org/soundcloud-id-from-mbid/json" + +def lookup_soundcloud_ids(recordings): + """ Given a list of Recording elements, try to find soundcloud track ids from labs api soundcloud lookup using mbids + and add those to the recordings. """ + response = requests.post( + SOUNDCLOUD_IDS_LOOKUP_URL, + json=[{"recording_mbid": recording.mbid} for recording in recordings] + ) + response.raise_for_status() + + soundcloud_data = response.json() + mbid_soundcloud_ids_index = {} + soundcloud_id_mbid_index = {} + for recording, lookup in zip(recordings, soundcloud_data): + if len(lookup["soundcloud_track_ids"]) > 0: + recording.soundcloud_id = lookup["soundcloud_track_ids"][0] + mbid_soundcloud_ids_index[recording.mbid] = lookup["soundcloud_track_ids"] + for soundcloud_id in lookup["soundcloud_track_ids"]: + soundcloud_id_mbid_index[soundcloud_id] = recording.mbid + return recordings, mbid_soundcloud_ids_index, soundcloud_id_mbid_index + + +def _check_unplayable_tracks(soundcloud: SoundcloudAPI, playlist_id: str): + """ Retrieve tracks for given soundcloud playlist and split into lists of playable and unplayable tracks """ + tracks = soundcloud.get_playlist_tracks(playlist_id, linked_partitioning=True, limit=100, access=['playable, preview ,blocked']) + track_details = [ + { + "id": track["id"], + "title": track["title"], + "access": track["access"] + } + for track in tracks + ] + + playable = [] + unplayable = [] + for idx, item in enumerate(track_details): + if item["access"] == "playable": + playable.append((idx, item["id"])) + else: + unplayable.append((idx, item["id"])) + return playable, unplayable + + +def _get_alternative_track_ids(unplayable, mbid_soundcloud_id_idx, soundcloud_id_mbid_idx): + """ For the list of unplayable track ids, find alternative tracks ids. + + mbid_soundcloud_id_idx is an index with mbid as key and list of equivalent soundcloud ids as value. + soundcloud_id_mbid_idx is an index with soundcloud_id as key and the corresponding mbid as value. + """ + index = defaultdict(list) + soundcloud_ids = [] + for idx, soundcloud_id in unplayable: + mbid = soundcloud_id_mbid_idx[str(soundcloud_id)] + other_soundcloud_ids = mbid_soundcloud_id_idx[mbid] + + for new_idx, new_soundcloud_id in enumerate(other_soundcloud_ids): + if new_soundcloud_id == soundcloud_id: + continue + soundcloud_ids.append(new_soundcloud_id) + index[idx].append(new_soundcloud_id) + + return soundcloud_ids, index + + +def _get_fixed_up_tracks(soundcloud: SoundcloudAPI, soundcloud_ids, index): + """ Lookup the all alternative soundcloud track ids, filter playable ones and if multiple track ids + for same item match, prefer the one occurring earlier. If no alternative is playable, ignore the + item altogether. + """ + new_tracks = soundcloud.get_track_details(soundcloud_ids) + + new_tracks_ids = set() + for item in new_tracks: + if item["access"] == "playable": + new_tracks_ids.add(item["id"]) + + fixed_up_items = [] + for idx, soundcloud_ids in index.items(): + for soundcloud_id in soundcloud_ids: + if soundcloud_id in new_tracks_ids: + fixed_up_items.append((idx, soundcloud_id)) + break + return fixed_up_items + + +def fixup_soundcloud_playlist(soundcloud: SoundcloudAPI, playlist_id: str, mbid_soundcloud_id_idx, soundcloud_id_mbid_idx): + """ Fix unplayable tracks in the given soundcloud playlist. + + Given a soundcloud playlist id, look it up and find unstreamable tracks. If there are any unstreamable tracks, try + alternative soundcloud track ids from index/reverse_index if available. If alternative is not available or alternatives + also are unplayable, remove the track entirely from the playlist. Finally, update the playlist if needed. + """ + playable, unplayable = _check_unplayable_tracks(soundcloud, playlist_id) + if not unplayable: + return + + alternative_ids, index = _get_alternative_track_ids(unplayable, mbid_soundcloud_id_idx, soundcloud_id_mbid_idx) + if not alternative_ids: + return + + fixed_up = _get_fixed_up_tracks(soundcloud, alternative_ids, index) + all_items = [] + all_items.extend(playable) + all_items.extend(fixed_up) + + # sort all items by index value to ensure the order of tracks of original playlist is preserved + all_items.sort(key=lambda x: x[0]) + # update all track ids the soundcloud playlist + finalized_ids = [x[1] for x in all_items] + + # clear existing playlist + soundcloud.update_playlist_details(playlist_id, track_ids=[]) + + # chunking requests to avoid hitting rate limits + for chunk in chunked(finalized_ids, 100): + soundcloud.add_playlist_tracks(playlist_id, chunk) -SOUNDCLOUD_URL = f"https://api.soundcloud.com/" def get_tracks_from_soundcloud_playlist(developer_token, playlist_id): """ Get tracks from the Soundcloud playlist. """ - http = create_http_session() + soundcloud = SoundcloudAPI(developer_token) + data = soundcloud.get_playlist_tracks(playlist_id, linked_partitioning=True, limit=100, access=['playable, preview ,blocked']) - headers = { - "Authorization": f"OAuth {developer_token}", - } - response = http.get(f"{SOUNDCLOUD_URL}/playlists/{playlist_id}", headers=headers) - response.raise_for_status() - - response = response.json() - tracks = response["tracks"] - name = response["title"] - description = response["description"] + tracks = data["tracks"] + name = data["title"] + description = data["description"] mapped_tracks = [ { @@ -27,3 +146,54 @@ def get_tracks_from_soundcloud_playlist(developer_token, playlist_id): ] return mapped_tracks, name, description + + +def submit_to_soundcloud(soundcloud: SoundcloudAPI, playlist, is_public: bool = True, + existing_url: str = None): + """ Submit or update an existing soundcloud playlist. + + If existing urls are specified then is_public and is_collaborative arguments are ignored. + """ + filtered_recordings = [r for r in playlist.recordings if r.mbid] + + _, mbid_soundcloud_index, soundcloud_mbid_index = lookup_soundcloud_ids(filtered_recordings) + soundcloud_track_ids = [r.soundcloud_id for r in filtered_recordings if r.soundcloud_id] + if len(soundcloud_track_ids) == 0: + return None, None + + logger.info("submit %d tracks" % len(soundcloud_track_ids)) + + playlist_id, playlist_url = None, None + if existing_url: + # update existing playlist + playlist_url = existing_url + playlist_id = existing_url.split("/")[-1] + try: + soundcloud.update_playlist_details(playlist_id=playlist_id, title=playlist.name, description=playlist.description) + except SoundCloudException as err: + # one possibility is that the user has deleted the soundcloud from playlist, so try creating a new one + logger.info("provided playlist url has been unfollowed/deleted by the user, creating a new one") + playlist_id, playlist_url = None, None + + if not playlist_id: + # create new playlist + soundcloud_playlist = soundcloud.create_playlist( + title=playlist.name, + sharing=is_public, + description=playlist.description + ) + playlist_id = soundcloud_playlist["id"] + playlist_url = soundcloud_playlist["permalink"] + else: + # existing playlist, clear it + tracks = map(lambda id: dict(id=id), [0]) + soundcloud.update_playlist(playlist_id, tracks) + + for chunk in chunked(soundcloud_track_ids, 100): + soundcloud.add_playlist_tracks(playlist_id, chunk) + + fixup_soundcloud_playlist(soundcloud, playlist_id, mbid_soundcloud_index, soundcloud_mbid_index) + + playlist.add_metadata({"external_urls": {"soundcloud": playlist_url}}) + + return playlist_url, playlist_id \ No newline at end of file diff --git a/troi/tools/spotify_lookup.py b/troi/tools/spotify_lookup.py index b1ed1c3..aebb3d3 100644 --- a/troi/tools/spotify_lookup.py +++ b/troi/tools/spotify_lookup.py @@ -181,11 +181,11 @@ def get_tracks_from_spotify_playlist(spotify_token, playlist_id): name = playlist_info["name"] description = playlist_info["description"] - tracks = convert_spotify_tracks_to_json(tracks) + tracks = _convert_spotify_tracks_to_json(tracks) return tracks, name, description -def convert_spotify_tracks_to_json(spotify_tracks): +def _convert_spotify_tracks_to_json(spotify_tracks): tracks = [] for track in spotify_tracks["items"]: artists = track["track"].get("artists", []) diff --git a/troi/tools/utils.py b/troi/tools/utils.py index dbada09..9bbfcfb 100644 --- a/troi/tools/utils.py +++ b/troi/tools/utils.py @@ -5,9 +5,9 @@ from urllib3.util.retry import Retry logger = logging.getLogger(__name__) +SOUNDCLOUD_URL = f"https://api.soundcloud.com/" APPLE_MUSIC_URL = f"https://api.music.apple.com/v1" - class AppleMusicException(Exception): def __init__(self, code, msg): self.code = code @@ -32,7 +32,7 @@ def _get_user_storefront(self): """ url=f"{APPLE_MUSIC_URL}/me/storefront" response = self.session.get(url, headers=self.headers) - + data = response.json()["data"][0]["id"] return data @@ -46,8 +46,7 @@ def create_playlist(self, name, is_public=True,description=None): } if description: data["attributes"]["description"] = description - response = self.session.post(url, headers=self.headers, data=json.dumps(data)) - return response.json() + self.session.post(url, headers=self.headers, data=json.dumps(data)) def playlist_add_tracks(self, playlist_id, track_ids): """ Adds tracks to a playlist in Apple Music, does not return response @@ -64,13 +63,94 @@ def get_playlist_tracks(self, playlist_id): return response.json() +class SoundCloudException(Exception): + def __init__(self, code, msg): + self.code = code + self.msg = msg + super().__init__(f"http error {code}: {msg}") + + +class SoundcloudAPI: + def __init__(self, access_token): + self.access_token = access_token + self.headers = { + "Authorization": f"OAuth {self.access_token}", + "Content-Type": "application/json" + } + self.session = create_http_session() + + def create_playlist(self, title, sharing="public", track_ids=None, description=None): + url = f"{SOUNDCLOUD_URL}/playlists" + data = { + "playlist": { + "title": title, + "sharing": sharing, + } + } + if track_ids: + data["playlist"]["tracks"] = [{"id": track_id} for track_id in track_ids] + + if description: + data["playlist"]["description"] = description + response = self.session.post(url, headers=self.headers, data=json.dumps(data)) + return response.json() + + def add_playlist_tracks(self, playlist_id, track_ids): + url = f"{SOUNDCLOUD_URL}/playlists/{playlist_id}" + data = { + "playlist": { + "tracks": [{"id": track_id} for track_id in track_ids] + } + } + response = self.session.put(url, headers=self.headers, data=json.dumps(data)) + return response.json() + + def update_playlist_details(self, playlist_id, title=None, description=None, track_ids=None): + url = f"{SOUNDCLOUD_URL}/playlists/{playlist_id}" + data = { + "playlist": {} + } + if title: + data["playlist"]["title"] = title + if description: + data["playlist"]["description"] = description + if track_ids: + data["playlist"]["tracks"] = [{"id": track_id} for track_id in track_ids] + + response = self.session.put(url, headers=self.headers, data=json.dumps(list(data))) + return response.json() + + def get_track_details(self, track_ids): + track_details = [] + + for track_id in track_ids: + url = f"{SOUNDCLOUD_URL}/tracks/{track_id}" + response = self.session.get(url, headers=self.headers) + data = response.json() + track_details.append(data) + + return track_details + + def get_playlist_tracks(self, playlist_id, **kwargs): + tracks = [] + url = f"{SOUNDCLOUD_URL}/playlists/{playlist_id}/tracks" + + while url: + response = self.session.get(url, headers=self.headers, params=kwargs) + data = response.json() + tracks.extend(data.get("collection", [])) + url = data.get("next_href") + + return tracks + + def create_http_session(): """ Create an HTTP session with retry strategy for handling rate limits and server errors. """ retry_strategy = Retry( total=3, - status_forcelist=[429, 500, 502, 503, 504], - allowed_methods=["HEAD", "GET", "OPTIONS"], + status_forcelist=[429, 500, 502, 503, 504], + allowed_methods=["HEAD", "GET", "OPTIONS"], backoff_factor=1 )