From 2c471370e4ac4b396a248f8e948a56ca127910ce Mon Sep 17 00:00:00 2001 From: Markury Date: Wed, 18 Mar 2026 11:08:08 -0400 Subject: [PATCH] feat(spotify): caching for Spotify to Deezer conversions to reduce API calls --- src-tauri/src/lib.rs | 30 ++- src/lib/library/spotify-database.ts | 57 ++++++ .../services/spotify/playlistDownloader.ts | 192 ++++++++++-------- 3 files changed, 185 insertions(+), 94 deletions(-) diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index a4d6aab..c38b356 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -299,10 +299,11 @@ pub fn run() { kind: MigrationKind::Up, }]; - let spotify_migrations = vec![Migration { - version: 1, - description: "create_spotify_cache_tables", - sql: " + let spotify_migrations = vec![ + Migration { + version: 1, + description: "create_spotify_cache_tables", + sql: " CREATE TABLE IF NOT EXISTS spotify_playlists ( id TEXT PRIMARY KEY, name TEXT NOT NULL, @@ -364,8 +365,25 @@ pub fn run() { CREATE INDEX IF NOT EXISTS idx_spotify_playlist_tracks_track ON spotify_playlist_tracks(track_id); CREATE INDEX IF NOT EXISTS idx_spotify_playlist_tracks_isrc ON spotify_playlist_tracks(isrc); ", - kind: MigrationKind::Up, - }]; + kind: MigrationKind::Up, + }, + Migration { + version: 2, + description: "create_spotify_deezer_conversion_cache", + sql: " + CREATE TABLE IF NOT EXISTS spotify_deezer_conversions ( + spotify_track_id TEXT PRIMARY KEY, + deezer_track_id INTEGER NOT NULL, + deezer_track_json TEXT NOT NULL, + match_method TEXT NOT NULL, + cached_at INTEGER NOT NULL + ); + + CREATE INDEX IF NOT EXISTS idx_conversions_deezer_id ON spotify_deezer_conversions(deezer_track_id); + ", + kind: MigrationKind::Up, + }, + ]; tauri::Builder::default() .plugin(tauri_plugin_oauth::init()) diff --git a/src/lib/library/spotify-database.ts b/src/lib/library/spotify-database.ts index ac40439..bfc304c 100644 --- a/src/lib/library/spotify-database.ts +++ b/src/lib/library/spotify-database.ts @@ -318,6 +318,63 @@ export async function upsertPlaylistTracks(playlistId: string, tracks: any[]): P } } +/** + * Cached Spotify→Deezer conversion result + */ +export interface CachedConversion { + spotify_track_id: string; + deezer_track_id: number; + deezer_track_json: string; + match_method: string; + cached_at: number; +} + +/** + * Get cached Spotify→Deezer conversions for a batch of track IDs + */ +export async function getCachedConversions( + spotifyTrackIds: string[] +): Promise> { + if (spotifyTrackIds.length === 0) return new Map(); + + const database = await initSpotifyDatabase(); + const map = new Map(); + + // Query in batches of 100 to avoid SQLite variable limits + for (let i = 0; i < spotifyTrackIds.length; i += 100) { + const batch = spotifyTrackIds.slice(i, i + 100); + const placeholders = batch.map((_, idx) => `$${idx + 1}`).join(','); + const rows = await database.select( + `SELECT * FROM spotify_deezer_conversions WHERE spotify_track_id IN (${placeholders})`, + batch + ); + for (const row of rows) { + map.set(row.spotify_track_id, row); + } + } + + return map; +} + +/** + * Cache a Spotify→Deezer conversion result + */ +export async function cacheConversion( + spotifyTrackId: string, + deezerTrackId: number, + deezerTrackJson: string, + matchMethod: string +): Promise { + const database = await initSpotifyDatabase(); + const now = Math.floor(Date.now() / 1000); + + await database.execute( + `INSERT OR REPLACE INTO spotify_deezer_conversions (spotify_track_id, deezer_track_id, deezer_track_json, match_method, cached_at) + VALUES ($1, $2, $3, $4, $5)`, + [spotifyTrackId, deezerTrackId, deezerTrackJson, matchMethod, now] + ); +} + /** * Clear all Spotify cache */ diff --git a/src/lib/services/spotify/playlistDownloader.ts b/src/lib/services/spotify/playlistDownloader.ts index 0ad9827..f0468dd 100644 --- a/src/lib/services/spotify/playlistDownloader.ts +++ b/src/lib/services/spotify/playlistDownloader.ts @@ -1,5 +1,9 @@ /** * Download Spotify playlist - converts tracks to Deezer via ISRC, adds to queue, and creates m3u8 file + * + * Uses a persistent SQLite cache for Spotify→Deezer conversions so that only + * new/uncached tracks require API calls. For a 200-track playlist where 3 + * tracks were added, this reduces API calls from ~400 to ~6. */ import { addToQueue } from '$lib/stores/downloadQueue'; @@ -13,6 +17,7 @@ import { setInfo, setSuccess, setWarning } from '$lib/stores/status'; import { get } from 'svelte/store'; import { mkdir } from '@tauri-apps/plugin-fs'; import { convertSpotifyTrackToDeezer, type SpotifyTrackInput } from './converter'; +import { getCachedConversions, cacheConversion } from '$lib/library/spotify-database'; import type { DeezerTrack } from '$lib/types/deezer'; export interface SpotifyPlaylistTrack { @@ -26,16 +31,8 @@ export interface SpotifyPlaylistTrack { } /** - * Download a Spotify playlist by converting tracks to Deezer equivalents - * - Converts all tracks via ISRC matching - * - Adds converted tracks to the download queue (respects overwrite setting) - * - Creates an m3u8 playlist file with relative paths - * - * @param playlistName - Name of the playlist - * @param spotifyTracks - Array of Spotify track objects - * @param playlistsFolder - Path to playlists folder - * @param musicFolder - Path to music folder - * @returns Object with m3u8 path and statistics + * Download a Spotify playlist by converting tracks to Deezer equivalents. + * Cached conversions are reused — only uncached tracks hit the Deezer API. */ export async function downloadSpotifyPlaylist( playlistName: string, @@ -49,12 +46,12 @@ export async function downloadSpotifyPlaylist( queued: number; skipped: number; failed: number; + cached: number; }; }> { const appSettings = get(settings); const authState = get(deezerAuth); - // Ensure Deezer is authenticated if (!authState.loggedIn || !authState.arl) { throw new Error('Deezer authentication required for downloads'); } @@ -64,95 +61,122 @@ export async function downloadSpotifyPlaylist( console.log(`[SpotifyPlaylistDownloader] Starting download for playlist: ${playlistName}`); console.log(`[SpotifyPlaylistDownloader] Tracks: ${spotifyTracks.length}`); - // Ensure playlists folder exists try { await mkdir(playlistsFolder, { recursive: true }); } catch (error) { // Folder might already exist } - // Track statistics + // --- Look up cached conversions in bulk --- + const spotifyIds = spotifyTracks.map((t) => t.track_id); + const cachedConversions = await getCachedConversions(spotifyIds); + + const cachedCount = cachedConversions.size; + const uncachedCount = spotifyTracks.length - cachedCount; + console.log( + `[SpotifyPlaylistDownloader] Cache: ${cachedCount} cached, ${uncachedCount} need conversion` + ); + let queuedCount = 0; let skippedCount = 0; let failedCount = 0; - // Track successful conversions for m3u8 generation const successfulTracks: Array<{ deezerTrack: DeezerTrack; spotifyTrack: SpotifyPlaylistTrack; }> = []; - // Convert and queue each track for (const spotifyTrack of spotifyTracks) { try { - // Convert Spotify track to Deezer - const conversionInput: SpotifyTrackInput = { - id: spotifyTrack.track_id, - name: spotifyTrack.name, - artists: [spotifyTrack.artist_name], - album: spotifyTrack.album_name, - duration_ms: spotifyTrack.duration_ms, - isrc: spotifyTrack.isrc - }; + let deezerTrack: DeezerTrack; - const conversionResult = await convertSpotifyTrackToDeezer(conversionInput); + // --- Check cache first --- + const cached = cachedConversions.get(spotifyTrack.track_id); - if (!conversionResult.success || !conversionResult.deezerTrack) { - console.warn( - `[SpotifyPlaylistDownloader] Failed to convert: ${spotifyTrack.name} - ${conversionResult.error}` + if (cached) { + // Use cached conversion — zero API calls + deezerTrack = JSON.parse(cached.deezer_track_json) as DeezerTrack; + } else { + // Convert via API (ISRC → metadata fallback) + const conversionInput: SpotifyTrackInput = { + id: spotifyTrack.track_id, + name: spotifyTrack.name, + artists: [spotifyTrack.artist_name], + album: spotifyTrack.album_name, + duration_ms: spotifyTrack.duration_ms, + isrc: spotifyTrack.isrc + }; + + const conversionResult = await convertSpotifyTrackToDeezer(conversionInput); + + if (!conversionResult.success || !conversionResult.deezerTrack) { + console.warn( + `[SpotifyPlaylistDownloader] Failed to convert: ${spotifyTrack.name} - ${conversionResult.error}` + ); + failedCount++; + continue; + } + + const deezerPublicTrack = conversionResult.deezerTrack; + + // Fetch full track data from GW API + const deezerTrackId = deezerPublicTrack.id.toString(); + const deezerFullTrack = await deezerAPI.getTrack(deezerTrackId); + + if (!deezerFullTrack || !deezerFullTrack.SNG_ID) { + console.warn( + `[SpotifyPlaylistDownloader] Could not fetch full Deezer track data for ID: ${deezerTrackId}` + ); + failedCount++; + continue; + } + + deezerTrack = { + id: parseInt(deezerFullTrack.SNG_ID, 10), + title: deezerFullTrack.SNG_TITLE, + artist: deezerFullTrack.ART_NAME, + artistId: parseInt(deezerFullTrack.ART_ID, 10), + artists: [deezerFullTrack.ART_NAME], + album: deezerFullTrack.ALB_TITLE, + albumId: parseInt(deezerFullTrack.ALB_ID, 10), + albumArtist: deezerFullTrack.ART_NAME, + albumArtistId: parseInt(deezerFullTrack.ART_ID, 10), + trackNumber: + typeof deezerFullTrack.TRACK_NUMBER === 'number' + ? deezerFullTrack.TRACK_NUMBER + : parseInt(deezerFullTrack.TRACK_NUMBER, 10), + discNumber: + typeof deezerFullTrack.DISK_NUMBER === 'number' + ? deezerFullTrack.DISK_NUMBER + : parseInt(deezerFullTrack.DISK_NUMBER, 10), + duration: + typeof deezerFullTrack.DURATION === 'number' + ? deezerFullTrack.DURATION + : parseInt(deezerFullTrack.DURATION, 10), + explicit: deezerFullTrack.EXPLICIT_LYRICS === 1, + md5Origin: deezerFullTrack.MD5_ORIGIN, + mediaVersion: deezerFullTrack.MEDIA_VERSION, + trackToken: deezerFullTrack.TRACK_TOKEN + }; + + // Cache the conversion for future runs + await cacheConversion( + spotifyTrack.track_id, + deezerTrack.id, + JSON.stringify(deezerTrack), + conversionResult.matchMethod || 'unknown' + ); + + console.log( + `[SpotifyPlaylistDownloader] Converted & cached: ${deezerTrack.title} (via ${conversionResult.matchMethod})` ); - failedCount++; - continue; } - const deezerPublicTrack = conversionResult.deezerTrack; - - // Fetch full track data from Deezer GW API (needed for download) - const deezerTrackId = deezerPublicTrack.id.toString(); - const deezerFullTrack = await deezerAPI.getTrack(deezerTrackId); - - if (!deezerFullTrack || !deezerFullTrack.SNG_ID) { - console.warn(`[SpotifyPlaylistDownloader] Could not fetch full Deezer track data for ID: ${deezerTrackId}`); - failedCount++; - continue; - } - - // Build DeezerTrack object - const deezerTrack: DeezerTrack = { - id: parseInt(deezerFullTrack.SNG_ID, 10), - title: deezerFullTrack.SNG_TITLE, - artist: deezerFullTrack.ART_NAME, - artistId: parseInt(deezerFullTrack.ART_ID, 10), - artists: [deezerFullTrack.ART_NAME], - album: deezerFullTrack.ALB_TITLE, - albumId: parseInt(deezerFullTrack.ALB_ID, 10), - albumArtist: deezerFullTrack.ART_NAME, - albumArtistId: parseInt(deezerFullTrack.ART_ID, 10), - trackNumber: - typeof deezerFullTrack.TRACK_NUMBER === 'number' - ? deezerFullTrack.TRACK_NUMBER - : parseInt(deezerFullTrack.TRACK_NUMBER, 10), - discNumber: - typeof deezerFullTrack.DISK_NUMBER === 'number' - ? deezerFullTrack.DISK_NUMBER - : parseInt(deezerFullTrack.DISK_NUMBER, 10), - duration: - typeof deezerFullTrack.DURATION === 'number' - ? deezerFullTrack.DURATION - : parseInt(deezerFullTrack.DURATION, 10), - explicit: deezerFullTrack.EXPLICIT_LYRICS === 1, - md5Origin: deezerFullTrack.MD5_ORIGIN, - mediaVersion: deezerFullTrack.MEDIA_VERSION, - trackToken: deezerFullTrack.TRACK_TOKEN - }; - - // Check if track already exists (if overwrite is disabled) + // Check if track already exists locally if (!appSettings.deezerOverwrite && appSettings.musicFolder) { const exists = await trackExists(deezerTrack, appSettings.musicFolder, appSettings.deezerFormat); if (exists) { - console.log(`[SpotifyPlaylistDownloader] Skipping "${deezerTrack.title}" - already exists`); skippedCount++; - // Still add to successful tracks for m3u8 generation successfulTracks.push({ deezerTrack, spotifyTrack }); continue; } @@ -170,10 +194,6 @@ export async function downloadSpotifyPlaylist( queuedCount++; successfulTracks.push({ deezerTrack, spotifyTrack }); - - console.log( - `[SpotifyPlaylistDownloader] Queued: ${deezerTrack.title} (matched via ${conversionResult.matchMethod})` - ); } catch (error) { console.error(`[SpotifyPlaylistDownloader] Error processing track ${spotifyTrack.name}:`, error); failedCount++; @@ -181,7 +201,7 @@ export async function downloadSpotifyPlaylist( } console.log( - `[SpotifyPlaylistDownloader] Queued ${queuedCount} tracks, skipped ${skippedCount}, failed ${failedCount}` + `[SpotifyPlaylistDownloader] Queued ${queuedCount}, skipped ${skippedCount}, failed ${failedCount}, cached ${cachedCount}` ); // Show queue status message @@ -189,6 +209,7 @@ export async function downloadSpotifyPlaylist( const parts = [`Queued ${queuedCount} track${queuedCount !== 1 ? 's' : ''}`]; if (skippedCount > 0) parts.push(`${skippedCount} skipped`); if (failedCount > 0) parts.push(`${failedCount} not found`); + if (cachedCount > 0) parts.push(`${cachedCount} from cache`); setInfo(parts.join(', ')); } else if (skippedCount > 0) { setWarning(`All ${skippedCount} tracks already exist`); @@ -196,13 +217,10 @@ export async function downloadSpotifyPlaylist( setWarning(`Could not find ${failedCount} tracks on Deezer`); } - // Generate m3u8 file using Deezer track paths - const m3u8Tracks: M3U8Track[] = successfulTracks.map(({ deezerTrack, spotifyTrack }) => { - // Generate expected path for this Deezer track + // Generate m3u8 file + const m3u8Tracks: M3U8Track[] = successfulTracks.map(({ deezerTrack }) => { const paths = generateTrackPath(deezerTrack, musicFolder, appSettings.deezerFormat, false); const absolutePath = `${paths.filepath}/${paths.filename}`; - - // Convert to relative path from playlists folder const relativePath = makeRelativePath(absolutePath, 'Music'); return { @@ -213,12 +231,9 @@ export async function downloadSpotifyPlaylist( }; }); - // Write m3u8 file const m3u8Path = await writeM3U8(playlistName, m3u8Tracks, playlistsFolder); console.log(`[SpotifyPlaylistDownloader] Playlist saved to: ${m3u8Path}`); - - // Show success message for playlist creation setSuccess(`Playlist created: ${playlistName} (${successfulTracks.length} tracks)`); return { @@ -227,7 +242,8 @@ export async function downloadSpotifyPlaylist( total: spotifyTracks.length, queued: queuedCount, skipped: skippedCount, - failed: failedCount + failed: failedCount, + cached: cachedCount } }; }