markury/shark
1
0
Fork 0
mirror of https://github.com/markuryy/shark.git synced 2026-06-18 18:41:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(spotify): caching for Spotify to Deezer conversions to reduce API calls

Browse Source
This commit is contained in:
Markury
2026-03-18 11:08:08 -04:00
parent a846080677
commit 2c471370e4
3 changed files with 185 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
src-tauri/src/lib.rs
View File

@@ -299,10 +299,11 @@ pub fn run() {
kind: MigrationKind::Up,
}];
let spotify_migrations = vec![Migration {
version: 1,
description: "create_spotify_cache_tables",
sql: "
let spotify_migrations = vec![
Migration {
version: 1,
description: "create_spotify_cache_tables",
sql: "
CREATE TABLE IF NOT EXISTS spotify_playlists (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
@@ -364,8 +365,25 @@ pub fn run() {
CREATE INDEX IF NOT EXISTS idx_spotify_playlist_tracks_track ON spotify_playlist_tracks(track_id);
CREATE INDEX IF NOT EXISTS idx_spotify_playlist_tracks_isrc ON spotify_playlist_tracks(isrc);
",
kind: MigrationKind::Up,
}];
kind: MigrationKind::Up,
},
Migration {
version: 2,
description: "create_spotify_deezer_conversion_cache",
sql: "
CREATE TABLE IF NOT EXISTS spotify_deezer_conversions (
spotify_track_id TEXT PRIMARY KEY,
deezer_track_id INTEGER NOT NULL,
deezer_track_json TEXT NOT NULL,
match_method TEXT NOT NULL,
cached_at INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_conversions_deezer_id ON spotify_deezer_conversions(deezer_track_id);
",
kind: MigrationKind::Up,
},
];
tauri::Builder::default()
.plugin(tauri_plugin_oauth::init())

57
src/lib/library/spotify-database.ts
View File

@@ -318,6 +318,63 @@ export async function upsertPlaylistTracks(playlistId: string, tracks: any[]): P
}
}
/**
* Cached Spotify→Deezer conversion result
*/
export interface CachedConversion {
spotify_track_id: string;
deezer_track_id: number;
deezer_track_json: string;
match_method: string;
cached_at: number;
}
/**
* Get cached Spotify→Deezer conversions for a batch of track IDs
*/
export async function getCachedConversions(
spotifyTrackIds: string[]
): Promise<Map<string, CachedConversion>> {
if (spotifyTrackIds.length === 0) return new Map();
const database = await initSpotifyDatabase();
const map = new Map<string, CachedConversion>();
// Query in batches of 100 to avoid SQLite variable limits
for (let i = 0; i < spotifyTrackIds.length; i += 100) {
const batch = spotifyTrackIds.slice(i, i + 100);
const placeholders = batch.map((_, idx) => `$${idx + 1}`).join(',');
const rows = await database.select<CachedConversion[]>(
`SELECT * FROM spotify_deezer_conversions WHERE spotify_track_id IN (${placeholders})`,
batch
);
for (const row of rows) {
map.set(row.spotify_track_id, row);
}
}
return map;
}
/**
* Cache a Spotify→Deezer conversion result
*/
export async function cacheConversion(
spotifyTrackId: string,
deezerTrackId: number,
deezerTrackJson: string,
matchMethod: string
): Promise<void> {
const database = await initSpotifyDatabase();
const now = Math.floor(Date.now() / 1000);
await database.execute(
`INSERT OR REPLACE INTO spotify_deezer_conversions (spotify_track_id, deezer_track_id, deezer_track_json, match_method, cached_at)
VALUES ($1, $2, $3, $4, $5)`,
[spotifyTrackId, deezerTrackId, deezerTrackJson, matchMethod, now]
);
}
/**
* Clear all Spotify cache
*/

192
src/lib/services/spotify/playlistDownloader.ts
View File

@@ -1,5 +1,9 @@
/**
* Download Spotify playlist - converts tracks to Deezer via ISRC, adds to queue, and creates m3u8 file
*
* Uses a persistent SQLite cache for Spotify→Deezer conversions so that only
* new/uncached tracks require API calls. For a 200-track playlist where 3
* tracks were added, this reduces API calls from ~400 to ~6.
*/
import { addToQueue } from '$lib/stores/downloadQueue';
@@ -13,6 +17,7 @@ import { setInfo, setSuccess, setWarning } from '$lib/stores/status';
import { get } from 'svelte/store';
import { mkdir } from '@tauri-apps/plugin-fs';
import { convertSpotifyTrackToDeezer, type SpotifyTrackInput } from './converter';
import { getCachedConversions, cacheConversion } from '$lib/library/spotify-database';
import type { DeezerTrack } from '$lib/types/deezer';
export interface SpotifyPlaylistTrack {
@@ -26,16 +31,8 @@ export interface SpotifyPlaylistTrack {
}
/**
* Download a Spotify playlist by converting tracks to Deezer equivalents
* - Converts all tracks via ISRC matching
* - Adds converted tracks to the download queue (respects overwrite setting)
* - Creates an m3u8 playlist file with relative paths
*
* @param playlistName - Name of the playlist
* @param spotifyTracks - Array of Spotify track objects
* @param playlistsFolder - Path to playlists folder
* @param musicFolder - Path to music folder
* @returns Object with m3u8 path and statistics
* Download a Spotify playlist by converting tracks to Deezer equivalents.
* Cached conversions are reused — only uncached tracks hit the Deezer API.
*/
export async function downloadSpotifyPlaylist(
playlistName: string,
@@ -49,12 +46,12 @@ export async function downloadSpotifyPlaylist(
queued: number;
skipped: number;
failed: number;
cached: number;
};
}> {
const appSettings = get(settings);
const authState = get(deezerAuth);
// Ensure Deezer is authenticated
if (!authState.loggedIn || !authState.arl) {
throw new Error('Deezer authentication required for downloads');
}
@@ -64,95 +61,122 @@ export async function downloadSpotifyPlaylist(
console.log(`[SpotifyPlaylistDownloader] Starting download for playlist: ${playlistName}`);
console.log(`[SpotifyPlaylistDownloader] Tracks: ${spotifyTracks.length}`);
// Ensure playlists folder exists
try {
await mkdir(playlistsFolder, { recursive: true });
} catch (error) {
// Folder might already exist
}
// Track statistics
// --- Look up cached conversions in bulk ---
const spotifyIds = spotifyTracks.map((t) => t.track_id);
const cachedConversions = await getCachedConversions(spotifyIds);
const cachedCount = cachedConversions.size;
const uncachedCount = spotifyTracks.length - cachedCount;
console.log(
`[SpotifyPlaylistDownloader] Cache: ${cachedCount} cached, ${uncachedCount} need conversion`
);
let queuedCount = 0;
let skippedCount = 0;
let failedCount = 0;
// Track successful conversions for m3u8 generation
const successfulTracks: Array<{
deezerTrack: DeezerTrack;
spotifyTrack: SpotifyPlaylistTrack;
}> = [];
// Convert and queue each track
for (const spotifyTrack of spotifyTracks) {
try {
// Convert Spotify track to Deezer
const conversionInput: SpotifyTrackInput = {
id: spotifyTrack.track_id,
name: spotifyTrack.name,
artists: [spotifyTrack.artist_name],
album: spotifyTrack.album_name,
duration_ms: spotifyTrack.duration_ms,
isrc: spotifyTrack.isrc
};
let deezerTrack: DeezerTrack;
const conversionResult = await convertSpotifyTrackToDeezer(conversionInput);
// --- Check cache first ---
const cached = cachedConversions.get(spotifyTrack.track_id);
if (!conversionResult.success || !conversionResult.deezerTrack) {
console.warn(
`[SpotifyPlaylistDownloader] Failed to convert: ${spotifyTrack.name} - ${conversionResult.error}`
if (cached) {
// Use cached conversion — zero API calls
deezerTrack = JSON.parse(cached.deezer_track_json) as DeezerTrack;
} else {
// Convert via API (ISRC → metadata fallback)
const conversionInput: SpotifyTrackInput = {
id: spotifyTrack.track_id,
name: spotifyTrack.name,
artists: [spotifyTrack.artist_name],
album: spotifyTrack.album_name,
duration_ms: spotifyTrack.duration_ms,
isrc: spotifyTrack.isrc
};
const conversionResult = await convertSpotifyTrackToDeezer(conversionInput);
if (!conversionResult.success || !conversionResult.deezerTrack) {
console.warn(
`[SpotifyPlaylistDownloader] Failed to convert: ${spotifyTrack.name} - ${conversionResult.error}`
);
failedCount++;
continue;
}
const deezerPublicTrack = conversionResult.deezerTrack;
// Fetch full track data from GW API
const deezerTrackId = deezerPublicTrack.id.toString();
const deezerFullTrack = await deezerAPI.getTrack(deezerTrackId);
if (!deezerFullTrack || !deezerFullTrack.SNG_ID) {
console.warn(
`[SpotifyPlaylistDownloader] Could not fetch full Deezer track data for ID: ${deezerTrackId}`
);
failedCount++;
continue;
}
deezerTrack = {
id: parseInt(deezerFullTrack.SNG_ID, 10),
title: deezerFullTrack.SNG_TITLE,
artist: deezerFullTrack.ART_NAME,
artistId: parseInt(deezerFullTrack.ART_ID, 10),
artists: [deezerFullTrack.ART_NAME],
album: deezerFullTrack.ALB_TITLE,
albumId: parseInt(deezerFullTrack.ALB_ID, 10),
albumArtist: deezerFullTrack.ART_NAME,
albumArtistId: parseInt(deezerFullTrack.ART_ID, 10),
trackNumber:
typeof deezerFullTrack.TRACK_NUMBER === 'number'
? deezerFullTrack.TRACK_NUMBER
: parseInt(deezerFullTrack.TRACK_NUMBER, 10),
discNumber:
typeof deezerFullTrack.DISK_NUMBER === 'number'
? deezerFullTrack.DISK_NUMBER
: parseInt(deezerFullTrack.DISK_NUMBER, 10),
duration:
typeof deezerFullTrack.DURATION === 'number'
? deezerFullTrack.DURATION
: parseInt(deezerFullTrack.DURATION, 10),
explicit: deezerFullTrack.EXPLICIT_LYRICS === 1,
md5Origin: deezerFullTrack.MD5_ORIGIN,
mediaVersion: deezerFullTrack.MEDIA_VERSION,
trackToken: deezerFullTrack.TRACK_TOKEN
};
// Cache the conversion for future runs
await cacheConversion(
spotifyTrack.track_id,
deezerTrack.id,
JSON.stringify(deezerTrack),
conversionResult.matchMethod || 'unknown'
);
console.log(
`[SpotifyPlaylistDownloader] Converted & cached: ${deezerTrack.title} (via ${conversionResult.matchMethod})`
);
failedCount++;
continue;
}
const deezerPublicTrack = conversionResult.deezerTrack;
// Fetch full track data from Deezer GW API (needed for download)
const deezerTrackId = deezerPublicTrack.id.toString();
const deezerFullTrack = await deezerAPI.getTrack(deezerTrackId);
if (!deezerFullTrack || !deezerFullTrack.SNG_ID) {
console.warn(`[SpotifyPlaylistDownloader] Could not fetch full Deezer track data for ID: ${deezerTrackId}`);
failedCount++;
continue;
}
// Build DeezerTrack object
const deezerTrack: DeezerTrack = {
id: parseInt(deezerFullTrack.SNG_ID, 10),
title: deezerFullTrack.SNG_TITLE,
artist: deezerFullTrack.ART_NAME,
artistId: parseInt(deezerFullTrack.ART_ID, 10),
artists: [deezerFullTrack.ART_NAME],
album: deezerFullTrack.ALB_TITLE,
albumId: parseInt(deezerFullTrack.ALB_ID, 10),
albumArtist: deezerFullTrack.ART_NAME,
albumArtistId: parseInt(deezerFullTrack.ART_ID, 10),
trackNumber:
typeof deezerFullTrack.TRACK_NUMBER === 'number'
? deezerFullTrack.TRACK_NUMBER
: parseInt(deezerFullTrack.TRACK_NUMBER, 10),
discNumber:
typeof deezerFullTrack.DISK_NUMBER === 'number'
? deezerFullTrack.DISK_NUMBER
: parseInt(deezerFullTrack.DISK_NUMBER, 10),
duration:
typeof deezerFullTrack.DURATION === 'number'
? deezerFullTrack.DURATION
: parseInt(deezerFullTrack.DURATION, 10),
explicit: deezerFullTrack.EXPLICIT_LYRICS === 1,
md5Origin: deezerFullTrack.MD5_ORIGIN,
mediaVersion: deezerFullTrack.MEDIA_VERSION,
trackToken: deezerFullTrack.TRACK_TOKEN
};
// Check if track already exists (if overwrite is disabled)
// Check if track already exists locally
if (!appSettings.deezerOverwrite && appSettings.musicFolder) {
const exists = await trackExists(deezerTrack, appSettings.musicFolder, appSettings.deezerFormat);
if (exists) {
console.log(`[SpotifyPlaylistDownloader] Skipping "${deezerTrack.title}" - already exists`);
skippedCount++;
// Still add to successful tracks for m3u8 generation
successfulTracks.push({ deezerTrack, spotifyTrack });
continue;
}
@@ -170,10 +194,6 @@ export async function downloadSpotifyPlaylist(
queuedCount++;
successfulTracks.push({ deezerTrack, spotifyTrack });
console.log(
`[SpotifyPlaylistDownloader] Queued: ${deezerTrack.title} (matched via ${conversionResult.matchMethod})`
);
} catch (error) {
console.error(`[SpotifyPlaylistDownloader] Error processing track ${spotifyTrack.name}:`, error);
failedCount++;
@@ -181,7 +201,7 @@ export async function downloadSpotifyPlaylist(
}
console.log(
`[SpotifyPlaylistDownloader] Queued ${queuedCount} tracks, skipped ${skippedCount}, failed ${failedCount}`
`[SpotifyPlaylistDownloader] Queued ${queuedCount}, skipped ${skippedCount}, failed ${failedCount}, cached ${cachedCount}`
);
// Show queue status message
@@ -189,6 +209,7 @@ export async function downloadSpotifyPlaylist(
const parts = [`Queued ${queuedCount} track${queuedCount !== 1 ? 's' : ''}`];
if (skippedCount > 0) parts.push(`${skippedCount} skipped`);
if (failedCount > 0) parts.push(`${failedCount} not found`);
if (cachedCount > 0) parts.push(`${cachedCount} from cache`);
setInfo(parts.join(', '));
} else if (skippedCount > 0) {
setWarning(`All ${skippedCount} tracks already exist`);
@@ -196,13 +217,10 @@ export async function downloadSpotifyPlaylist(
setWarning(`Could not find ${failedCount} tracks on Deezer`);
}
// Generate m3u8 file using Deezer track paths
const m3u8Tracks: M3U8Track[] = successfulTracks.map(({ deezerTrack, spotifyTrack }) => {
// Generate expected path for this Deezer track
// Generate m3u8 file
const m3u8Tracks: M3U8Track[] = successfulTracks.map(({ deezerTrack }) => {
const paths = generateTrackPath(deezerTrack, musicFolder, appSettings.deezerFormat, false);
const absolutePath = `${paths.filepath}/${paths.filename}`;
// Convert to relative path from playlists folder
const relativePath = makeRelativePath(absolutePath, 'Music');
return {
@@ -213,12 +231,9 @@ export async function downloadSpotifyPlaylist(
};
});
// Write m3u8 file
const m3u8Path = await writeM3U8(playlistName, m3u8Tracks, playlistsFolder);
console.log(`[SpotifyPlaylistDownloader] Playlist saved to: ${m3u8Path}`);
// Show success message for playlist creation
setSuccess(`Playlist created: ${playlistName} (${successfulTracks.length} tracks)`);
return {
@@ -227,7 +242,8 @@ export async function downloadSpotifyPlaylist(
total: spotifyTracks.length,
queued: queuedCount,
skipped: skippedCount,
failed: failedCount
failed: failedCount,
cached: cachedCount
}
};
}