From b7f91c11f398fe062ba7be4684c0cf149e196bd2 Mon Sep 17 00:00:00 2001 From: Christian Meffert Date: Sat, 1 Feb 2025 12:19:41 +0000 Subject: [PATCH 1/3] [db] DB migration to 22.03; new table files_metadata --- src/db.c | 239 +++++++++++++++++++++++++++++++++++++++++++---- src/db.h | 64 +++++++++++++ src/db_init.c | 22 +++++ src/db_init.h | 2 +- src/db_upgrade.c | 34 +++++++ 5 files changed, 344 insertions(+), 17 deletions(-) diff --git a/src/db.c b/src/db.c index 2b209b95f9..c3010f93f4 100644 --- a/src/db.c +++ b/src/db.c @@ -118,6 +118,8 @@ struct db_statements sqlite3_stmt *queue_items_insert; sqlite3_stmt *queue_items_update; + + sqlite3_stmt *files_metadata_insert; }; struct col_type_map { @@ -146,6 +148,7 @@ struct fixup_ctx }; struct query_clause { + char *join; char *where; char *group; char *having; @@ -155,6 +158,7 @@ struct query_clause { struct browse_clause { char *select; + char *from; char *where; char *group; }; @@ -301,6 +305,20 @@ static const struct col_type_map qi_cols_map[] = { "channels", qi_offsetof(channels), DB_TYPE_INT }, }; +/* This list must be kept in sync with + * - the order of the columns in the files_metadata table + * - the type and name of the fields in struct media_file_metadata_info + */ +static const struct col_type_map mfmi_cols_map[] = + { + { "file_id", mfmi_offsetof(file_id), DB_TYPE_INT, DB_FIXUP_STANDARD }, + { "songalbumid", mfmi_offsetof(songalbumid), DB_TYPE_INT64 }, + { "songartistid", mfmi_offsetof(songartistid), DB_TYPE_INT64 }, + { "metadata_kind", mfmi_offsetof(metadata_kind), DB_TYPE_INT }, + { "idx", mfmi_offsetof(idx), DB_TYPE_INT }, + { "value", mfmi_offsetof(value), DB_TYPE_STRING }, + }; + /* This list must be kept in sync with * - the order of the columns in the files table * - the name of the fields in struct db_media_file_info @@ -427,6 +445,21 @@ static const ssize_t dbgri_cols_map[] = dbgri_offsetof(seek), }; + +/* This list must be kept in sync with + * - the order of the columns in the files_metadata table + * - the name of the fields in struct db_media_file_metadata_info + */ +static const ssize_t dbmfmi_cols_map[] = + { + dbmfmi_offsetof(file_id), + dbmfmi_offsetof(songalbumid), + dbmfmi_offsetof(songartistid), + dbmfmi_offsetof(metadata_kind), + dbmfmi_offsetof(idx), + dbmfmi_offsetof(value), + }; + /* This list must be kept in sync with * - the order of fields in the Q_BROWSE_INFO query * - the name of the fields in struct db_browse_info @@ -514,6 +547,7 @@ static const char *sort_clause[] = "pos", "shuffle_pos", "f.date_released DESC, f.title_sort DESC", + "m.value", }; /* Browse clauses, used for SELECT, WHERE, GROUP BY and for default ORDER BY @@ -522,16 +556,18 @@ static const char *sort_clause[] = */ static const struct browse_clause browse_clause[] = { - { "", "", "" }, - { "f.album_artist, f.album_artist_sort", "f.album_artist", "f.album_artist_sort, f.album_artist" }, - { "f.album, f.album_sort", "f.album", "f.album_sort, f.album" }, - { "f.genre, f.genre", "f.genre", "f.genre" }, - { "f.composer, f.composer_sort", "f.composer", "f.composer_sort, f.composer" }, - { "f.year, f.year", "f.year", "f.year" }, - { "f.disc, f.disc", "f.disc", "f.disc" }, - { "f.track, f.track", "f.track", "f.track" }, - { "f.virtual_path, f.virtual_path", "f.virtual_path", "f.virtual_path" }, - { "f.path, f.path", "f.path", "f.path" }, + { "", "", "", "" }, + { "f.album_artist, f.album_artist_sort", "", "f.album_artist", "f.album_artist_sort, f.album_artist" }, + { "f.album, f.album_sort", "", "f.album", "f.album_sort, f.album" }, + { "f.genre, f.genre", "", "f.genre", "f.genre" }, + { "f.composer, f.composer_sort", "", "f.composer", "f.composer_sort, f.composer" }, + { "f.year, f.year", "", "f.year", "f.year" }, + { "f.disc, f.disc", "", "f.disc", "f.disc" }, + { "f.track, f.track", "", "f.track", "f.track" }, + { "f.virtual_path, f.virtual_path", "", "f.virtual_path", "f.virtual_path" }, + { "f.path, f.path", "", "f.path", "f.path" }, + { "m.value, m.value", "JOIN files_metadata m ON f.id = m.file_id", "m.metadata_kind = 1 AND m.value", "m.value" }, + { "m.value, m.value", "JOIN files_metadata m ON f.id = m.file_id", "m.metadata_kind = 5 AND m.value", "m.value" }, }; @@ -648,6 +684,28 @@ db_pl_type_label(enum pl_type pl_type) return NULL; } +/* Keep in sync with enum metadata_kind */ +static struct metadata_kind_info metadata_infos[] = + { + { "lyrics", 0 }, + { "genre", 1 }, + { "musicbrainz_albumid", 0 }, + { "musicbrainz_artistid", 0 }, + { "musicbrainz_albumartistid", 0 }, + { "composer", 1 }, + }; + +const struct metadata_kind_info * +db_metadata_kind_info_get(enum metadata_kind metadata_kind) +{ + if (metadata_kind < ARRAY_SIZE(metadata_infos)) + { + return &metadata_infos[metadata_kind]; + } + + return NULL; +} + /* Shuffle RNG state */ struct rng_ctx shuffle_rng; @@ -788,6 +846,36 @@ free_mfi(struct media_file_info *mfi, int content_only) memset(mfi, 0, sizeof(struct media_file_info)); } +void +free_mfmi(struct media_file_metadata_info *mfi_extra_md, int content_only) +{ + if (!mfi_extra_md) + return; + + free(mfi_extra_md->value); + + if (!content_only) + free(mfi_extra_md); + else + memset(mfi_extra_md, 0, sizeof(struct media_file_metadata_info)); +} + +void +free_mfmi_list(struct media_file_metadata_info **mfmi, int content_only) +{ + struct media_file_metadata_info *next; + + if (!mfmi) + return; + + while (*mfmi) + { + next = (*mfmi)->next; + free_mfmi(*mfmi, content_only); + *mfmi = next; + } +} + void free_pli(struct playlist_info *pli, int content_only) { @@ -846,6 +934,7 @@ free_query_params(struct query_params *qp, int content_only) free(qp->having); free(qp->order); free(qp->group); + free(qp->join); if (!content_only) free(qp); @@ -1435,6 +1524,12 @@ bind_qi(sqlite3_stmt *stmt, struct db_queue_item *qi) return bind_generic(stmt, qi, qi_cols_map, ARRAY_SIZE(qi_cols_map), qi->id); } +static int +bind_mfmi(sqlite3_stmt *stmt, struct media_file_metadata_info *mfmi) +{ + return bind_generic(stmt, mfmi, mfmi_cols_map, ARRAY_SIZE(mfmi_cols_map), 0); +} + /* Unlock notification support */ static void unlock_notify_cb(void **args, int nargs) @@ -1963,6 +2058,7 @@ db_free_query_clause(struct query_clause *qc) if (!qc) return; + sqlite3_free(qc->join); sqlite3_free(qc->where); sqlite3_free(qc->group); sqlite3_free(qc->having); @@ -2012,6 +2108,11 @@ db_build_query_clause(struct query_params *qp) else qc->order = sqlite3_mprintf(""); + if (qp->join) + qc->join = sqlite3_mprintf("%s", qp->join); + else + qc->join = sqlite3_mprintf(""); + switch (qp->idx_type) { case I_FIRST: @@ -2234,21 +2335,33 @@ db_build_query_plitems(struct query_params *qp, struct query_clause *qc) return query; } +static char * +db_build_query_file_metadata(struct query_params *qp, struct query_clause *qc) +{ + char *count; + char *query; + + count = sqlite3_mprintf("SELECT COUNT(*) FROM files_metadata fm JOIN files f ON fm.file_id = f.id %s AND fm.file_id = %d;", qc->where, qp->id); + query = sqlite3_mprintf("SELECT fm.* FROM files_metadata fm JOIN files f ON fm.file_id = f.id %s AND fm.file_id = %d ORDER BY fm.metadata_kind ASC, fm.idx ASC %s;", qc->where, qp->id, qc->index); + + return db_build_query_check(qp, count, query); +} + static char * db_build_query_group_albums(struct query_params *qp, struct query_clause *qc) { char *count; char *query; - count = sqlite3_mprintf("SELECT COUNT(DISTINCT f.songalbumid) FROM files f %s;", qc->where); + count = sqlite3_mprintf("SELECT COUNT(DISTINCT f.songalbumid) FROM files f %s %s;", qc->join, qc->where); query = sqlite3_mprintf("SELECT" \ " g.id, g.persistentid, f.album, f.album_sort, COUNT(f.id) AS track_count," \ " 1 AS album_count, f.album_artist, f.songartistid," \ " SUM(f.song_length) AS song_length, MIN(f.data_kind) AS data_kind, MIN(f.media_kind) AS media_kind," \ " MAX(f.year) AS year, MAX(f.date_released) AS date_released," \ " MAX(f.time_added) AS time_added, MAX(f.time_played) AS time_played, MAX(f.seek) AS seek " \ - "FROM files f JOIN groups g ON f.songalbumid = g.persistentid %s " \ - "GROUP BY f.songalbumid %s %s %s;", qc->where, qc->having, qc->order, qc->index); + "FROM files f JOIN groups g ON f.songalbumid = g.persistentid %s %s " \ + "GROUP BY f.songalbumid %s %s %s;", qc->join, qc->where, qc->having, qc->order, qc->index); return db_build_query_check(qp, count, query); } @@ -2339,20 +2452,22 @@ static char * db_build_query_browse(struct query_params *qp, struct query_clause *qc) { const char *where; + const char *from; const char *select; char *count; char *query; select = browse_clause[qp->type & ~Q_F_BROWSE].select; + from = browse_clause[qp->type & ~Q_F_BROWSE].from; where = browse_clause[qp->type & ~Q_F_BROWSE].where; - count = sqlite3_mprintf("SELECT COUNT(*) FROM (SELECT %s FROM files f %s AND %s != '' %s);", select, qc->where, where, qc->group); + count = sqlite3_mprintf("SELECT COUNT(*) FROM (SELECT %s FROM files f %s %s AND %s != '' %s);", select, from, qc->where, where, qc->group); query = sqlite3_mprintf("SELECT %s, COUNT(f.id) AS track_count, COUNT(DISTINCT f.songalbumid) AS album_count, COUNT(DISTINCT f.songartistid) AS artist_count," " SUM(f.song_length) AS song_length, MIN(f.data_kind) AS data_kind, MIN(f.media_kind) AS media_kind," " MAX(f.year) AS year, MAX(f.date_released) AS date_released," " MAX(f.time_added) AS time_added, MAX(f.time_played) AS time_played, MAX(f.seek) AS seek " - "FROM files f %s AND %s != '' %s %s %s;", - select, qc->where, where, qc->group, qc->order, qc->index); + "FROM files f %s %s AND %s != '' %s %s %s;", + select, from, qc->where, where, qc->group, qc->order, qc->index); return db_build_query_check(qp, count, query); } @@ -2404,6 +2519,10 @@ db_query_start(struct query_params *qp) query = db_build_query_plitems(qp, qc); break; + case Q_FILE_METADATA: + query = db_build_query_file_metadata(qp, qc); + break; + case Q_GROUP_ALBUMS: query = db_build_query_group_albums(qp, qc); break; @@ -2577,6 +2696,26 @@ db_query_fetch_file(struct db_media_file_info *dbmfi, struct query_params *qp) return ret; } +int +db_query_fetch_file_metadata(struct db_media_file_metadata_info *dbmfmi, struct query_params *qp) +{ + int ret; + + memset(dbmfmi, 0, sizeof(struct db_media_file_metadata_info)); + + if (qp->type != Q_FILE_METADATA) + { + DPRINTF(E_LOG, L_DB, "Not a file metadata query!\n"); + return -1; + } + + ret = db_query_fetch(dbmfmi, qp, dbmfmi_cols_map, ARRAY_SIZE(dbmfmi_cols_map)); + if (ret < 0) { + DPRINTF(E_LOG, L_DB, "Failed to fetch db_media_file_metadata_info\n"); + } + return ret; +} + int db_query_fetch_pl(struct db_playlist_info *dbpli, struct query_params *qp) { @@ -3597,6 +3736,71 @@ db_file_update_directoryid(const char *path, int dir_id) #undef Q_TMPL } +/* Files metadata */ + +int +db_file_metadata_add(int file_id, int64_t songalbumid, int64_t songartistid, struct media_file_metadata_info *mfmi) +{ + int ret = 0; + + if (file_id == 0) + { + DPRINTF(E_WARN, L_DB, "Trying to add file_metadata with zero file_id\n"); + return -1; + } + + mfmi->file_id = file_id; + mfmi->songalbumid = songalbumid; + mfmi->songartistid = songartistid; + + ret = bind_mfmi(db_statements.files_metadata_insert, mfmi); + if (ret < 0) + return -1; + + ret = db_statement_run(db_statements.files_metadata_insert, 0); + if (ret < 0) + return -1; + + return 0; +} + +int +db_file_metadata_add_all(int file_id, int64_t songalbumid, int64_t songartistid, struct media_file_metadata_info *mfmi) +{ + struct media_file_metadata_info *tmp; + int ret = 0; + + tmp = mfmi; + while (tmp && ret >= 0) + { + ret = db_file_metadata_add(file_id, songalbumid, songartistid, tmp); + if (ret < 0) + { + return -1; + } + tmp = tmp->next; + } + + library_update_trigger(LISTENER_DATABASE); + + return 0; +} + +void +db_file_metadata_clear(int file_id) +{ +#define Q_TMPL "DELETE FROM files_metadata WHERE file_id = %d;" + char *query; + + query = sqlite3_mprintf(Q_TMPL, file_id); + db_query_run(query, 1, LISTENER_DATABASE); + +#undef Q_TMPL +} + +/* File Metadata */ + + /* Playlists */ int @@ -7132,9 +7336,12 @@ db_statements_prepare(void) db_statements.queue_items_insert = db_statements_prepare_insert(qi_cols_map, ARRAY_SIZE(qi_cols_map), "queue"); db_statements.queue_items_update = db_statements_prepare_update(qi_cols_map, ARRAY_SIZE(qi_cols_map), "queue"); + db_statements.files_metadata_insert = db_statements_prepare_insert(mfmi_cols_map, ARRAY_SIZE(mfmi_cols_map), "files_metadata"); + if ( !db_statements.files_insert || !db_statements.files_update || !db_statements.files_ping || !db_statements.playlists_insert || !db_statements.playlists_update || !db_statements.queue_items_insert || !db_statements.queue_items_update + || !db_statements.files_metadata_insert ) return -1; diff --git a/src/db.h b/src/db.h index e024df689e..d517046a1a 100644 --- a/src/db.h +++ b/src/db.h @@ -32,6 +32,7 @@ enum sort_type { S_POS, S_SHUFFLE_POS, S_DATE_RELEASED, + S_MD_VALUE, }; #define Q_F_BROWSE (1 << 15) @@ -46,6 +47,7 @@ enum query_type { Q_GROUP_ITEMS = 7, Q_GROUP_DIRS = 8, Q_COUNT_ITEMS = 9, + Q_FILE_METADATA = 10, // Keep in sync with browse_clause[] Q_BROWSE_ARTISTS = Q_F_BROWSE | 1, @@ -57,6 +59,8 @@ enum query_type { Q_BROWSE_TRACKS = Q_F_BROWSE | 7, Q_BROWSE_VPATH = Q_F_BROWSE | 8, Q_BROWSE_PATH = Q_F_BROWSE | 9, + Q_BROWSE_GENRES_MD = Q_F_BROWSE | 10, + Q_BROWSE_COMPOSERS_MD = Q_F_BROWSE | 11, }; #define ARTWORK_UNKNOWN 0 @@ -92,6 +96,7 @@ struct query_params { char *having; char *order; char *group; + char *join; char *filter; @@ -253,6 +258,47 @@ struct media_file_info { #define mfi_offsetof(field) offsetof(struct media_file_info, field) +/* Keep in sync with metadata_infos */ +enum metadata_kind { + MD_LYRICS = 0, + MD_GENRE = 1, + MD_MUSICBRAINZ_ALBUMID = 2, + MD_MUSICBRAINZ_ARTISTID = 3, + MD_MUSICBRAINZ_ALBUMARTISTID = 4, + MD_COMPOSER = 5, +}; + +struct metadata_kind_info { + char *label; + int is_list; +}; + +const struct metadata_kind_info * +db_metadata_kind_info_get(enum metadata_kind metadata_kind); + +struct media_file_metadata_info { + uint32_t file_id; + int64_t songalbumid; + int64_t songartistid; + uint32_t idx; + enum metadata_kind metadata_kind; + char *value; + struct media_file_metadata_info *next; +}; + +#define mfmi_offsetof(field) offsetof(struct media_file_metadata_info, field) + +struct db_media_file_metadata_info { + char *file_id; + char *songalbumid; + char *songartistid; + char *idx; + char *metadata_kind; + char *value; +}; + +#define dbmfmi_offsetof(field) offsetof(struct db_media_file_metadata_info, field) + /* Keep in sync with pl_type_label[] */ /* PL_SPECIAL value must be in sync with type value in Q_PL* in db_init.c */ enum pl_type { @@ -588,6 +634,12 @@ free_pi(struct pairing_info *pi, int content_only); void free_mfi(struct media_file_info *mfi, int content_only); +void +free_mfmi(struct media_file_metadata_info *mfmi, int content_only); + +void +free_mfmi_list(struct media_file_metadata_info **mfmi, int content_only); + void free_pli(struct playlist_info *pli, int content_only); @@ -636,6 +688,9 @@ db_query_end(struct query_params *qp); int db_query_fetch_file(struct db_media_file_info *dbmfi, struct query_params *qp); +int +db_query_fetch_file_metadata(struct db_media_file_metadata_info *dbmfmi, struct query_params *qp); + int db_query_fetch_pl(struct db_playlist_info *dbpli, struct query_params *qp); @@ -973,6 +1028,15 @@ db_queue_get_count(uint32_t *nitems); int db_queue_get_pos(uint32_t item_id, char shuffle); +/* Files extra metadata */ + +int +db_file_metadata_add(int file_id, int64_t songalbumid, int64_t songartistid, struct media_file_metadata_info *mfmi); +int +db_file_metadata_add_all(int file_id, int64_t songalbumid, int64_t songartistid, struct media_file_metadata_info *mfmi); +void +db_file_metadata_clear(int file_id); + /* Inotify */ int db_watch_clear(void); diff --git a/src/db_init.c b/src/db_init.c index db1492a2d2..9663eda395 100644 --- a/src/db_init.c +++ b/src/db_init.c @@ -208,6 +208,16 @@ " channels INTEGER DEFAULT 0" \ ");" +#define T_FILES_METADATA \ + "CREATE TABLE IF NOT EXISTS files_metadata (" \ + " file_id INTEGER NOT NULL," \ + " songalbumid INTEGER NOT NULL," \ + " songartistid INTEGER NOT NULL," \ + " metadata_kind INTEGER NOT NULL," \ + " idx INTEGER DEFAULT 0," \ + " value TEXT NOT NULL COLLATE DAAP" \ + ");" + #define Q_PL1 \ "INSERT INTO playlists (id, title, type, query, db_timestamp, path, idx, special_id)" \ " VALUES(1, 'Library', 0, '1 = 1', 0, '', 0, 0);" @@ -277,6 +287,7 @@ static const struct db_init_query db_init_table_queries[] = { T_INOTIFY, "create table inotify" }, { T_DIRECTORIES, "create table directories" }, { T_QUEUE, "create table queue" }, + { T_FILES_METADATA, "create table files_metadata" }, { Q_PL1, "create default playlist" }, { Q_PL2, "create default smart playlist 'Music'" }, @@ -378,6 +389,15 @@ static const struct db_init_query db_init_table_queries[] = #define I_QUEUE_SHUFFLEPOS \ "CREATE INDEX IF NOT EXISTS idx_queue_shufflepos ON queue(shuffle_pos);" +#define I_MD_FILEID_TYPE_IDX \ + "CREATE INDEX IF NOT EXISTS idx_filesmd_fileid_type_idx ON files_metadata(file_id, metadata_kind, idx);" + +#define I_MD_ALBUMPERSID_TYPE_IDX \ + "CREATE INDEX IF NOT EXISTS idx_filesmd_albumid_type_idx ON files_metadata(songalbumid, metadata_kind, idx);" + +#define I_MD_ARTISTPERSID_TYPE_IDX \ + "CREATE INDEX IF NOT EXISTS idx_filesmd_artistid_type_idx ON files_metadata(songartistid, metadata_kind, idx);" + static const struct db_init_query db_init_index_queries[] = { { I_RESCAN, "create rescan index" }, @@ -412,6 +432,8 @@ static const struct db_init_query db_init_index_queries[] = { I_QUEUE_POS, "create queue pos index" }, { I_QUEUE_SHUFFLEPOS, "create queue shuffle pos index" }, + + { I_MD_FILEID_TYPE_IDX, "create files_metadata file_id type idx index" }, }; diff --git a/src/db_init.h b/src/db_init.h index 20052f1d12..d4dc8635d7 100644 --- a/src/db_init.h +++ b/src/db_init.h @@ -26,7 +26,7 @@ * is a major upgrade. In other words minor version upgrades permit downgrading * the server after the database was upgraded. */ #define SCHEMA_VERSION_MAJOR 22 -#define SCHEMA_VERSION_MINOR 2 +#define SCHEMA_VERSION_MINOR 3 int db_init_indices(sqlite3 *hdl); diff --git a/src/db_upgrade.c b/src/db_upgrade.c index efc1c74a9d..36c435c36b 100644 --- a/src/db_upgrade.c +++ b/src/db_upgrade.c @@ -1265,6 +1265,33 @@ static const struct db_upgrade_query db_upgrade_v2202_queries[] = }; +/* ---------------------------- 22.02 -> 22.03 ------------------------------ */ + + +#define U_V2203_TABLE_FILES_METADATA \ + "CREATE TABLE IF NOT EXISTS files_metadata (" \ + " file_id INTEGER NOT NULL," \ + " songalbumid INTEGER NOT NULL," \ + " songartistid INTEGER NOT NULL," \ + " metadata_kind INTEGER NOT NULL," \ + " idx INTEGER DEFAULT 0," \ + " value TEXT NOT NULL COLLATE DAAP" \ + ");" + +#define U_v2203_SCVER_MAJOR \ + "UPDATE admin SET value = '22' WHERE key = 'schema_version_major';" +#define U_v2203_SCVER_MINOR \ + "UPDATE admin SET value = '03' WHERE key = 'schema_version_minor';" + +static const struct db_upgrade_query db_upgrade_v2203_queries[] = + { + { U_V2203_TABLE_FILES_METADATA, "create table files_metadata" }, + + { U_v2203_SCVER_MAJOR, "set schema_version_major to 22" }, + { U_v2203_SCVER_MINOR, "set schema_version_minor to 03" }, + }; + + /* -------------------------- Main upgrade handler -------------------------- */ int @@ -1489,6 +1516,13 @@ db_upgrade(sqlite3 *hdl, int db_ver) if (ret < 0) return -1; + /* FALLTHROUGH */ + + case 2202: + ret = db_generic_upgrade(hdl, db_upgrade_v2203_queries, ARRAY_SIZE(db_upgrade_v2203_queries)); + if (ret < 0) + return -1; + /* Last case statement is the only one that ends with a break statement! */ break; From 422bb1ee6e8cede8935793cd75fe9c0c121f6e00 Mon Sep 17 00:00:00 2001 From: Christian Meffert Date: Sat, 1 Feb 2025 12:22:20 +0000 Subject: [PATCH 2/3] [scan] Scan lyrics, genre, composer, musicbrainz IDs into files_metadata for local files --- src/library.c | 22 +++++- src/library.h | 2 +- src/library/filescanner.c | 11 +-- src/library/filescanner.h | 2 +- src/library/filescanner_ffmpeg.c | 106 +++++++++++++++++++++++++++-- src/library/filescanner_playlist.c | 4 +- src/library/rssscanner.c | 2 +- src/library/spotify_webapi.c | 2 +- 8 files changed, 134 insertions(+), 17 deletions(-) diff --git a/src/library.c b/src/library.c index a21f22e595..5bc7d394c5 100644 --- a/src/library.c +++ b/src/library.c @@ -125,8 +125,9 @@ static struct library_callback_register library_cb_register[LIBRARY_MAX_CALLBACK /* ------------------- CALLED BY LIBRARY SOURCE MODULES -------------------- */ int -library_media_save(struct media_file_info *mfi) +library_media_save(struct media_file_info *mfi, struct media_file_metadata_info *mfmi) { + int file_id; int ret; if (!mfi->path || !mfi->fname || !mfi->scan_kind) @@ -144,9 +145,24 @@ library_media_save(struct media_file_info *mfi) } if (mfi->id == 0) - ret = db_file_add(mfi); + { + ret = db_file_add(mfi); + if (ret == 0 && mfmi) + { + file_id = db_file_id_bypath(mfi->path); + if (file_id > 0) + ret = db_file_metadata_add_all(file_id, mfi->songalbumid, mfi->songartistid, mfmi); + } + } else - ret = db_file_update(mfi); + { + ret = db_file_update(mfi); + if (ret == 0 && mfmi) + { + db_file_metadata_clear(mfi->id); + ret = db_file_metadata_add_all(mfi->id, mfi->songalbumid, mfi->songartistid, mfmi); + } + } return ret; } diff --git a/src/library.h b/src/library.h index 9bb09bd2b9..614514966d 100644 --- a/src/library.h +++ b/src/library.h @@ -138,7 +138,7 @@ struct library_source * @return 0 if operation succeeded, -1 on failure. */ int -library_media_save(struct media_file_info *mfi); +library_media_save(struct media_file_info *mfi, struct media_file_metadata_info *mfmi); /* * Adds a playlist if pli->id == 0, otherwise updates. diff --git a/src/library/filescanner.c b/src/library/filescanner.c index ffcf9e5952..2d79404621 100644 --- a/src/library/filescanner.c +++ b/src/library/filescanner.c @@ -569,6 +569,7 @@ process_regular_file(const char *file, struct stat *sb, int type, int flags, int { bool is_bulkscan = (flags & F_SCAN_BULK); struct media_file_info mfi; + struct media_file_metadata_info *mfmi = NULL; char virtual_path[PATH_MAX]; int ret; @@ -625,20 +626,22 @@ process_regular_file(const char *file, struct stat *sb, int type, int flags, int mfi.album_artist = safe_strdup(cfg_getstr(cfg_getsec(cfg, "library"), "compilation_artist")); } - ret = scan_metadata_ffmpeg(&mfi, file); + ret = scan_metadata_ffmpeg(&mfi, &mfmi, file); if (ret < 0) { free_mfi(&mfi, 1); + free_mfmi_list(&mfmi, 0); return; } } - library_media_save(&mfi); + library_media_save(&mfi, mfmi); cache_artwork_ping(file, sb->st_mtime, !is_bulkscan); // TODO [artworkcache] If entry in artwork cache exists for no artwork available, delete the entry if media file has embedded artwork free_mfi(&mfi, 1); + free_mfmi_list(&mfmi, 0); } /* Thread: scan */ @@ -2001,7 +2004,7 @@ playlist_add_files(FILE *fp, int pl_id, const char *virtual_path) memset(&mfi, 0, sizeof(struct media_file_info)); scan_metadata_stream(&mfi, path); - library_media_save(&mfi); + library_media_save(&mfi, NULL); free_mfi(&mfi, 1); ret = playlist_add_path(fp, pl_id, path); @@ -2159,7 +2162,7 @@ queue_save(const char *virtual_path) memset(&mfi, 0, sizeof(struct media_file_info)); scan_metadata_stream(&mfi, queue_item.path); - library_media_save(&mfi); + library_media_save(&mfi, NULL); free_mfi(&mfi, 1); } else diff --git a/src/library/filescanner.h b/src/library/filescanner.h index 52370c1e23..3f9fdc96f8 100644 --- a/src/library/filescanner.h +++ b/src/library/filescanner.h @@ -8,7 +8,7 @@ /* --------------------------- Actual scanners ---------------------------- */ int -scan_metadata_ffmpeg(struct media_file_info *mfi, const char *file); +scan_metadata_ffmpeg(struct media_file_info *mfi, struct media_file_metadata_info **mfmi, const char *file); void scan_metadata_stream(struct media_file_info *mfi, const char *path); diff --git a/src/library/filescanner_ffmpeg.c b/src/library/filescanner_ffmpeg.c index e7d440a589..81852b421e 100644 --- a/src/library/filescanner_ffmpeg.c +++ b/src/library/filescanner_ffmpeg.c @@ -62,6 +62,13 @@ struct metadata_map { int flags; }; +struct files_metadata_map { + char *key; + enum metadata_kind metadata_kind; + int (*handler_function)(struct media_file_metadata_info **, const char *); + int flags; +}; + // Used for passing errors to DPRINTF (can't count on av_err2str being present) static char errbuf[64]; @@ -323,6 +330,56 @@ static const struct metadata_map md_map_id3[] = { NULL, 0, 0, NULL } }; + +static int +parse_list(struct media_file_metadata_info **mfmi, enum metadata_kind md_kind, const char *val, const char *delim) +{ + char *str; + char *token; + char *ptr; + int idx = 0; + + str = strdup(val); + + token = strtok_r(str, delim, &ptr); + for (token = strtok_r(str, delim, &ptr); token; token = strtok_r(NULL, delim, &ptr)) + { + struct media_file_metadata_info *mfmi_new; + mfmi_new = calloc(1, sizeof(struct media_file_metadata_info)); + mfmi_new->metadata_kind = md_kind; + mfmi_new->value = strdup(token); + mfmi_new->idx = idx++; + mfmi_new->next = *mfmi; + *mfmi = mfmi_new; + } + + free(str); + return idx; +} + +static int +parse_genre_list(struct media_file_metadata_info **mfmi, const char *val) +{ + return parse_list(mfmi, MD_GENRE, val, ";/,"); +} + +static int +parse_composer_list(struct media_file_metadata_info **mfmi, const char *val) +{ + return parse_list(mfmi, MD_COMPOSER, val, ";/,"); +} + +struct files_metadata_map files_md_map[] = + { + { "genre", MD_GENRE, parse_genre_list, 0 }, + { "composer", MD_COMPOSER, parse_composer_list, 0 }, + { "lyrics", MD_LYRICS, NULL, AV_DICT_IGNORE_SUFFIX }, + { "MusicBrainz Album Id", MD_MUSICBRAINZ_ALBUMID, NULL, 0 }, + { "MusicBrainz Artist Id", MD_MUSICBRAINZ_ARTISTID, NULL, 0 }, + { "MusicBrainz Album Artist Id", MD_MUSICBRAINZ_ALBUMARTISTID, NULL, 0 }, + { NULL, 0, NULL, 0 } + }; + static int extract_metadata_from_dict(struct media_file_info *mfi, AVDictionary *md, const struct metadata_map *md_map) { @@ -379,7 +436,42 @@ extract_metadata_from_dict(struct media_file_info *mfi, AVDictionary *md, const } static int -extract_metadata(struct media_file_info *mfi, AVFormatContext *ctx, AVStream *audio_stream, AVStream *video_stream, const struct metadata_map *md_map) +extract_extra_metadata_from_dict(struct media_file_metadata_info **mfmi, AVDictionary *md, const struct files_metadata_map *md_map) +{ + AVDictionaryEntry *mdt; + char *strval; + struct media_file_metadata_info *mfmi_new; + int mdcount = 0; + int i; + + for (i = 0; md_map[i].key != NULL; i++) + { + mdt = av_dict_get(md, md_map[i].key, NULL, md_map[i].flags); + if (!mdt || !mdt->value || strlen(mdt->value) == 0) + continue; + + if (md_map[i].handler_function) + { + mdcount += md_map[i].handler_function(mfmi, mdt->value); + continue; + } + else + { + strval = strdup(mdt->value); + mfmi_new = calloc(1, sizeof(struct media_file_metadata_info)); + mfmi_new->metadata_kind = md_map[i].metadata_kind; + mfmi_new->value = strval; + mfmi_new->next = *mfmi; + *mfmi = mfmi_new; + mdcount++; + } + } + + return mdcount; +} + +static int +extract_metadata(struct media_file_info *mfi, struct media_file_metadata_info **mfmi, AVFormatContext *ctx, AVStream *audio_stream, AVStream *video_stream, const struct metadata_map *md_map) { int mdcount = 0; int ret; @@ -387,6 +479,8 @@ extract_metadata(struct media_file_info *mfi, AVFormatContext *ctx, AVStream *au if (ctx->metadata) { ret = extract_metadata_from_dict(mfi, ctx->metadata, md_map); + if (mfmi) + ret += extract_extra_metadata_from_dict(mfmi, ctx->metadata, files_md_map); mdcount += ret; DPRINTF(E_DBG, L_SCAN, "Picked up %d tags from file metadata\n", ret); @@ -395,6 +489,8 @@ extract_metadata(struct media_file_info *mfi, AVFormatContext *ctx, AVStream *au if (audio_stream->metadata) { ret = extract_metadata_from_dict(mfi, audio_stream->metadata, md_map); + if (mfmi) + ret += extract_extra_metadata_from_dict(mfmi, audio_stream->metadata, files_md_map); mdcount += ret; DPRINTF(E_DBG, L_SCAN, "Picked up %d tags from audio stream metadata\n", ret); @@ -403,6 +499,8 @@ extract_metadata(struct media_file_info *mfi, AVFormatContext *ctx, AVStream *au if (video_stream && video_stream->metadata) { ret = extract_metadata_from_dict(mfi, video_stream->metadata, md_map); + if (mfmi) + ret += extract_extra_metadata_from_dict(mfmi, video_stream->metadata, files_md_map); mdcount += ret; DPRINTF(E_DBG, L_SCAN, "Picked up %d tags from video stream metadata\n", ret); @@ -422,7 +520,7 @@ extract_metadata(struct media_file_info *mfi, AVFormatContext *ctx, AVStream *au * - fname: (filename) used as fallback for artist */ int -scan_metadata_ffmpeg(struct media_file_info *mfi, const char *file) +scan_metadata_ffmpeg(struct media_file_info *mfi, struct media_file_metadata_info **mfmi, const char *file) { AVFormatContext *ctx; AVDictionary *options; @@ -765,13 +863,13 @@ scan_metadata_ffmpeg(struct media_file_info *mfi, const char *file) if (extra_md_map) { - ret = extract_metadata(mfi, ctx, audio_stream, video_stream, extra_md_map); + ret = extract_metadata(mfi, NULL, ctx, audio_stream, video_stream, extra_md_map); mdcount += ret; DPRINTF(E_DBG, L_SCAN, "Picked up %d tags with extra md_map\n", ret); } - ret = extract_metadata(mfi, ctx, audio_stream, video_stream, md_map_generic); + ret = extract_metadata(mfi, mfmi, ctx, audio_stream, video_stream, md_map_generic); mdcount += ret; DPRINTF(E_DBG, L_SCAN, "Picked up %d tags with generic md_map, %d tags total\n", ret, mdcount); diff --git a/src/library/filescanner_playlist.c b/src/library/filescanner_playlist.c index 8fcb3c99bb..714d2af906 100644 --- a/src/library/filescanner_playlist.c +++ b/src/library/filescanner_playlist.c @@ -163,7 +163,7 @@ scan_metadata_stream(struct media_file_info *mfi, const char *path) mfi->directory_id = DIR_HTTP; mfi->scan_kind = SCAN_KIND_FILES; - ret = scan_metadata_ffmpeg(mfi, path); + ret = scan_metadata_ffmpeg(mfi, NULL, path); if (ret < 0) { DPRINTF(E_LOG, L_SCAN, "Playlist URL '%s' is unavailable for probe/metadata, assuming MP3 encoding\n", path); @@ -275,7 +275,7 @@ process_url(int pl_id, const char *path, struct media_file_info *mfi) else scan_metadata_stream(mfi, path); - ret = library_media_save(mfi); + ret = library_media_save(mfi, NULL); if (ret < 0) return -1; diff --git a/src/library/rssscanner.c b/src/library/rssscanner.c index 8191c7c921..b85a3605f5 100644 --- a/src/library/rssscanner.c +++ b/src/library/rssscanner.c @@ -449,7 +449,7 @@ rss_save(struct playlist_info *pli, int *count, enum rss_scan_type scan_type) mfi_metadata_fixup(&mfi, &ri, feed_title, feed_author, time_added); - library_media_save(&mfi); + library_media_save(&mfi, NULL); free_mfi(&mfi, 1); } diff --git a/src/library/spotify_webapi.c b/src/library/spotify_webapi.c index c028d7c44e..27c3abfec7 100644 --- a/src/library/spotify_webapi.c +++ b/src/library/spotify_webapi.c @@ -1462,7 +1462,7 @@ track_add(struct spotify_track *track, struct spotify_album *album, const char * map_track_to_mfi(&mfi, track, album, pl_name); - library_media_save(&mfi); + library_media_save(&mfi, NULL); free_mfi(&mfi, 1); } From 937184c961e329b03e1c70a5d167805e945ea76e Mon Sep 17 00:00:00 2001 From: Christian Meffert Date: Sat, 1 Feb 2025 12:23:43 +0000 Subject: [PATCH 3/3] [jsonapi] Use files_metadata for genre/composer; add new browse-albums endpoint; add new tracks-metadata endpoint --- src/httpd_jsonapi.c | 164 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 157 insertions(+), 7 deletions(-) diff --git a/src/httpd_jsonapi.c b/src/httpd_jsonapi.c index f7e349c2a1..f53a8be1c6 100644 --- a/src/httpd_jsonapi.c +++ b/src/httpd_jsonapi.c @@ -3348,6 +3348,83 @@ jsonapi_reply_library_tracks_put_byid(struct httpd_request *hreq) return HTTP_OK; } +static int +jsonapi_reply_library_track_metadata(struct httpd_request *hreq) +{ + const char *track_id; + struct query_params query_params = { 0 }; + struct db_media_file_metadata_info dbmfmi = { 0 }; + json_object *reply; + json_object *list; + int id; + int intval; + const struct metadata_kind_info *metadata_kind_info; + int ret = 0; + + if (!is_modified(hreq, DB_ADMIN_DB_UPDATE)) + return HTTP_NOTMODIFIED; + + track_id = hreq->path_parts[3]; + if (safe_atoi32(track_id, &id) < 0) + { + DPRINTF(E_LOG, L_WEB, "Error converting track id '%s' to int.\n", track_id); + return HTTP_INTERNAL; + } + + reply = json_object_new_object(); + + query_params.type = Q_FILE_METADATA; + query_params.id = id; + ret = db_query_start(&query_params); + if (ret < 0) + goto error; + + while (((ret = db_query_fetch_file_metadata(&dbmfmi, &query_params)) == 0) && (dbmfmi.file_id)) + { + ret = safe_atoi32(dbmfmi.metadata_kind, &intval); + if (ret == 0) + { + metadata_kind_info = db_metadata_kind_info_get(intval); + if (metadata_kind_info) + { + if (metadata_kind_info->is_list) + { + list = json_object_object_get(reply, metadata_kind_info->label); + if (!list) + { + list = json_object_new_array(); + json_object_object_add(reply, metadata_kind_info->label, list); + } + json_object_array_add(list, json_object_new_string(dbmfmi.value)); + } + else + { + json_object_object_add(reply, metadata_kind_info->label, json_object_new_string(dbmfmi.value)); + } + } + else + DPRINTF(E_LOG, L_WEB, "Error converting metadata kind '%s' to label.\n", dbmfmi.metadata_kind); + } + else + { + DPRINTF(E_LOG, L_WEB, "Error converting metadata kind '%s' to int.\n", dbmfmi.metadata_kind); + } + } + + ret = evbuffer_add_printf(hreq->out_body, "%s", json_object_to_json_string(reply)); + if (ret < 0) + DPRINTF(E_LOG, L_WEB, "browse: Couldn't add track metadata to response buffer.\n"); + + error: + db_query_end(&query_params); + jparse_free(reply); + + if (ret < 0) + return HTTP_INTERNAL; + + return HTTP_OK; +} + static int jsonapi_reply_library_track_playlists(struct httpd_request *hreq) { @@ -3822,8 +3899,8 @@ jsonapi_reply_library_browse(struct httpd_request *hreq) } else if (strcmp(browse_type, "composers") == 0) { - query_params.type = Q_BROWSE_COMPOSERS; - query_params.sort = S_COMPOSER; + query_params.type = Q_BROWSE_COMPOSERS_MD; + query_params.sort = S_MD_VALUE; query_params.idx_type = I_NONE; } else @@ -3885,16 +3962,16 @@ jsonapi_reply_library_browseitem(struct httpd_request *hreq) if (strcmp(browse_type, "genres") == 0) { query_params.type = Q_BROWSE_GENRES; - query_params.sort = S_GENRE; + query_params.sort = S_MD_VALUE; query_params.idx_type = I_NONE; - query_params.filter = db_mprintf("(f.genre = %Q)", item_name); + query_params.filter = db_mprintf("(m.value = %Q)", item_name); } else if (strcmp(browse_type, "composers") == 0) { - query_params.type = Q_BROWSE_COMPOSERS; - query_params.sort = S_COMPOSER; + query_params.type = Q_BROWSE_COMPOSERS_MD; + query_params.sort = S_MD_VALUE; query_params.idx_type = I_NONE; - query_params.filter = db_mprintf("(f.composer = %Q)", item_name); + query_params.filter = db_mprintf("(m.value = %Q)", item_name); } else { @@ -3931,6 +4008,77 @@ jsonapi_reply_library_browseitem(struct httpd_request *hreq) return HTTP_OK; } +static int +jsonapi_reply_library_browse_albums(struct httpd_request *hreq) +{ + struct query_params query_params; + const char *browse_type; + const char *item_name; + json_object *reply; + json_object *items; + int total; + int ret = 0; + + if (!is_modified(hreq, DB_ADMIN_DB_UPDATE)) + return HTTP_NOTMODIFIED; + + browse_type = hreq->path_parts[2]; + item_name = hreq->path_parts[3]; + DPRINTF(E_DBG, L_WEB, "Albums browse item query with type '%s' and item '%s'\n", browse_type, item_name); + + reply = json_object_new_object(); + items = json_object_new_array(); + json_object_object_add(reply, "items", items); + + memset(&query_params, 0, sizeof(struct query_params)); + + ret = query_params_limit_set(&query_params, hreq); + if (ret < 0) + goto error; + + query_params.type = Q_GROUP_ALBUMS; + query_params.sort = S_ALBUM; + + if (strcmp(browse_type, "genres") == 0) + { + query_params.join = db_mprintf("JOIN files_metadata m ON f.id = m.file_id AND m.metadata_kind = %d", MD_GENRE); + query_params.filter = db_mprintf("(m.value = %Q)", item_name); + } + else if (strcmp(browse_type, "composers") == 0) + { + query_params.join = db_mprintf("JOIN files_metadata m ON f.id = m.file_id AND m.metadata_kind = %d", MD_COMPOSER); + query_params.filter = db_mprintf("(m.value = %Q)", item_name); + } + else + { + DPRINTF(E_LOG, L_WEB, "Invalid browse type '%s'\n", browse_type); + goto error; + } + + ret = fetch_albums(&query_params, items, &total); + free(query_params.filter); + free(query_params.join); + + if (ret < 0) + goto error; + + json_object_object_add(reply, "total", json_object_new_int(total)); + json_object_object_add(reply, "offset", json_object_new_int(query_params.offset)); + json_object_object_add(reply, "limit", json_object_new_int(query_params.limit)); + + ret = evbuffer_add_printf(hreq->out_body, "%s", json_object_to_json_string(reply)); + if (ret < 0) + DPRINTF(E_LOG, L_WEB, "browse: Couldn't add albums to response buffer.\n"); + + error: + jparse_free(reply); + + if (ret < 0) + return HTTP_INTERNAL; + + return HTTP_OK; +} + static int jsonapi_reply_library_count(struct httpd_request *hreq) { @@ -4649,7 +4797,9 @@ static struct httpd_uri_map adm_handlers[] = { HTTPD_METHOD_GET, "^/api/library/tracks/[[:digit:]]+$", jsonapi_reply_library_tracks_get_byid }, { HTTPD_METHOD_PUT, "^/api/library/tracks/[[:digit:]]+$", jsonapi_reply_library_tracks_put_byid }, { HTTPD_METHOD_GET, "^/api/library/tracks/[[:digit:]]+/playlists$", jsonapi_reply_library_track_playlists }, + { HTTPD_METHOD_GET, "^/api/library/tracks/[[:digit:]]+/metadata$", jsonapi_reply_library_track_metadata }, { HTTPD_METHOD_GET, "^/api/library/(genres|composers)$", jsonapi_reply_library_browse }, + { HTTPD_METHOD_GET, "^/api/library/(genres|composers)/.*/albums$", jsonapi_reply_library_browse_albums }, { HTTPD_METHOD_GET, "^/api/library/(genres|composers)/.*$", jsonapi_reply_library_browseitem }, { HTTPD_METHOD_GET, "^/api/library/count$", jsonapi_reply_library_count }, { HTTPD_METHOD_GET, "^/api/library/files$", jsonapi_reply_library_files },