Skip to content

wip #316

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
0xConstant1 wants to merge 4 commits into from
Closed

wip #316

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d3807f4
wip
Feb 15, 2026
df77dae
Merge branch 'dev' into merge
0xConstant1 Feb 15, 2026
65cb169
remove unused var
Feb 15, 2026
a9279f9
Merge branch 'merge' of https://github.com/0xConstant1/aiometadata in…
Feb 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
354 changes: 69 additions & 285 deletions addon/index.js
View file
Open in desktop

Large diffs are not rendered by default.

387 changes: 387 additions & 0 deletions addon/lib/catalogResolver.js
View file
Open in desktop

Large diffs are not rendered by default.

20 changes: 10 additions & 10 deletions addon/lib/comprehensiveCatalogWarmer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -601,18 +601,18 @@ class ComprehensiveCatalogWarmer {
const result = await cacheWrapCatalog(uuid, catalogKey, async () => {
// Check if this is a MAL catalog
if (catalogId.startsWith('mal.')) {
const configWithUUID = { ...config, userUUID: uuid };
const configWithUUID = { ...config, userUUID: uuid };
return await this.warmMALCatalog(catalogId, derivedPage, configWithUUID, extraArgs);
} else if (catalogId === 'tmdb.trending') {
// Special handling for tmdb.trending - call getTrending directly
if (!uuid) {
throw new Error(`UUID is required for catalog ${catalogId}`);
}
} else if (catalogId === 'tmdb.trending') {
// Special handling for tmdb.trending - call getTrending directly
if (!uuid) {
throw new Error(`UUID is required for catalog ${catalogId}`);
}
const configWithUUID = { ...config, userUUID: uuid };
const { getTrending } = require('./getTrending');
// getTrending is page-based; derive page from totalSeen/pageSize to keep behavior aligned
return await getTrending(catalog.type, config.language, derivedPage, extraArgs.genre || null, configWithUUID, uuid, true);
} else {
const { getTrending } = require('./getTrending');
// getTrending is page-based; derive page from totalSeen/pageSize to keep behavior aligned
return await getTrending(catalog.type, config.language, derivedPage, extraArgs.genre || null, configWithUUID, uuid, true);
} else {
// Everything else goes through getCatalog
if (!uuid) {
throw new Error(`UUID is required for catalog ${catalogId}`);
Expand Down
113 changes: 109 additions & 4 deletions addon/lib/configApi.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,109 @@ class ConfigApi {
return { cleaned: false };
}

sanitizeMergedCatalogs(config) {
if (!config || !Array.isArray(config.catalogs)) return;

const catalogs = config.catalogs;
const ensureMergedChildRestoreState = (catalog) => {
const current = catalog?.metadata?.mergedChildState;
if (
current &&
typeof current.showInHome === 'boolean' &&
typeof current.randomizePerPage === 'boolean'
) {
return;
}

catalog.metadata = {
...(catalog.metadata || {}),
mergedChildState: {
showInHome: !!catalog.showInHome,
randomizePerPage: !!catalog.randomizePerPage,
},
};
};
const restoreMergedChildState = (catalog) => {
const restoreState = catalog?.metadata?.mergedChildState;
if (restoreState && typeof restoreState.showInHome === 'boolean') {
catalog.showInHome = restoreState.showInHome;
}
if (restoreState && typeof restoreState.randomizePerPage === 'boolean') {
catalog.randomizePerPage = restoreState.randomizePerPage;
}
if (catalog.metadata && typeof catalog.metadata === 'object') {
delete catalog.metadata.mergedChildState;
if (Object.keys(catalog.metadata).length === 0) {
delete catalog.metadata;
}
}
delete catalog.mergedInto;
};
const catalogByKey = new Map(catalogs.map(c => [`${c.id}-${c.type}`, c]));
const validMergedParents = new Set();
const childrenByParent = new Map();

for (const catalog of catalogs) {
const isMergedParent = catalog?.source === 'merged' || String(catalog?.id || '').startsWith('merge.');
if (!isMergedParent) continue;

const mergedMeta = catalog?.metadata?.merged;
if (!mergedMeta || !Array.isArray(mergedMeta.children)) continue;
const allowMixedTypes = (catalog.type === 'series' || catalog.type === 'movie' || catalog.type === 'anime' || catalog.type === 'all') && mergedMeta.allowMixedTypes === true;

const seen = new Set();
const normalizedChildren = [];
for (const childRef of mergedMeta.children) {
const childKey = `${childRef?.id}-${childRef?.type}`;
if (!childRef?.id || !childRef?.type || seen.has(childKey)) continue;
if (childRef.id === catalog.id && childRef.type === catalog.type) continue;
const resolvedChild = catalogByKey.get(childKey);
if (!resolvedChild) continue;
const parentType = catalog.type;
if (allowMixedTypes) {
if (!(resolvedChild.type === 'movie' || resolvedChild.type === 'series' || resolvedChild.type === 'anime' || resolvedChild.type === 'all')) continue;
} else if (resolvedChild.type !== parentType) {
continue;
}
seen.add(childKey);
normalizedChildren.push({
id: resolvedChild.id,
type: resolvedChild.type,
...(typeof childRef.weight === 'number' ? { weight: childRef.weight } : {}),
});
}

mergedMeta.version = 1;
mergedMeta.children = normalizedChildren;
mergedMeta.strategy = mergedMeta.strategy === 'interleaved' ? 'interleaved' : 'sequential';
mergedMeta.genreMode = 'strict';
if (mergedMeta.dedupe === undefined) mergedMeta.dedupe = true;
mergedMeta.allowMixedTypes = allowMixedTypes;

if (normalizedChildren.length >= 2) {
validMergedParents.add(catalog.id);
childrenByParent.set(catalog.id, normalizedChildren);
}
}

for (const catalog of catalogs) {
if (catalog.mergedInto && !validMergedParents.has(catalog.mergedInto)) {
restoreMergedChildState(catalog);
}
}

for (const [parentId, children] of childrenByParent.entries()) {
for (const childRef of children) {
const child = catalogByKey.get(`${childRef.id}-${childRef.type}`);
if (!child) continue;
ensureMergedChildRestoreState(child);
child.mergedInto = parentId;
child.showInHome = false;
child.randomizePerPage = false;
}
}
}

// Validate required API keys
validateRequiredKeys(config) {
const requiredKeys = ['tmdb'];
Expand Down Expand Up @@ -158,6 +261,7 @@ class ConfigApi {
}

await this.sanitizeTraktToken(config);
this.sanitizeMergedCatalogs(config);

// Use existing UUID if provided, otherwise generate a new one
const userUUID = existingUUID || database.generateUserUUID();
Expand Down Expand Up @@ -229,7 +333,7 @@ class ConfigApi {

if (rpdbChanged || mdblistChanged) {
// RPDB/MDBList API key changes affect catalog rendering, clear user's catalogs
patterns.push(`catalog:${userUUID}:*`); // User-scoped catalog cache
patterns.push(`*catalog:${userUUID}:*`); // User-scoped catalog cache
logger.debug(`API keys changed - RPDB: ${rpdbChanged}, MDBList: ${mdblistChanged} - clearing user's catalog cache`);
}
}
Expand Down Expand Up @@ -330,7 +434,7 @@ class ConfigApi {
if (mdblistChanged) {
// Clear cache for specific MDBList catalogs that changed
for (const catalogId of changedCatalogs) {
const pattern = `catalog:${userUUID}:*${catalogId}*`;
const pattern = `*catalog:${userUUID}:*${catalogId}*`;
patterns.push(pattern);
logger.debug(`Added cache invalidation pattern for MDBList catalog: ${pattern}`);
}
Expand Down Expand Up @@ -501,6 +605,7 @@ class ConfigApi {
}

await this.sanitizeTraktToken(config);
this.sanitizeMergedCatalogs(config);

// Verify existing config exists
const existingConfig = await database.verifyUserAndGetConfig(userUUID, password);
Expand Down Expand Up @@ -571,7 +676,7 @@ class ConfigApi {
const mdblistChanged = config.apiKeys.mdblist !== oldConfig.apiKeys.mdblist;

if (rpdbChanged || mdblistChanged) {
patterns.push(`catalog:${userUUID}:*`); // User-scoped catalog cache
patterns.push(`*catalog:${userUUID}:*`); // User-scoped catalog cache
logger.debug(`API keys changed - RPDB: ${rpdbChanged}, MDBList: ${mdblistChanged} - clearing user's catalog cache`);
}
}
Expand Down Expand Up @@ -690,7 +795,7 @@ class ConfigApi {
// Clear each pattern
for (const pattern of patterns) {
try {
const deleted = await deleteKeysByPattern(`*:${userUUID}:${pattern}`);
const deleted = await deleteKeysByPattern(pattern);
if (deleted > 0) {
logger.debug(`Cleared ${deleted} cache entries for pattern: ${pattern}`);
}
Expand Down
114 changes: 109 additions & 5 deletions addon/lib/getCache.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ const { loadConfigFromDatabase } = require('./configApi');
const consola = require('consola');
const idMapper = require('./id-mapper');
const crypto = require('crypto');
const { getMergedChildAuthFingerprint } = require('./mergedAuthFingerprint');

// Helper to hash config
function hashConfig(configObj) {
Expand Down Expand Up @@ -146,6 +147,39 @@ function stableStringify(value) {
return '{' + keys.map(k => JSON.stringify(k) + ':' + stableStringify(value[k])).join(',') + '}';
}

function normalizeMergedCacheTtl(ttlValue, fallback) {
const ttl = Number(ttlValue);
if (Number.isFinite(ttl) && ttl > 0) return Math.floor(ttl);
const fallbackTtl = Number(fallback);
if (Number.isFinite(fallbackTtl) && fallbackTtl > 0) return Math.floor(fallbackTtl);
return CATALOG_TTL;
}

function getTtlEpoch(ttlSeconds, nowMs = Date.now()) {
const safeTtl = Math.max(1, Math.floor(ttlSeconds));
return Math.floor(nowMs / (safeTtl * 1000));
}

function buildMergedChildEpochSnapshot(mergedMetadata, catalogs, parentTtl, nowMs = Date.now()) {
const childEpochs = {
__parent: getTtlEpoch(parentTtl, nowMs)
};
const childTtls = [];

if (!mergedMetadata || !Array.isArray(mergedMetadata.children)) {
return { childEpochs, childTtls };
}

for (const child of mergedMetadata.children) {
const childCatalog = catalogs?.find(c => c.id === child.id && c.type === child.type);
const childTtl = normalizeMergedCacheTtl(childCatalog?.cacheTTL, parentTtl);
childTtls.push(childTtl);
childEpochs[`${child.id}-${child.type}`] = getTtlEpoch(childTtl, nowMs);
}

return { childEpochs, childTtls };
}

// Lightweight stable hash for short log signatures
function shortSignature(input) {
try {
Expand Down Expand Up @@ -768,6 +802,7 @@ async function cacheWrapCatalog(userUUID, catalogKey, method, options = {}) {
const isTraktCatalog = idOnly.startsWith('trakt.');
const isLetterboxdCatalog = idOnly.startsWith('letterboxd.');
const isStreamingCatalog = idOnly.startsWith('streaming.');
const isMergedCatalog = idOnly.startsWith('merge.');
const isTmdbDiscoverCatalog = idOnly.startsWith('tmdb.discover.');
const isTvdbDiscoverCatalog = idOnly.startsWith('tvdb.discover.');
const isAniListDiscoverCatalog = idOnly.startsWith('anilist.discover.');
Expand Down Expand Up @@ -809,6 +844,49 @@ async function cacheWrapCatalog(userUUID, catalogKey, method, options = {}) {
: (config.apiKeys?.rpdb || '')) : '',
usePosterProxy: !!config.usePosterProxy,
};

if (isMergedCatalog && catalogFromConfig?.metadata?.merged) {
const mergedMetadata = catalogFromConfig.metadata.merged;
const mergedParentTtl = normalizeMergedCacheTtl(catalogFromConfig?.cacheTTL, CATALOG_TTL);
const { childEpochs } = buildMergedChildEpochSnapshot(mergedMetadata, config.catalogs || [], mergedParentTtl);
const childSnapshots = Array.isArray(mergedMetadata.children)
? mergedMetadata.children.map(child => {
const childCatalog = config.catalogs?.find(
c => c.id === child.id && c.type === child.type
);
const authFingerprint = getMergedChildAuthFingerprint(childCatalog || child, config);
if (!childCatalog) {
return { id: child.id, type: child.type, missing: true, authFingerprint };
}
return {
id: childCatalog.id,
type: childCatalog.type,
source: childCatalog.source || null,
sourceUrl: childCatalog.sourceUrl || null,
pageSize: childCatalog.pageSize || null,
cacheTTL: childCatalog.cacheTTL || null,
sort: childCatalog.sort || null,
order: childCatalog.order || null,
sortDirection: childCatalog.sortDirection || null,
metadata: childCatalog.metadata || null,
enableRatingPosters: childCatalog.enableRatingPosters !== false,
randomizePerPage: !!childCatalog.randomizePerPage,
mergedInto: childCatalog.mergedInto || null,
displayType: childCatalog.displayType || null,
authFingerprint,
};
})
: [];

catalogConfig.merged = {
strategy: mergedMetadata.strategy || 'sequential',
genreMode: 'strict',
dedupe: mergedMetadata.dedupe !== false,
pageSize: mergedMetadata.pageSize || null,
children: childSnapshots,
childEpochs,
};
}

// Only include MDBList API key for MDBList catalogs
if (isMDBListCatalog) {
Expand Down Expand Up @@ -933,6 +1011,21 @@ async function cacheWrapCatalog(userUUID, catalogKey, method, options = {}) {
}
}

if (isMergedCatalog) {
const catalogConfig = config.catalogs?.find(c => c.id === idOnly && c.type === catalogType);
const mergedMeta = catalogConfig?.metadata?.merged;
const parentTtl = normalizeMergedCacheTtl(catalogConfig?.cacheTTL, cacheTTL);
let effectiveMergedTtl = parentTtl;
if (mergedMeta?.children && Array.isArray(mergedMeta.children)) {
const { childTtls } = buildMergedChildEpochSnapshot(mergedMeta, config.catalogs || [], parentTtl);
if (childTtls.length > 0) {
effectiveMergedTtl = Math.min(parentTtl, Math.min(...childTtls));
}
}
cacheTTL = Math.max(300, effectiveMergedTtl);
cacheLogger.debug(`[Catalog] Using effective cache TTL for merged catalog ${idOnly}: ${cacheTTL}s`);
}

// Use custom cache TTL for custom TMDB discover catalogs if specified
if (isDiscoverCatalog) {
const catalogConfig = config.catalogs?.find(c => c.id === idOnly);
Expand All @@ -958,7 +1051,7 @@ async function cacheWrapCatalog(userUUID, catalogKey, method, options = {}) {
const day = String(now.getDate()).padStart(2, '0');
const today = `${year}-${month}-${day}`; // YYYY-MM-DD format in local timezone
key = `catalog:${today}:${configHash}:${cacheTTL}:${catalogKey}`;
} else if (idOnly.startsWith('mdblist.') || idOnly.startsWith('trakt.') || idOnly.startsWith('simkl.watchlist.') || (idOnly.startsWith('anilist.') && idOnly !== 'anilist.trending') || idOnly.includes('stremthru.') || idOnly.startsWith('custom.') || idOnly.startsWith('letterboxd.') || isDiscoverCatalog) {
} else if (idOnly.startsWith('mdblist.') || idOnly.startsWith('trakt.') || idOnly.startsWith('simkl.watchlist.') || (idOnly.startsWith('anilist.') && idOnly !== 'anilist.trending') || idOnly.includes('stremthru.') || idOnly.startsWith('custom.') || idOnly.startsWith('letterboxd.') || isDiscoverCatalog || isMergedCatalog) {
key = `catalog:${userUUID}:${configHash}:${cacheTTL}:${catalogKey}`;
} else if (idOnly.startsWith('simkl.') || idOnly.startsWith('anilist.')) {
key = `catalog:${configHash}:${catalogKey}`;
Expand All @@ -968,12 +1061,18 @@ async function cacheWrapCatalog(userUUID, catalogKey, method, options = {}) {

const cacheKeyIdentifier = isAuthCatalog ? (config.sessionId || 'no-session') : (userUUID || '');
const catalogSig = shortSignature(`${cacheKeyIdentifier}|${idOnly}|${configHash}|ttl:${cacheTTL}`);
const isUserScopedCatalog = idOnly.startsWith('mdblist.') || idOnly.startsWith('trakt.') || idOnly.startsWith('simkl.watchlist.') || (idOnly.startsWith('anilist.') && idOnly !== 'anilist.trending') || idOnly.includes('stremthru.') || idOnly.startsWith('custom.') || idOnly.startsWith('letterboxd.') || isDiscoverCatalog || isAuthCatalog;
const isUserScopedCatalog = idOnly.startsWith('mdblist.') || idOnly.startsWith('trakt.') || idOnly.startsWith('simkl.watchlist.') || (idOnly.startsWith('anilist.') && idOnly !== 'anilist.trending') || idOnly.includes('stremthru.') || idOnly.startsWith('custom.') || idOnly.startsWith('letterboxd.') || isDiscoverCatalog || isAuthCatalog || isMergedCatalog;
cacheLogger.debug(`[Catalog] Key detail (${idOnly}) [sig:${catalogSig}] userScoped:${isUserScopedCatalog} ttl:${cacheTTL}s`);

// Set module-level context for this catalog request
// This allows reconstruction to access the correct RPDB state
currentRequestContext.catalogConfig = catalogFromConfig;
const contextCatalogConfig =
config?._currentCatalogConfig &&
config._currentCatalogConfig.id === idOnly &&
config._currentCatalogConfig.type === catalogType
? config._currentCatalogConfig
: catalogFromConfig;
currentRequestContext.catalogConfig = contextCatalogConfig;

try {
return await cacheWrap(key, method, cacheTTL, options);
Expand Down Expand Up @@ -1646,14 +1745,15 @@ async function reconstructMetaFromComponents(userUUID, metaId, ttl = META_TTL, o
}


const posterRatingEnabled = currentRequestContext.catalogConfig?.enableRatingPosters !== false;

// Add other components
availableComponents.forEach(({ componentName, data }) => {
if (componentName === 'basic') return; // Already handled

if (componentName === 'poster') {
// Apply poster rating logic during reconstruction if enabled
// Use module-level context to get accurate enablement state
const posterRatingEnabled = currentRequestContext.catalogConfig?.enableRatingPosters !== false;
const host = process.env.HOST_NAME.startsWith('http')
? process.env.HOST_NAME
: `https://${process.env.HOST_NAME}`;
Expand All @@ -1676,10 +1776,14 @@ async function reconstructMetaFromComponents(userUUID, metaId, ttl = META_TTL, o
if (isUpNextWithEpisodeThumbnail || hasEpisodeThumbnailShape) {
cacheLogger.debug(`[Reconstruct] Preserving cached episode thumbnail for ${metaId} (useShowPoster=${useShowPoster}, posterShape=${reconstructedMeta.posterShape}), poster: ${data.poster?.substring(0, 100)}...`);
}
reconstructedMeta.poster = data.poster;
// If rating posters are disabled, prefer the raw poster URL when available.
reconstructedMeta.poster = reconstructedMeta._rawPosterUrl || data.poster;
}
} else if (componentName === 'rawPoster') {
reconstructedMeta._rawPosterUrl = data._rawPosterUrl;
if (!posterRatingEnabled && data._rawPosterUrl) {
reconstructedMeta.poster = data._rawPosterUrl;
}
} else if (componentName === 'background') {
reconstructedMeta.background = data.background;
} else if (componentName === 'landscapePoster') {
Expand Down
Loading