From ac4d6f4f823931d6ebfb91cec6123d347b1da0ed Mon Sep 17 00:00:00 2001 From: Steven Vandevelde Date: Thu, 29 May 2025 01:06:52 +0200 Subject: [PATCH] feat: improve metadata processing, don't download the entire track --- deno.lock | 2 + package-lock.json | 88 +++++++++++++++++-- package.json | 2 + src/pages/configurator/input/_applet.astro | 5 +- src/pages/core/types.d.ts | 4 + .../orchestrator/input-cache/_applet.astro | 40 +++++---- .../orchestrator/single-queue/_applet.astro | 4 +- .../processor/metadata-fetcher/_applet.astro | 57 +++++++++--- .../processor/metadata-fetcher/_manifest.json | 14 ++- 9 files changed, 174 insertions(+), 42 deletions(-) diff --git a/deno.lock b/deno.lock index ba8b56e..d9dd8da 100644 --- a/deno.lock +++ b/deno.lock @@ -24,6 +24,8 @@ "npm:@jsr/bradenmacdonald__s3-lite-client@0.9", "npm:@jsr/std__media-types@^1.1.0", "npm:@picocss/pico@^2.1.1", + "npm:@tokenizer/http@~0.9.2", + "npm:@tokenizer/range@0.13", "npm:@types/throttle-debounce@^5.0.2", "npm:astro-purgecss@^5.2.2", "npm:astro-scope@^3.0.1", diff --git a/package-lock.json b/package-lock.json index 1f16ad4..8fee151 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,12 +8,13 @@ "@bradenmacdonald/s3-lite-client": "npm:@jsr/bradenmacdonald__s3-lite-client@^0.9.0", "@picocss/pico": "^2.1.1", "@std/media-types": "npm:@jsr/std__media-types@^1.1.0", + "@tokenizer/http": "^0.9.2", + "@tokenizer/range": "^0.13.0", "@web-applets/sdk": "https://gitpkg.vercel.app/unternet-co/web-applets/sdk?tokono.ma/experiment&scripts.postinstall=npm%20i%20%40types%2Fnode%20%26%26%20npx%20tsc", "iconoir": "^7.11.0", "idb-keyval": "^6.2.1", "music-metadata": "^11.2.3", "native-file-system-adapter": "^3.0.1", - "node-s3-url-encode": "^0.0.4", "query-string": "^9.1.2", "spellcaster": "^6.0.0", "throttle-debounce": "^5.0.2", @@ -1703,6 +1704,68 @@ "tslib": "^2.8.0" } }, + "node_modules/@tokenizer/http": { + "version": "0.9.2", + "resolved": "https://registry.npmjs.org/@tokenizer/http/-/http-0.9.2.tgz", + "integrity": "sha512-rzJwHcqDjO3FdBPr+FK2R6dYE6Qbg6QZP7S47rhCEtG+/YqEFLqZ+gFCLcL8y5D39aYQB9vDssiwbsJlRLePPg==", + "license": "MIT", + "dependencies": { + "@tokenizer/range": "^0.12.0", + "debug": "^4.3.7", + "strtok3": "^10.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/@tokenizer/http/node_modules/@tokenizer/range": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@tokenizer/range/-/range-0.12.0.tgz", + "integrity": "sha512-xvJ1OflWjopkC5EgLge+9HrwsWStgVewQkmusoF2BxgCuGdm1KuhZAMVMNzC7h1WNei9JA6xKQlkbPNJtjZ6aw==", + "license": "MIT", + "dependencies": { + "debug": "^4.3.7", + "strtok3": "^9.1.1" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/@tokenizer/http/node_modules/@tokenizer/range/node_modules/strtok3": { + "version": "9.1.1", + "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-9.1.1.tgz", + "integrity": "sha512-FhwotcEqjr241ZbjFzjlIYg6c5/L/s4yBGWSMvJ9UoExiSqL+FnFA/CaeZx17WGaZMS/4SOZp8wH18jSS4R4lw==", + "license": "MIT", + "dependencies": { + "@tokenizer/token": "^0.3.0", + "peek-readable": "^5.3.1" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/@tokenizer/http/node_modules/peek-readable": { + "version": "5.4.2", + "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-5.4.2.tgz", + "integrity": "sha512-peBp3qZyuS6cNIJ2akRNG1uo1WJ1d0wTxg/fxMdZ0BqCVhx242bSFHM9eNqflfJVS9SsgkzgT/1UgnsurBOTMg==", + "license": "MIT", + "engines": { + "node": ">=14.16" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, "node_modules/@tokenizer/inflate": { "version": "0.2.7", "resolved": "https://registry.npmjs.org/@tokenizer/inflate/-/inflate-0.2.7.tgz", @@ -1721,6 +1784,23 @@ "url": "https://github.com/sponsors/Borewit" } }, + "node_modules/@tokenizer/range": { + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/@tokenizer/range/-/range-0.13.0.tgz", + "integrity": "sha512-ibLGQRU8an1g/y952+OxeZDGIj+W1HW8AQPtk26VIFWzy3tvQImmGBwYbpHJXMMAz1nhCPAAepCRptGKB8YrKg==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "strtok3": "^10.2.0" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, "node_modules/@tokenizer/token": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", @@ -5017,12 +5097,6 @@ "integrity": "sha512-0uGYQ1WQL1M5kKvGRXWQ3uZCHtLTO8hln3oBjIusM75WoesZ909uQJs/Hb946i2SS+Gsrhkaa6iAO17jRIv6DQ==", "dev": true }, - "node_modules/node-s3-url-encode": { - "version": "0.0.4", - "resolved": "https://registry.npmjs.org/node-s3-url-encode/-/node-s3-url-encode-0.0.4.tgz", - "integrity": "sha512-l0IizfnxE1hb9dadzYBpA27syfL9LFkPzCKH6YWrssv2sPLjVuCent67A8GPe4isdj4bEsbgdPWLTcV4gxEg9w==", - "license": "MIT" - }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", diff --git a/package.json b/package.json index 2f5bf39..8465a5e 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,8 @@ "@bradenmacdonald/s3-lite-client": "npm:@jsr/bradenmacdonald__s3-lite-client@^0.9.0", "@picocss/pico": "^2.1.1", "@std/media-types": "npm:@jsr/std__media-types@^1.1.0", + "@tokenizer/http": "^0.9.2", + "@tokenizer/range": "^0.13.0", "@web-applets/sdk": "https://gitpkg.vercel.app/unternet-co/web-applets/sdk?tokono.ma/experiment&scripts.postinstall=npm%20i%20%40types%2Fnode%20%26%26%20npx%20tsc", "iconoir": "^7.11.0", "idb-keyval": "^6.2.1", diff --git a/src/pages/configurator/input/_applet.astro b/src/pages/configurator/input/_applet.astro index ef03a0e..9424ac7 100644 --- a/src/pages/configurator/input/_applet.astro +++ b/src/pages/configurator/input/_applet.astro @@ -19,10 +19,7 @@

- More options coming soon!
S3-compatible APIs, Dropbox, etc. + More options coming soon!

diff --git a/src/pages/core/types.d.ts b/src/pages/core/types.d.ts index a8a200a..30857be 100644 --- a/src/pages/core/types.d.ts +++ b/src/pages/core/types.d.ts @@ -29,5 +29,9 @@ export interface TrackStats { export interface TrackTags { album?: string; artist?: string; + disc: { no: number; of?: number }; + genre?: string; title: string; + track: { no: number; of?: number }; + year?: number; } diff --git a/src/pages/orchestrator/input-cache/_applet.astro b/src/pages/orchestrator/input-cache/_applet.astro index 535c6ab..c8d8109 100644 --- a/src/pages/orchestrator/input-cache/_applet.astro +++ b/src/pages/orchestrator/input-cache/_applet.astro @@ -8,6 +8,7 @@ // SETUP //////////////////////////////////////////// const context = applets.register<{ ready: boolean }>(); + const topContext = self.top || self.parent; // Initial data context.data = { @@ -16,17 +17,19 @@ // Applet connections const configurator = { - input: await applet("../../configurator/input", { context: self.top || self.parent }), + input: await applet("../../configurator/input", { context: topContext }), }; const orchestrator = { output: await applet("../../orchestrator/output-management", { - context: self.parent, + context: topContext, }), }; const processor = { - metadataFetcher: await applet("../../processor/metadata-fetcher", { context: self.parent }), + metadataFetcher: await applet("../../processor/metadata-fetcher", { + context: topContext, + }), }; // 🚀 @@ -50,7 +53,7 @@ async (promise: Promise, track: Track) => { const acc = await promise; - if (track.tags) return [...acc, track]; + if (track.tags && track.stats) return [...acc, track]; const getURL = await configurator.input.sendAction( "resolve", @@ -60,22 +63,25 @@ }, ); - if (!getURL) return acc; + const headURL = await configurator.input.sendAction( + "resolve", + { method: "HEAD", uri: track.uri }, + { + timeoutDuration: 60000, + }, + ); - // TODO: Do we need to pass the HEAD URL too? - const meta = await processor.metadataFetcher.sendAction("extract", getURL, { - timeoutDuration: 60000, - }); + if (!getURL) return acc; - const stats: TrackStats = { - duration: meta.format.duration, - }; + const { stats, tags } = await processor.metadataFetcher.sendAction( + "extract", + { urls: { get: getURL, head: headURL || getURL } }, + { + timeoutDuration: 60000, + }, + ); - const tags: TrackTags = { - album: meta.common.album, - artist: meta.common.artist, - title: meta.common.title, - }; + console.log(stats, tags); return [...acc, { ...track, stats, tags }]; }, diff --git a/src/pages/orchestrator/single-queue/_applet.astro b/src/pages/orchestrator/single-queue/_applet.astro index 97cd4e9..e83ed87 100644 --- a/src/pages/orchestrator/single-queue/_applet.astro +++ b/src/pages/orchestrator/single-queue/_applet.astro @@ -24,7 +24,9 @@ }; const orchestrator = { - output: await applet("../../orchestrator/output-management", { context: self.parent }), + output: await applet("../../orchestrator/output-management", { + context: self.top || self.parent, + }), }; //////////////////////////////////////////// diff --git a/src/pages/processor/metadata-fetcher/_applet.astro b/src/pages/processor/metadata-fetcher/_applet.astro index 259999c..0ecbca8 100644 --- a/src/pages/processor/metadata-fetcher/_applet.astro +++ b/src/pages/processor/metadata-fetcher/_applet.astro @@ -1,29 +1,64 @@ diff --git a/src/pages/processor/metadata-fetcher/_manifest.json b/src/pages/processor/metadata-fetcher/_manifest.json index e97a1e0..891d14b 100644 --- a/src/pages/processor/metadata-fetcher/_manifest.json +++ b/src/pages/processor/metadata-fetcher/_manifest.json @@ -7,8 +7,18 @@ "title": "Extract", "description": "Get the metadata for a given URL.", "params_schema": { - "type": "string", - "description": "URL" + "type": "object", + "properties": { + "urls": { + "type": "object", + "properties": { + "get": { "type": "string" }, + "head": { "type": "string" } + }, + "required": ["get", "head"] + } + }, + "required": ["urls"] } } } -- knotmirror 0.0.1