commit 81c8e17a8ed56a5fb3c84db591c5687793509e5c Author: Pastilhas Date: Thu Aug 20 02:18:14 2026 +0100 Start av-club: what it is, and a verified installer Media processing pulled out of the website that first needed it. ffmpeg, ImageMagick, libvips and exiftool are a real installation rather than an npm install, and encoder settings get revisited as browser support moves — neither of which belongs on the same clock as page templates and database schemas. No processing code yet. What is here is the tooling requirement, an installer, and docs/PIPELINE.md writing down what has NOT been decided — how a consumer talks to this, whether processing blocks an upload, where output goes — so those get chosen deliberately instead of by whatever gets written first. verify.sh checks by running: it encodes a video and probes the result against what ffmpeg was told to make, loads sharp under Bun specifically, and asserts a re-encode still strips GPS. That last one guards a real leak — if a future sharp changes its default, this fails a check rather than quietly publishing the coordinates of a client's living room. Confirmed non-vacuous by hiding ffmpeg and watching it exit 1. Co-Authored-By: Claude Opus 5 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f0dd57d --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +node_modules +dist +*.tgz +coverage +*.log +.cache +*.tsbuildinfo +.idea +.DS_Store + +.env +.env.local + +# scratch space for trying things against real files +tmp/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..f04e473 --- /dev/null +++ b/README.md @@ -0,0 +1,59 @@ +# av-club + +Turning whatever someone uploads into assets a website can actually serve. + +People upload photos and videos straight off a phone or a camera: a 12 MB HEIC, a 4K clip in a codec +Safari refuses to play, a JPEG carrying the GPS coordinates of the room it was shot in. None of that +belongs on a page as-is. This is the thing in between. + +## Status + +**Early.** Right now this repo contains the tooling requirements, an installer, and the +documentation of what the pipeline is meant to become. The processing code is not written yet. + +| | | +|---|---| +| `scripts/install.sh` | Brings a machine to the state this needs, then verifies it | +| `scripts/verify.sh` | Checks the tooling by running it — usable standalone | +| `docs/TOOLING.md` | What each tool is for, and the two things that bite | +| `docs/PIPELINE.md` | What the pipeline is supposed to do, and what is still undecided | + +## Quick start + +```bash +./scripts/install.sh # system packages + bun deps, then verifies +./scripts/verify.sh # verification on its own +``` + +Verified on Ubuntu 24.04. The installer says so rather than guessing if it finds something else. + +## Why this is its own repo + +**It has system dependencies a web app should not have to assume.** ffmpeg, ImageMagick, libvips and +exiftool are a real installation, not an `npm install`. Keeping them here means a consuming project +can state that it needs this, rather than quietly failing on a machine that happens not to have +`ffprobe` on its `PATH`. + +**It changes on a different clock.** Encoder settings, derivative sizes and format choices get +revisited as browser support moves. Page templates and database schemas do not. + +**Nothing about it is specific to one site.** Resizing an image and extracting a poster frame are the +same problem everywhere, which is why this is not buried inside the project that first needed it. + +## Its first consumer + +`fofos/platform` — a website with a public site, an admin, and a database. Admins upload media +through the admin; this processes it; the platform stores what comes back. + +**How that handoff works is not decided yet.** The honest options are a library the platform imports, +a CLI it shells out to, or a service it talks to. Each has real consequences for deployment and for +whether processing blocks an admin's upload while they wait. That question is written down in +`docs/PIPELINE.md` as open, rather than being answered by whatever happens to get built first. + +## Working agreements + +- **Commit and push every turn that changes anything** — working, broken, or half-finished. The + history is the artifact; it is how decisions stay traceable and mistakes stay learnable. +- **Docs are live documents, and the code is the source of truth.** Docs move with the code in the + same commit. A doc contradicting the code is a bug in the doc. +- **Direction changes fast, and that is expected.** Nothing written here is frozen. diff --git a/bun.lock b/bun.lock new file mode 100644 index 0000000..236c114 --- /dev/null +++ b/bun.lock @@ -0,0 +1,88 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "av-club", + "dependencies": { + "sharp": "^0.35.3", + }, + "devDependencies": { + "@types/bun": "latest", + }, + }, + }, + "packages": { + "@emnapi/runtime": ["@emnapi/runtime@1.11.3", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA=="], + + "@img/colour": ["@img/colour@1.1.0", "", {}, "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ=="], + + "@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.3.2" }, "os": "darwin", "cpu": "arm64" }, "sha512-RMnFX7YQsMoh7lWfcM4NEHHymBX/rLuKNPVM84XE9ONPcaSCDgE7CHIHpSgPcO2xcRthgBy1HfNO319mwhIAkg=="], + + "@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.3.2" }, "os": "darwin", "cpu": "x64" }, "sha512-Xo+5uFBtLN0BKqieTxiFzFPQAUlBbbH5iBKyRX/z1JrbnYsHTfKJnUfL8+p2TPXr1pXqao4eeL4Rl144uDpK9w=="], + + "@img/sharp-freebsd-wasm32": ["@img/sharp-freebsd-wasm32@0.35.3", "", { "dependencies": { "@img/sharp-wasm32": "0.35.3" }, "os": "freebsd" }, "sha512-lUxcqWIj2wMQ9BrwNjngcr1gWUr5xgaGThBRqPPalIC2n67Cqj1uPh8NnA/ZhAg8hUbKl+kVHKwgUIwe6ZYPrg=="], + + "@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.3.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-9J6ypZFpQBj4YnePGoq/S38w6nz+vqg5WZLrLGY4YuSemdMq47GMLBPO42MzwdGwpg/agZ7xzZcFHa48xlywfg=="], + + "@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.3.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-m2pW1n6cns9VaubNwsZ+c3CRYjxNQWgJ5gPlnL1nbBcpkBvFm6SCFN5o0psFHI8w9n11NKhFkeEDns98tiqbEw=="], + + "@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.3.2", "", { "os": "linux", "cpu": "arm" }, "sha512-1eMLzy92I4J6rmi4mAT8yC3HxOtniyGELlzGbNMLLeqe052ahFQ0h6LFq+lh5DsDIdYViIDst08abvSbcEdLXQ=="], + + "@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.3.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-dqVSFynCox4C/J8kT16V7SIFAns0IjgLwkvYT7p8LQVmJ5OS5b6tI9IGflxTeuBS//zXeFIUbwt5dwxyZ17cnA=="], + + "@img/sharp-libvips-linux-ppc64": ["@img/sharp-libvips-linux-ppc64@1.3.2", "", { "os": "linux", "cpu": "ppc64" }, "sha512-3z0NHDxD6n5I9gc05U1eW1AyRm+Gznzq3naMrthPNqE6oYykcogW0l/jfpJdjYnuNl8R7yI9pNbE1XiUeyq0Aw=="], + + "@img/sharp-libvips-linux-riscv64": ["@img/sharp-libvips-linux-riscv64@1.3.2", "", { "os": "linux", "cpu": "none" }, "sha512-bsb4rI+NldGOsXuej2r8OdSS8+zXDVaCWxyWrcv6kneTOlgAHtZABRzBBCwdsPiD90J4myNJuHpg6kA20ImW/w=="], + + "@img/sharp-libvips-linux-s390x": ["@img/sharp-libvips-linux-s390x@1.3.2", "", { "os": "linux", "cpu": "s390x" }, "sha512-/ABshyj8gCpyIrNXnHn4LorDJ0HHm1VhXPBlxZ8zAtfVPAaSafXPGn+sUSIRiwaSBy0mmFjSjiXI5mkcwdChKQ=="], + + "@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.3.2", "", { "os": "linux", "cpu": "x64" }, "sha512-ITPEtgffGJ0S6G9dRyw/366tJQqFRcHWPHhC+Stpg3Z8AEMrDrTr2lhdz4f/Y/HMbRh//7Z5mBzEpVdi62Oc3w=="], + + "@img/sharp-libvips-linuxmusl-arm64": ["@img/sharp-libvips-linuxmusl-arm64@1.3.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-zE9EdiUzUmg5mDT5a1rk5fYJ6GWPloTwWBYDS14naqHsL+EaMpDj1AWnpLgh3u0YCORv2Tt50wrcrpYqkP97Kw=="], + + "@img/sharp-libvips-linuxmusl-x64": ["@img/sharp-libvips-linuxmusl-x64@1.3.2", "", { "os": "linux", "cpu": "x64" }, "sha512-m0lrLiUt+lBYnCFr8qV/65yMR4E/c7/wf78I5eKTdkEakFAlZ9QlzEM3QIhhAwVeUhLAHLcCq7a7Vszq/oFNZQ=="], + + "@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.3.2" }, "os": "linux", "cpu": "arm" }, "sha512-affVWCTLooy8TSxbDx2qkzuDeaWLNVBA+P//FNBirHsXpP2fuBhk5AuboYUnrDnzoXes8GFjpTx0SBFOCRg+FA=="], + + "@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.3.2" }, "os": "linux", "cpu": "arm64" }, "sha512-QgKDspHPnrU+GQ55XPhGwyhC8acLVOOSyAvo1oVfFmrIXLkDNmGWzAfDZ4xK8oSA1qBQrALcHX0G5UZni/SuFQ=="], + + "@img/sharp-linux-ppc64": ["@img/sharp-linux-ppc64@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-linux-ppc64": "1.3.2" }, "os": "linux", "cpu": "ppc64" }, "sha512-sMd8rDxmpLOwv/7N44klFjOD5DUO7FLdjiXDI0hoxYaf7Ar262dQIEkosE98bps+5HPLtp/EvNqeqQtOycP/IA=="], + + "@img/sharp-linux-riscv64": ["@img/sharp-linux-riscv64@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-linux-riscv64": "1.3.2" }, "os": "linux", "cpu": "none" }, "sha512-0Eob78yjlYPfL5vMNWAW55l3R9Y6BQS/gOfe0ZcP9mEz9ohhKSt4im1hayiknXgf8AWrFqMvJcKIdmLmEe7yeQ=="], + + "@img/sharp-linux-s390x": ["@img/sharp-linux-s390x@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-linux-s390x": "1.3.2" }, "os": "linux", "cpu": "s390x" }, "sha512-KgAxQ0DxpNOq1rG2t5cgTgShJFGSuU7XO45cqC+1NVOuZnP6tlgZRuSYOfNupGkHID0o3cJOsw4DVeJpMovcGw=="], + + "@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.3.2" }, "os": "linux", "cpu": "x64" }, "sha512-8pqvxubL2PGdhlPy6GLqzDYMUjyRmKAwKHYKixpdJYBUK7PJ0C029XdsnpFIdgRZG68fZiGdHVWcKPvtiPB4cA=="], + + "@img/sharp-linuxmusl-arm64": ["@img/sharp-linuxmusl-arm64@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-arm64": "1.3.2" }, "os": "linux", "cpu": "arm64" }, "sha512-Vz0iQjzzcSX3HCbfwFfCSG/9SCIqyO0mH2sXyiHaAYfBk0cRsCWXRyQYX0ovCK/PAQBbTzQ0dsPQHh5MAFL59w=="], + + "@img/sharp-linuxmusl-x64": ["@img/sharp-linuxmusl-x64@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-x64": "1.3.2" }, "os": "linux", "cpu": "x64" }, "sha512-6O1NPKcDVj9QEdg7Hx549EX8U0rp6yXQERqru6yRN7fGBn32UvIRJUlWnk+8xDCiG76hXVBbX82NZ/ZKr0euIg=="], + + "@img/sharp-wasm32": ["@img/sharp-wasm32@0.35.3", "", { "dependencies": { "@emnapi/runtime": "^1.11.1" } }, "sha512-cZ0XkcYGpHZkqW6iCkqTcmUC0CD9DhD5d/qeZlZkfRBn6GnHniZXLUo5+9xw8Iv76YE6LQFN9YNBlKREcCG76w=="], + + "@img/sharp-webcontainers-wasm32": ["@img/sharp-webcontainers-wasm32@0.35.3", "", { "dependencies": { "@img/sharp-wasm32": "0.35.3" }, "cpu": "none" }, "sha512-2rnq7bX3NzeR2T4YWgz8qiG4h3TSdMe+vN1iQXpJleSJ3SM5zQ8Fy2SyyXAWlbxpEZ2Y+Z4u1BePgJEYbSy80Q=="], + + "@img/sharp-win32-arm64": ["@img/sharp-win32-arm64@0.35.3", "", { "os": "win32", "cpu": "arm64" }, "sha512-4bPwFdMbeC4JQ8L8LOyWp6nsHcboP5fxkp6iPOXz2Vg49R42TuMs2whkJ5OAP4/Ul035qOzy0AecOF9VOscn4w=="], + + "@img/sharp-win32-ia32": ["@img/sharp-win32-ia32@0.35.3", "", { "os": "win32", "cpu": "ia32" }, "sha512-r53mXsBN6lFUDiST764SvgwUdHAqM4rPAiDzAmf4fLoB6X/rkfyTrLCg6+g17wJJiCmB3JYgHuUldCWUIRFSXw=="], + + "@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.35.3", "", { "os": "win32", "cpu": "x64" }, "sha512-D4y1vNeZrIIJCN+uHaWVtH86B+aCrdMYYjicy9pXHvbGZeGYLLSd3wdVuC37FxVXlU1ARsk84eKWfWMXGYEqvA=="], + + "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], + + "@types/node": ["@types/node@26.2.0", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-5IviulTZeRNp2vAJ514cc/HUlY5nZ9fCbq9DMyC52BrhFZACo3nI0R7qBxhQmo/d27NFe96ur/b7Wwxklda+kg=="], + + "bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], + + "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], + + "semver": ["semver@7.8.5", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA=="], + + "sharp": ["sharp@0.35.3", "", { "dependencies": { "@img/colour": "^1.1.0", "detect-libc": "^2.1.2", "semver": "^7.8.5" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.35.3", "@img/sharp-darwin-x64": "0.35.3", "@img/sharp-freebsd-wasm32": "0.35.3", "@img/sharp-libvips-darwin-arm64": "1.3.2", "@img/sharp-libvips-darwin-x64": "1.3.2", "@img/sharp-libvips-linux-arm": "1.3.2", "@img/sharp-libvips-linux-arm64": "1.3.2", "@img/sharp-libvips-linux-ppc64": "1.3.2", "@img/sharp-libvips-linux-riscv64": "1.3.2", "@img/sharp-libvips-linux-s390x": "1.3.2", "@img/sharp-libvips-linux-x64": "1.3.2", "@img/sharp-libvips-linuxmusl-arm64": "1.3.2", "@img/sharp-libvips-linuxmusl-x64": "1.3.2", "@img/sharp-linux-arm": "0.35.3", "@img/sharp-linux-arm64": "0.35.3", "@img/sharp-linux-ppc64": "0.35.3", "@img/sharp-linux-riscv64": "0.35.3", "@img/sharp-linux-s390x": "0.35.3", "@img/sharp-linux-x64": "0.35.3", "@img/sharp-linuxmusl-arm64": "0.35.3", "@img/sharp-linuxmusl-x64": "0.35.3", "@img/sharp-webcontainers-wasm32": "0.35.3", "@img/sharp-win32-arm64": "0.35.3", "@img/sharp-win32-ia32": "0.35.3", "@img/sharp-win32-x64": "0.35.3" }, "peerDependencies": { "@types/node": "*" }, "optionalPeers": ["@types/node"] }, "sha512-ej0zVHuZGHCiABXcNxeYhpRnPNPAcvbG8RMdBAhDAxLKkCRVSpK3Iyu7qbqw3JMzoj0REeM6f3tJLtVwl0023Q=="], + + "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + + "undici-types": ["undici-types@8.3.0", "", {}, "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ=="], + } +} diff --git a/docs/PIPELINE.md b/docs/PIPELINE.md new file mode 100644 index 0000000..8d09f76 --- /dev/null +++ b/docs/PIPELINE.md @@ -0,0 +1,101 @@ +# The pipeline + +What av-club is meant to do, and — just as importantly — what has not been decided yet. + +**None of this is built.** This document exists so the decisions get made deliberately rather than +being settled by whatever gets written first. Where something is open, it says so. + +## The problem + +Someone uploads a file. It might be: + +- a 12 MB photo straight off a phone, 4000px wide, in HEIC +- a 4K video in a codec Safari will not play +- a JPEG carrying the GPS coordinates of the room it was taken in +- a file named `photo.jpg` that is actually a PNG, or a PDF, or nothing at all +- something enormous, uploaded by accident + +What a page needs is none of those: a few sizes in a modern format, a poster frame for video, +dimensions known ahead of time so layout does not shift, and nothing embedded that the person who +uploaded it did not mean to publish. + +## What it should do + +### 1. Identify, do not trust + +Establish what a file **actually is** by inspecting it, never from its extension or the +`Content-Type` a browser attached. A rejected upload should say something a non-technical person can +act on, not a MIME type. + +### 2. Strip what should not travel + +GPS coordinates, camera serial numbers, timestamps, editing history. `sharp` drops these by default +on re-encode — see `docs/TOOLING.md`. Video metadata needs the same treatment and is not free. + +### 3. Probe, and keep the numbers + +Width, height, duration, codec. These get stored, because knowing an image's aspect ratio before it +loads is what lets a page reserve the space and not shift as it arrives. + +### 4. Derive + +Images: a small number of widths, in a modern format, never upscaled past the original. +Video: something every browser plays, plus a poster frame. + +### 5. Name outputs so they can be cached forever + +Content-addressed or otherwise unique, so a derivative can be served `immutable` honestly and +replacing an image never serves a stale one from a cache. + +## What is NOT decided + +These are real forks. Picking one by accident is how a project ends up with an answer nobody chose. + +### How a consumer talks to this + +| Option | Consequence | +|---|---| +| **A library** the consumer imports | Simplest. Processing happens in the web server's process — a big video transcode blocks a worker and competes for CPU with page rendering. | +| **A CLI** the consumer shells out to | Clean isolation, no shared runtime. Every call pays process startup, and errors arrive as exit codes and stderr rather than as values. | +| **A service** the consumer calls | Scales and isolates properly. It is another thing to deploy, monitor and secure, which is real work for a small site. | + +### Whether processing is synchronous + +Does an admin wait for a 4K video to transcode while staring at a spinner, or does the upload return +immediately and the derivative appear later? + +Asynchronous is better for them and means a job queue, a way to show "still processing", and a +decision about what a page renders in the meantime. Synchronous is far simpler and is fine for +images; it is not fine for video. + +**These two questions are related** — a queue implies something long-lived, which pushes toward a +service. + +### Where output goes + +Not decided, and it interacts with the consumer's deployment: whether a deploy replaces the working +directory determines whether generated files can live inside it at all. This is currently an open +question in `fofos/platform` too. + +### The derivative ladder + +Which widths, which formats, what quality. Deliberately unanswered — it should follow from a real +design, and guessing produces sizes nobody uses and gaps where one was needed. + +### Externally hosted media + +A consumer may store a URL to a video hosted elsewhere rather than a file. Whether av-club has +anything to say about those — probing them, fetching poster frames, or ignoring them entirely — is +not decided. + +## Principles worth keeping + +**The original is never destroyed.** Derivatives are derivatives. A quality setting judged wrong in +six months should be re-runnable, and it cannot be if the source was thrown away. + +**Rejections explain themselves.** The person uploading is not a developer. "That file is 180 MB; +the limit is 50 MB" is usable. "Unsupported media type" is not. + +**Verify by running, not by asserting.** `scripts/verify.sh` is the existing example: it encodes a +video and reads it back rather than checking a version string. The same standard applies to the +processing code. diff --git a/docs/TOOLING.md b/docs/TOOLING.md new file mode 100644 index 0000000..8a741b2 --- /dev/null +++ b/docs/TOOLING.md @@ -0,0 +1,88 @@ +# Tooling + +What is installed to process images and video, what each piece is for, and the failures that look +like something else. + +## Bringing a machine up + +```bash +./scripts/install.sh +``` + +Verified on Ubuntu 24.04. What it does, if you would rather do it by hand: + +```bash +sudo apt-get install -y ffmpeg libimage-exiftool-perl imagemagick libvips-tools webp libavif-bin +bun install +./scripts/verify.sh +``` + +## What each is for + +| Tool | Verified at | Used for | +|---|---|---| +| `ffmpeg` / `ffprobe` | 6.1.1 | Video and audio. Transcoding, extracting poster frames, and probing a file for duration, codec and dimensions rather than trusting what a browser claimed about it. | +| `sharp` (bun dep) | 0.35.3, libvips 8.18.3 | **The image path the code actually calls.** In-process resize and re-encode to WebP/AVIF/JPEG, libvips-backed, so no process spawn per image. | +| `exiftool` | 12.76 | Reading and stripping metadata. Client photos routinely carry GPS coordinates and camera serial numbers. | +| `imagemagick` | 6.9.12 (`convert`, `identify`) | Inspecting a file by hand when something looks wrong, and formats sharp does not cover. | +| `libvips-tools` | 8.15.1 (`vipsthumbnail`) | The CLI side of the same library sharp uses — for comparing behaviour outside the app. | +| `webp` | 1.3.2 (`cwebp`, `dwebp`) | WebP at the command line. | +| `libavif-bin` | 1.0.4 (`avifenc`, `avifdec`) | AVIF at the command line. | + +The command-line encoders are for inspection and one-off work. Processing goes through `sharp` and +`ffmpeg`. + +## Two things that bite + +Both of these fail in a way that looks like a broken installation and is not. + +### There is no `magick` command + +Ubuntu 24.04 ships ImageMagick **6**, which provides `convert` and `identify`. The single `magick` +entrypoint is ImageMagick **7**. A script written against `magick` fails with "command not found" on +a machine that has ImageMagick fully installed and working. + +Use `convert` and `identify`, or install IM7 deliberately and write down that you did. + +### `sharp` must be resolved from the repo + +Run from anywhere else — `bun -e` in `/tmp`, a script executed from another directory — Bun resolves +`sharp` from its **global install cache** and lands on the `wasm32` build, which is missing +`@emnapi/runtime` and dies with: + +``` +TypeError: sharp.libvipsVersion is not a function +``` + +The native `linux-x64` binary is in this repo's `node_modules`; the process just has to be able to +see it. Anything using sharp runs with the repo as its working directory. `scripts/verify.sh` does +exactly this, and the reason is commented at the call site. + +## Metadata is stripped by default, and that is load-bearing + +A `sharp` re-encode drops EXIF unless `.withMetadata()` is called explicitly. + +Measured on a JPEG carrying injected GPS and camera tags: **34 tags before, 19 after**, and the 19 +survivors are filesystem and image-structure facts only — `ImageWidth`, `MIMEType`, `FileSize` and +similar. GPS latitude and longitude, `Make` and `Model` were all gone. + +So the default is the safe one, and **`.withMetadata()` is the call that publishes a client's +location.** Do not add it without a reason. `scripts/verify.sh` asserts this on every run, so if a +future sharp version changes the default, it fails a check rather than quietly publishing +coordinates. + +## Behaviour worth remembering + +**`sharp(...).resize(400)` enlarges a smaller source.** A 300×300 input comes out 400×400 and soft. +Anywhere a derivative size is requested will want `withoutEnlargement: true`, or small uploads get +upscaled into blur that nobody asked for. + +## Verifying + +```bash +./scripts/verify.sh +``` + +Every check runs the tool and compares the result against what was asked for — a version string +proves a binary exists, not that its encoders are present or that a native module loads. The script +exits non-zero on failure, so it is usable in CI. diff --git a/package.json b/package.json new file mode 100644 index 0000000..f6f1bf9 --- /dev/null +++ b/package.json @@ -0,0 +1,17 @@ +{ + "name": "av-club", + "version": "0.0.1", + "private": true, + "type": "module", + "description": "Turning whatever someone uploads into assets a website can actually serve", + "scripts": { + "install:tools": "./scripts/install.sh", + "verify": "./scripts/verify.sh" + }, + "dependencies": { + "sharp": "^0.35.3" + }, + "devDependencies": { + "@types/bun": "latest" + } +} diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100755 index 0000000..173e979 --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Bring a machine to the state av-club needs, then prove it by running verify.sh. +# +# Idempotent: apt-get install on an already-installed package is a no-op, so re-running this is safe +# and is the intended way to check a machine is still correct. + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +# Everything the pipeline shells out to. Kept as one list so the README, the docs and this script +# cannot disagree about what an install consists of. +PACKAGES=( + ffmpeg # ffmpeg + ffprobe: transcode, poster frames, probe duration/codec/size + libimage-exiftool-perl # exiftool: read and strip metadata — client photos carry GPS + imagemagick # convert + identify: odd formats, and inspecting a file by hand + libvips-tools # vipsthumbnail: the CLI side of the library sharp uses + webp # cwebp + dwebp + libavif-bin # avifenc + avifdec +) + +say() { printf '\n\033[1m%s\033[0m\n' "$1"; } +warn() { printf '\033[33m%s\033[0m\n' "$1"; } +die() { printf '\033[31m%s\033[0m\n' "$1" >&2; exit 1; } + +say "Checking the machine" + +[ -f /etc/os-release ] || die "No /etc/os-release — cannot tell what this machine is." +# shellcheck disable=SC1091 +. /etc/os-release + +echo " ${PRETTY_NAME:-unknown}" + +# Say so rather than guessing. The package names below are Debian/Ubuntu ones and would simply not +# exist elsewhere; failing here with a sentence beats failing later with "E: Unable to locate package". +if [ "${ID:-}" != "ubuntu" ] && [ "${ID_LIKE:-}" != *debian* ]; then + die "This installer only knows apt-based systems. Install these by hand: ${PACKAGES[*]}" +fi +if [ "${VERSION_ID:-}" != "24.04" ]; then + warn " Verified on Ubuntu 24.04; this is ${VERSION_ID:-unknown}. Continuing — check the output." +fi + +command -v bun >/dev/null 2>&1 || die "bun is not on PATH. See https://bun.sh" +echo " bun $(bun --version)" + +SUDO="" +if [ "$(id -u)" -ne 0 ]; then + command -v sudo >/dev/null 2>&1 || die "Not root and no sudo available." + SUDO="sudo" +fi + +say "Installing system packages" +echo " ${PACKAGES[*]}" +echo + +# noninteractive so a package with a debconf prompt cannot hang an unattended run. +$SUDO env DEBIAN_FRONTEND=noninteractive apt-get update -qq +$SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y "${PACKAGES[@]}" + +say "Installing bun dependencies" +(cd "$REPO_ROOT" && bun install) + +say "Verifying" +"$REPO_ROOT/scripts/verify.sh" diff --git a/scripts/verify.sh b/scripts/verify.sh new file mode 100755 index 0000000..e127665 --- /dev/null +++ b/scripts/verify.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +# Check the media tooling by RUNNING it, not by asking the package manager whether it is present. +# +# A package can be installed and still not work: sharp is a native module that can resolve to a +# broken wasm build, and a codec can be missing from an ffmpeg that reports a version happily. Every +# check here does real work and compares the result against what was asked for. +# +# Exits non-zero if anything fails, so it is usable in CI or as the last step of install.sh. + +set -uo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +FAILED=0 + +pass() { printf ' \033[32mok\033[0m %s\n' "$1"; } +fail() { printf ' \033[31mFAIL\033[0m %s\n' "$1"; FAILED=1; } +note() { printf ' \033[33m--\033[0m %s\n' "$1"; } + +echo +echo "Binaries on PATH" + +# `magick` is deliberately absent from this list: Ubuntu 24.04 ships ImageMagick 6, which provides +# convert/identify and no `magick`. See docs/TOOLING.md. +for cmd in ffmpeg ffprobe convert identify exiftool vipsthumbnail cwebp dwebp avifenc avifdec; do + if command -v "$cmd" >/dev/null 2>&1; then + pass "$cmd" + else + fail "$cmd is not on PATH" + fi +done + +echo +echo "ffmpeg encodes and ffprobe reads back what was asked for" + +if ffmpeg -hide_banner -loglevel error \ + -f lavfi -i testsrc=duration=3:size=1280x720:rate=30 \ + -c:v libx264 -pix_fmt yuv420p "$WORK/test.mp4" 2>"$WORK/ffmpeg.err"; then + + probe=$(ffprobe -v error \ + -show_entries stream=codec_name,width,height \ + -show_entries format=duration \ + -of default=noprint_wrappers=1 "$WORK/test.mp4" 2>/dev/null) + + # Compared against what ffmpeg was TOLD to produce. A version string proves nothing; this proves + # the h264 encoder is actually present and the container is readable. + for expect in "codec_name=h264" "width=1280" "height=720" "duration=3.000000"; do + if grep -qx "$expect" <<<"$probe"; then pass "$expect"; else fail "expected $expect"; fi + done +else + fail "ffmpeg could not encode h264: $(head -1 "$WORK/ffmpeg.err")" +fi + +echo +echo "ffmpeg extracts a poster frame" + +if ffmpeg -hide_banner -loglevel error -i "$WORK/test.mp4" -ss 1 -frames:v 1 "$WORK/poster.jpg" 2>/dev/null \ + && [ -s "$WORK/poster.jpg" ]; then + pass "poster.jpg $(identify -format '%wx%h' "$WORK/poster.jpg" 2>/dev/null)" +else + fail "could not extract a poster frame" +fi + +echo +echo "sharp runs under Bun" + +# Run with the REPO as cwd, deliberately. Resolved from anywhere else, Bun finds sharp in its global +# install cache and lands on the wasm32 build, which is missing @emnapi/runtime and dies with +# "sharp.libvipsVersion is not a function" — a broken-looking failure that is not a broken install. +if [ -d "$REPO_ROOT/node_modules/sharp" ]; then + if out=$(cd "$REPO_ROOT" && bun -e ' + import sharp from "sharp"; + const px = 64; + const src = await sharp({ create: { width: 300, height: 200, channels: 3, background: "#888" } }) + .jpeg().toBuffer(); + for (const fmt of ["webp", "avif", "jpeg"]) { + const buf = await sharp(src).resize(px)[fmt]().toBuffer(); + const m = await sharp(buf).metadata(); + if (m.width !== px) throw new Error(`${fmt}: expected ${px}px, got ${m.width}`); + } + console.log(`${sharp.versions.sharp} libvips ${sharp.versions.vips}`); + ' 2>&1); then + pass "sharp $out" + else + fail "sharp failed under Bun: $(tail -3 <<<"$out" | tr '\n' ' ')" + fi +else + note "sharp not installed — run 'bun install' in $REPO_ROOT" + FAILED=1 +fi + +echo +echo "exiftool reads metadata, and a sharp re-encode strips it" + +if [ -s "$WORK/poster.jpg" ]; then + exiftool -q -overwrite_original \ + -GPSLatitude=38.7223 -GPSLatitudeRef=N \ + -GPSLongitude=-9.1393 -GPSLongitudeRef=W \ + -Make="Apple" -Model="iPhone 15 Pro" "$WORK/poster.jpg" 2>/dev/null + + if exiftool -s3 -GPSLatitude "$WORK/poster.jpg" 2>/dev/null | grep -q .; then + pass "exiftool wrote and read back GPS tags" + + if [ -d "$REPO_ROOT/node_modules/sharp" ] \ + && (cd "$REPO_ROOT" && bun -e " + import sharp from 'sharp'; + await sharp('$WORK/poster.jpg').resize(300).jpeg().toFile('$WORK/stripped.jpg'); + " 2>/dev/null); then + + # The point of the check: sharp drops EXIF unless .withMetadata() is called. If that ever + # stops being true, client GPS coordinates start getting published and nothing else says so. + leaked=$(exiftool -s3 -GPSLatitude -GPSLongitude -Make -Model "$WORK/stripped.jpg" 2>/dev/null | grep -c .) + if [ "$leaked" -eq 0 ]; then + pass "GPS, Make and Model gone after re-encode" + else + fail "$leaked identifying tag(s) SURVIVED the re-encode" + fi + else + note "skipped strip check — sharp unavailable" + fi + else + fail "exiftool could not write test tags" + fi +else + note "skipped — no poster frame to test with" +fi + +echo +echo "Command-line encoders" + +cwebp -quiet "$WORK/poster.jpg" -o "$WORK/t.webp" 2>/dev/null && [ -s "$WORK/t.webp" ] \ + && pass "cwebp" || fail "cwebp" +avifenc -s 10 "$WORK/poster.jpg" "$WORK/t.avif" >/dev/null 2>&1 && [ -s "$WORK/t.avif" ] \ + && pass "avifenc" || fail "avifenc" +vipsthumbnail "$WORK/poster.jpg" -s 100 -o "$WORK/thumb-%s.jpg" >/dev/null 2>&1 \ + && [ -s "$WORK/thumb-poster.jpg" ] && pass "vipsthumbnail" || fail "vipsthumbnail" + +echo +if [ "$FAILED" -eq 0 ]; then + printf '\033[32mAll checks passed.\033[0m\n\n' +else + printf '\033[31mSomething failed above.\033[0m See docs/TOOLING.md.\n\n' +fi +exit "$FAILED" diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..129d371 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "lib": ["ESNext"], + "target": "ESNext", + "module": "Preserve", + "moduleDetection": "force", + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + + "strict": true, + "skipLibCheck": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedIndexedAccess": true, + "noImplicitOverride": true + } +}