312 lines
10 KiB
TypeScript
312 lines
10 KiB
TypeScript
|
// Daemon-side port of `scripts/sync-community-pets.ts`. Downloads pets
|
||
|
|
// from the public Codex Pet Share + j20 Hatchery catalogs into the
|
||
|
|
// `${CODEX_HOME:-$HOME/.codex}/pets/` registry that `codex-pets.ts`
|
||
|
|
// scans. Surfaced via `POST /api/codex-pets/sync` so the web Pet
|
||
|
|
// settings can offer a one-click refresh of the community catalog.
|
||
|
|
//
|
||
|
|
// Kept identical in spirit to the CLI script; tweaks here should be
|
||
|
|
// mirrored there (and vice versa) until both grow a shared package.
|
||
|
|
|
||
|
|
import { mkdir, stat, writeFile } from 'node:fs/promises';
|
||
|
|
import path from 'node:path';
|
||
|
|
|
||
|
|
import { resolveCodexPetsRoot } from './codex-pets.js';
|
||
|
|
|
||
|
|
const PETSHARE_BASE = 'https://ihzwckyzfcuktrljwpha.supabase.co/functions/v1/petshare';
|
||
|
|
const HATCHERY_LIST = 'https://j20.nz/hatchery/api/pets.json';
|
||
|
|
|
||
|
|
export interface SyncOptions {
|
||
|
|
// 'petshare' | 'hatchery' | 'all' — controls which catalogs we hit.
|
||
|
|
source?: 'petshare' | 'hatchery' | 'all';
|
||
|
|
// Re-download pets that already have a folder on disk.
|
||
|
|
force?: boolean;
|
||
|
|
// Cap the number of pets per source (handy for smoke tests).
|
||
|
|
limit?: number | null;
|
||
|
|
// Parallel downloads (defaults to 6).
|
||
|
|
concurrency?: number;
|
||
|
|
}
|
||
|
|
|
||
|
|
export interface SyncResult {
|
||
|
|
// How many pets were freshly written to disk.
|
||
|
|
wrote: number;
|
||
|
|
// Pets that already had a complete folder and were left alone.
|
||
|
|
skipped: number;
|
||
|
|
// Pets that errored during list / download / write.
|
||
|
|
failed: number;
|
||
|
|
// Total pets considered after de-duplication across catalogs.
|
||
|
|
total: number;
|
||
|
|
// Absolute path of the on-disk pet root we wrote into.
|
||
|
|
rootDir: string;
|
||
|
|
// Up to a handful of human-readable error messages — surfaced in the
|
||
|
|
// UI so users get actionable feedback when a transient catalog hiccup
|
||
|
|
// breaks an otherwise-good run.
|
||
|
|
errors: string[];
|
||
|
|
}
|
||
|
|
|
||
|
|
interface PetTask {
|
||
|
|
source: 'petshare' | 'hatchery';
|
||
|
|
folder: string;
|
||
|
|
manifest: Record<string, unknown>;
|
||
|
|
spritesheetUrl: string;
|
||
|
|
spritesheetExt: 'webp' | 'png' | 'gif';
|
||
|
|
}
|
||
|
|
|
||
|
|
interface PetShareItem {
|
||
|
|
id?: string;
|
||
|
|
displayName?: string;
|
||
|
|
description?: string;
|
||
|
|
spritesheetPath?: string;
|
||
|
|
spritesheetUrl?: string;
|
||
|
|
ownerName?: string;
|
||
|
|
tags?: string[];
|
||
|
|
}
|
||
|
|
|
||
|
|
interface PetShareListResponse {
|
||
|
|
pets?: PetShareItem[];
|
||
|
|
totalPages?: number;
|
||
|
|
}
|
||
|
|
|
||
|
|
interface HatcheryItem {
|
||
|
|
id?: string;
|
||
|
|
petManifestId?: string;
|
||
|
|
displayName?: string;
|
||
|
|
description?: string;
|
||
|
|
spritesheetUrl?: string;
|
||
|
|
authorLabel?: string;
|
||
|
|
authorXUrl?: string;
|
||
|
|
galleryUrl?: string;
|
||
|
|
}
|
||
|
|
|
||
|
|
interface HatcheryListResponse {
|
||
|
|
pets?: HatcheryItem[];
|
||
|
|
}
|
||
|
|
|
||
|
|
function sanitizeFolder(value: unknown): string {
|
||
|
|
return String(value ?? '')
|
||
|
|
.toLowerCase()
|
||
|
|
.replace(/[^a-z0-9._-]+/g, '-')
|
||
|
|
.replace(/-+/g, '-')
|
||
|
|
.replace(/^[._-]+|[._-]+$/g, '')
|
||
|
|
.slice(0, 80);
|
||
|
|
}
|
||
|
|
|
||
|
|
function extOf(url: string | undefined): 'webp' | 'png' | 'gif' {
|
||
|
|
const clean = (url || '').split('?')[0] ?? '';
|
||
|
|
const ext = clean.split('.').pop()?.toLowerCase() ?? 'webp';
|
||
|
|
if (ext === 'webp' || ext === 'png' || ext === 'gif') return ext;
|
||
|
|
return 'webp';
|
||
|
|
}
|
||
|
|
|
||
|
|
async function pathExists(p: string): Promise<boolean> {
|
||
|
|
try {
|
||
|
|
await stat(p);
|
||
|
|
return true;
|
||
|
|
} catch {
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
async function listPetSharePets(limit: number | null): Promise<PetTask[]> {
|
||
|
|
const tasks: PetTask[] = [];
|
||
|
|
let page = 1;
|
||
|
|
const pageSize = 24;
|
||
|
|
for (;;) {
|
||
|
|
const url = `${PETSHARE_BASE}/api/pets?page=${page}&pageSize=${pageSize}`;
|
||
|
|
const resp = await fetch(url);
|
||
|
|
if (!resp.ok) {
|
||
|
|
throw new Error(`petshare list page ${page} failed: ${resp.status} ${resp.statusText}`);
|
||
|
|
}
|
||
|
|
const data = (await resp.json()) as PetShareListResponse;
|
||
|
|
for (const pet of data.pets ?? []) {
|
||
|
|
const folder = sanitizeFolder(pet.id);
|
||
|
|
if (!folder) continue;
|
||
|
|
const spritesheetUrl = pet.spritesheetUrl?.startsWith('http')
|
||
|
|
? pet.spritesheetUrl
|
||
|
|
: `${PETSHARE_BASE}${pet.spritesheetUrl ?? ''}`;
|
||
|
|
const ext = extOf(pet.spritesheetPath ?? spritesheetUrl);
|
||
|
|
tasks.push({
|
||
|
|
source: 'petshare',
|
||
|
|
folder,
|
||
|
|
manifest: {
|
||
|
|
id: pet.id,
|
||
|
|
displayName: pet.displayName,
|
||
|
|
description: pet.description ?? '',
|
||
|
|
spritesheetPath: `spritesheet.${ext}`,
|
||
|
|
author: pet.ownerName,
|
||
|
|
tags: pet.tags ?? [],
|
||
|
|
source: 'codex-pet-share',
|
||
|
|
sourceUrl: `https://codex-pet-share.pages.dev/#/pets/${encodeURIComponent(pet.id ?? '')}`,
|
||
|
|
},
|
||
|
|
spritesheetUrl,
|
||
|
|
spritesheetExt: ext,
|
||
|
|
});
|
||
|
|
if (limit && tasks.length >= limit) return tasks;
|
||
|
|
}
|
||
|
|
if (page >= (data.totalPages ?? 1)) break;
|
||
|
|
page++;
|
||
|
|
}
|
||
|
|
return tasks;
|
||
|
|
}
|
||
|
|
|
||
|
|
async function listHatcheryPets(limit: number | null): Promise<PetTask[]> {
|
||
|
|
const resp = await fetch(HATCHERY_LIST);
|
||
|
|
if (!resp.ok) {
|
||
|
|
throw new Error(`hatchery list failed: ${resp.status} ${resp.statusText}`);
|
||
|
|
}
|
||
|
|
const data = (await resp.json()) as HatcheryListResponse;
|
||
|
|
const tasks: PetTask[] = [];
|
||
|
|
for (const pet of data.pets ?? []) {
|
||
|
|
const folder = sanitizeFolder(pet.petManifestId || pet.id);
|
||
|
|
if (!folder) continue;
|
||
|
|
if (!pet.spritesheetUrl) continue;
|
||
|
|
tasks.push({
|
||
|
|
source: 'hatchery',
|
||
|
|
folder,
|
||
|
|
manifest: {
|
||
|
|
id: pet.petManifestId || pet.id,
|
||
|
|
displayName: pet.displayName,
|
||
|
|
description: pet.description ?? '',
|
||
|
|
spritesheetPath: 'spritesheet.webp',
|
||
|
|
author: pet.authorLabel,
|
||
|
|
authorXUrl: pet.authorXUrl,
|
||
|
|
source: 'j20-hatchery',
|
||
|
|
sourceUrl: pet.galleryUrl,
|
||
|
|
},
|
||
|
|
spritesheetUrl: pet.spritesheetUrl,
|
||
|
|
spritesheetExt: extOf(pet.spritesheetUrl),
|
||
|
|
});
|
||
|
|
if (limit && tasks.length >= limit) break;
|
||
|
|
}
|
||
|
|
return tasks;
|
||
|
|
}
|
||
|
|
|
||
|
|
async function downloadBinary(url: string): Promise<Buffer> {
|
||
|
|
const resp = await fetch(url);
|
||
|
|
if (!resp.ok) {
|
||
|
|
throw new Error(`download ${url} failed: ${resp.status} ${resp.statusText}`);
|
||
|
|
}
|
||
|
|
const ab = await resp.arrayBuffer();
|
||
|
|
return Buffer.from(ab);
|
||
|
|
}
|
||
|
|
|
||
|
|
async function writePet(
|
||
|
|
task: PetTask,
|
||
|
|
outRoot: string,
|
||
|
|
force: boolean,
|
||
|
|
): Promise<'wrote' | 'skipped'> {
|
||
|
|
const dir = path.join(outRoot, task.folder);
|
||
|
|
const sheetPath = path.join(dir, `spritesheet.${task.spritesheetExt}`);
|
||
|
|
const manifestPath = path.join(dir, 'pet.json');
|
||
|
|
if (!force && (await pathExists(sheetPath)) && (await pathExists(manifestPath))) {
|
||
|
|
return 'skipped';
|
||
|
|
}
|
||
|
|
await mkdir(dir, { recursive: true });
|
||
|
|
const bytes = await downloadBinary(task.spritesheetUrl);
|
||
|
|
if (bytes.length < 16) {
|
||
|
|
throw new Error(`${task.folder}: spritesheet too small (${bytes.length} bytes)`);
|
||
|
|
}
|
||
|
|
// Reject HTML error pages dressed as `.webp` so the UI doesn't end up
|
||
|
|
// adopting a pet whose sprite is `<!doctype html>`.
|
||
|
|
const head = bytes.subarray(0, 12);
|
||
|
|
const isWebp = head.toString('ascii', 0, 4) === 'RIFF' && head.toString('ascii', 8, 12) === 'WEBP';
|
||
|
|
const isPng = head.toString('hex', 0, 8) === '89504e470d0a1a0a';
|
||
|
|
const isGif = head.toString('ascii', 0, 6) === 'GIF87a' || head.toString('ascii', 0, 6) === 'GIF89a';
|
||
|
|
if (!isWebp && !isPng && !isGif) {
|
||
|
|
throw new Error(`${task.folder}: spritesheet is not webp/png/gif`);
|
||
|
|
}
|
||
|
|
await writeFile(sheetPath, bytes);
|
||
|
|
await writeFile(manifestPath, JSON.stringify(task.manifest, null, 2) + '\n', 'utf8');
|
||
|
|
return 'wrote';
|
||
|
|
}
|
||
|
|
|
||
|
|
async function runPool<T, R>(
|
||
|
|
items: T[],
|
||
|
|
concurrency: number,
|
||
|
|
worker: (item: T, index: number) => Promise<R>,
|
||
|
|
): Promise<R[]> {
|
||
|
|
const results: R[] = new Array(items.length);
|
||
|
|
let cursor = 0;
|
||
|
|
const workers = Array.from(
|
||
|
|
{ length: Math.min(concurrency, items.length) },
|
||
|
|
async () => {
|
||
|
|
for (;;) {
|
||
|
|
const idx = cursor++;
|
||
|
|
if (idx >= items.length) return;
|
||
|
|
results[idx] = await worker(items[idx]!, idx);
|
||
|
|
}
|
||
|
|
},
|
||
|
|
);
|
||
|
|
await Promise.all(workers);
|
||
|
|
return results;
|
||
|
|
}
|
||
|
|
|
||
|
|
export async function syncCommunityPets(options: SyncOptions = {}): Promise<SyncResult> {
|
||
|
|
const sourceArg = options.source ?? 'all';
|
||
|
|
const sources = new Set<'petshare' | 'hatchery'>();
|
||
|
|
if (sourceArg === 'all' || sourceArg === 'petshare') sources.add('petshare');
|
||
|
|
if (sourceArg === 'all' || sourceArg === 'hatchery') sources.add('hatchery');
|
||
|
|
const force = Boolean(options.force);
|
||
|
|
const limit =
|
||
|
|
options.limit && Number.isFinite(options.limit) ? Math.max(1, options.limit) : null;
|
||
|
|
const concurrency =
|
||
|
|
options.concurrency && Number.isFinite(options.concurrency)
|
||
|
|
? Math.max(1, options.concurrency)
|
||
|
|
: 6;
|
||
|
|
|
||
|
|
const rootDir = resolveCodexPetsRoot();
|
||
|
|
await mkdir(rootDir, { recursive: true });
|
||
|
|
|
||
|
|
const errors: string[] = [];
|
||
|
|
const tasks: PetTask[] = [];
|
||
|
|
|
||
|
|
if (sources.has('petshare')) {
|
||
|
|
try {
|
||
|
|
tasks.push(...(await listPetSharePets(limit)));
|
||
|
|
} catch (err) {
|
||
|
|
errors.push((err as Error).message ?? String(err));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (sources.has('hatchery')) {
|
||
|
|
try {
|
||
|
|
tasks.push(...(await listHatcheryPets(limit)));
|
||
|
|
} catch (err) {
|
||
|
|
errors.push((err as Error).message ?? String(err));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Earlier sources win when two catalogs publish the same folder name
|
||
|
|
// — matches the CLI script's de-duplication so a sync from the UI
|
||
|
|
// produces the same on-disk layout as `pnpm sync:community-pets`.
|
||
|
|
const dedup = new Map<string, PetTask>();
|
||
|
|
for (const task of tasks) {
|
||
|
|
if (!dedup.has(task.folder)) dedup.set(task.folder, task);
|
||
|
|
}
|
||
|
|
const unique = Array.from(dedup.values());
|
||
|
|
|
||
|
|
let wrote = 0;
|
||
|
|
let skipped = 0;
|
||
|
|
let failed = 0;
|
||
|
|
await runPool(unique, concurrency, async (task) => {
|
||
|
|
try {
|
||
|
|
const result = await writePet(task, rootDir, force);
|
||
|
|
if (result === 'wrote') wrote++;
|
||
|
|
else skipped++;
|
||
|
|
} catch (err) {
|
||
|
|
failed++;
|
||
|
|
const message = (err as Error).message ?? String(err);
|
||
|
|
// Cap the surfaced errors so a fully-broken catalog doesn't ship
|
||
|
|
// a 200KB JSON response; the daemon log keeps the rest.
|
||
|
|
if (errors.length < 10) errors.push(`${task.folder}: ${message}`);
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
return {
|
||
|
|
wrote,
|
||
|
|
skipped,
|
||
|
|
failed,
|
||
|
|
total: unique.length,
|
||
|
|
rootDir,
|
||
|
|
errors,
|
||
|
|
};
|
||
|
|
}
|