refactor: resources

app/entry.server.jsx

@@ -64,6 +64,10 @@ function handleBotRequest(
           const stream = createReadableStreamFromReadable(body);
 
           responseHeaders.set("Content-Type", "text/html");
+          // security
+          responseHeaders.set("Cross-Origin-Embedder-Policy", "require-corp");
+          responseHeaders.set("Cross-Origin-Opener-Policy", "same-origin");
+          responseHeaders.set("Cross-Origin-Resource-Policy", "same-site");
 
           resolve(
             new Response(stream, {

app/react/components/client-ecs.js

@@ -1,11 +1,5 @@
-import {LRUCache} from 'lru-cache';
-
 import Ecs from '@/ecs/ecs.js';
-import {withResolvers} from '@/util/promise.js';
-
-const cache = new LRUCache({
-  max: 128,
-});
+import {readAsset} from '@/util/resources.js';
 
 export default class ClientEcs extends Ecs {
   constructor(specification) {
@@ -19,16 +13,10 @@ export default class ClientEcs extends Ecs {
       }
     });
   }
-  async readAsset(uri) {
-    if (!cache.has(uri)) {
-      const {promise, resolve, reject} = withResolvers();
-      cache.set(uri, promise);
-      fetch(new URL(uri, location.origin))
-        .then(async (response) => {
-          resolve(response.ok ? response.arrayBuffer() : new ArrayBuffer(0));
-        })
-        .catch(reject);
-    }
-    return cache.get(uri);
+  async readAsset(path) {
+    const resource = await readAsset(path);
+    return resource
+      ? resource
+      : new ArrayBuffer(0);
   }
 }

app/react/components/ui.jsx

@@ -8,7 +8,6 @@ import {RESOLUTION} from '@/util/constants.js';
 import EventEmitter from '@/util/event-emitter.js';
 
 import addKeyListener from './add-key-listener.js';
-import ClientEcs from './client-ecs.js';
 import Disconnected from './dom/disconnected.jsx';
 import Chat from './dom/chat/chat.jsx';
 import Bag from './dom/bag.jsx';
@@ -49,8 +48,6 @@ function Ui({disconnected}) {
   const [inventorySlots, setInventorySlots] = useState(emptySlots());
   const [activeSlot, setActiveSlot] = useState(0);
   const [scale, setScale] = useState(2);
-  const [Components, setComponents] = useState();
-  const [Systems, setSystems] = useState();
   const monopolizers = useRef([]);
   const [message, setMessage] = useState('');
   const [chatIsOpen, setChatIsOpen] = useState(false);
@@ -64,24 +61,6 @@ function Ui({disconnected}) {
   const [externalInventory, setExternalInventory] = useState();
   const [externalInventorySlots, setExternalInventorySlots] = useState();
   const [particleWorker, setParticleWorker] = useState();
-  const refreshEcs = useCallback(() => {
-    class ClientEcsPerf extends ClientEcs {
-      markChange() {}
-    }
-    ecsRef.current = new ClientEcsPerf({Components, Systems});
-  }, [ecsRef, Components, Systems]);
-  useEffect(() => {
-    async function setEcsStuff() {
-      const {default: Components} = await import('@/ecs/components/index.js');
-      const {default: Systems} = await import('@/ecs/systems/index.js');
-      setComponents(Components);
-      setSystems(Systems);
-    }
-    setEcsStuff();
-  }, []);
-  useEffect(() => {
-    refreshEcs();
-  }, [refreshEcs]);
   useEffect(() => {
     let handle;
     if (disconnected) {
@@ -238,10 +217,11 @@ function Ui({disconnected}) {
     });
   }, [client, keepHotbarOpen, setDebug]);
   const onEcsChangePacket = useCallback(() => {
-    refreshEcs();
     mainEntityRef.current = undefined;
     monopolizers.current = [];
-  }, [refreshEcs, mainEntityRef]);
+  }, [
+    mainEntityRef,
+  ]);
   usePacket('EcsChange', onEcsChangePacket);
   const onTickPacket = useCallback(async (payload, client) => {
     if (0 === Object.keys(payload.ecs).length) {

app/routes/_main-menu.play.$.$/route.jsx

@@ -1,4 +1,3 @@
-import {json} from "@remix-run/node";
 import {useCallback, useEffect, useRef, useState} from 'react';
 import {useOutletContext, useParams} from 'react-router-dom';
 
@@ -10,13 +9,9 @@ import EcsContext from '@/react/context/ecs.js';
 import MainEntityContext from '@/react/context/main-entity.js';
 import RadiansContext from '@/react/context/radians.js';
 import useAnimationFrame from '@/react/hooks/use-animation-frame.js';
-import {juggleSession} from '@/server/session.server.js';
 import {TAU} from '@/util/math.js';
 
-export async function loader({request}) {
-  await juggleSession(request);
-  return json({});
-}
+import ClientEcs from '@/react/components/client-ecs.js';
 
 export default function PlaySpecific() {
   const Client = useOutletContext();
@@ -24,6 +19,8 @@ export default function PlaySpecific() {
   const [client, setClient] = useState();
   const mainEntityRef = useRef();
   const debugTuple = useState(false);
+  const [Components, setComponents] = useState();
+  const [Systems, setSystems] = useState();
   const ecsRef = useRef();
   const [disconnected, setDisconnected] = useState(false);
   const params = useParams();
@@ -64,6 +61,36 @@ export default function PlaySpecific() {
       removeEventListener('beforeunload', onBeforeUnload);
     };
   });
+  const refreshEcs = useCallback(() => {
+    class ClientEcsPerf extends ClientEcs {
+      markChange() {}
+    }
+    ecsRef.current = new ClientEcsPerf({Components, Systems});
+  }, [ecsRef, Components, Systems]);
+  useEffect(() => {
+    async function setEcsStuff() {
+      const {default: Components} = await import('@/ecs/components/index.js');
+      const {default: Systems} = await import('@/ecs/systems/index.js');
+      setComponents(Components);
+      setSystems(Systems);
+    }
+    setEcsStuff();
+  }, []);
+  useEffect(() => {
+    refreshEcs();
+  }, [refreshEcs]);
+  useEffect(() => {
+    if (!client) {
+      return;
+    }
+    function onEcsChangePacket() {
+      refreshEcs();
+    }
+    client.addPacketListener('EcsChange', onEcsChangePacket);
+    return () => {
+      client.removePacketListener('EcsChange', onEcsChangePacket);
+    };
+  }, [client, refreshEcs]);
   useEffect(() => {
     if (!client) {
       return;

app/routes/_main-menu.play.$/route.jsx

@@ -1,12 +1,66 @@
+import {settings} from '@pixi/core';
+import {json, useLoaderData} from "@remix-run/react";
 import {useEffect, useState} from 'react';
 import {Outlet, useParams} from 'react-router-dom';
 
+import {
+  computeMissing,
+  fetchResources,
+  get,
+  readAsset,
+  set,
+} from '@/util/resources.js';
+
 import styles from './play.module.css';
 
+settings.ADAPTER.fetch = async (path) => {
+  const resource = await readAsset(path);
+  return resource ? new Response(resource) : new Response(undefined, {status: 404});
+};
+
+export async function loader({request}) {
+  const {juggleSession} = await import('@/server/session.server.js');
+  const {loadManifest} = await import('@/util/resources.server.js');
+  await juggleSession(request);
+  return json({
+    manifest: await loadManifest(),
+  });
+}
+
 export default function Play() {
+  const {manifest} = useLoaderData();
   const [Client, setClient] = useState();
   const params = useParams();
   const [type] = params['*'].split('/');
+  useEffect(() => {
+    const controller = new AbortController();
+    const {signal} = controller;
+    async function receiveResources() {
+      const current = await get();
+      const paths = await computeMissing(current, manifest);
+      if (paths.length > 0 && !signal.aborted) {
+        try {
+          const resources = await fetchResources(paths, {signal});
+          if (resources) {
+            for (const key in resources) {
+              current[key] = resources[key];
+            }
+            await set(current);
+          }
+        }
+        catch (e) {
+          if ((e instanceof DOMException) && 'AbortError' === e.name) {
+            return;
+          }
+          throw e;
+        }
+      }
+    }
+    receiveResources();
+    return () => {
+      controller.abort();
+    };
+  }, [manifest]);
   useEffect(() => {
     async function loadClient() {
       let Client;

app/routes/resources.stream/route.jsx

@@ -0,0 +1,8 @@
+import {encodeResources, loadResources} from '@/util/resources.server.js';
+
+export async function action({request}) {
+  const paths = await request.json();
+  const assets = await loadResources();
+  const buffer = await encodeResources(paths.map((path) => assets[path]));
+  return new Response(buffer);
+}

app/server/websocket.js

@@ -8,7 +8,9 @@ import {getSession} from '@/server/session.server.js';
 
 import Engine from './engine.js';
 
-const isInsecure = process.env.SILPHIUS_INSECURE_HTTP;
+const {
+  RESOURCES_PATH = [process.cwd(), 'resources'].join('/'),
+} = process.env;
 
 global.__silphiusWebsocket = null;
 
@@ -20,13 +22,18 @@ class SocketServer extends Server {
     return join(import.meta.dirname, '..', '..', 'data', 'remote', 'UNIVERSE', path);
   }
   async readAsset(path) {
-    const url = new URL(path, 'https://localhost:3000')
-    if (isInsecure) {
-      url.protocol = 'http:';
+    const {pathname} = new URL(path, 'http://example.org');
+    const resourcePath = pathname.slice('/resources/'.length);
+    try {
+      const {buffer} = await readFile([RESOURCES_PATH, resourcePath].join('/'));
+      return buffer;
+    }
+    catch (error) {
+      if ('ENOENT' !== error.code) {
+        throw error;
+      }
+      return new ArrayBuffer(0);
     }
-    return fetch(url.href).then((response) => (
-      response.ok ? response.arrayBuffer() : new ArrayBuffer(0)
-    ));
   }
   async readData(path) {
     const qualified = this.constructor.qualify(path);

app/server/worker.js

@@ -3,6 +3,7 @@ import {del, get, set} from 'idb-keyval';
 import {encode} from '@/net/packets/index.js';
 import Server from '@/net/server.js';
 import {withResolvers} from '@/util/promise.js';
+import {readAsset} from '@/util/resources.js';
 
 import createEcs from './create/ecs.js';
 import './create/forest.js';
@@ -21,9 +22,10 @@ class WorkerServer extends Server {
     return ['UNIVERSE', path].join('/');
   }
   async readAsset(path) {
-    return fetch(path).then((response) => (
-      response.ok ? response.arrayBuffer() : new ArrayBuffer(0)
-    ));
+    const resource = await readAsset(path);
+    return resource
+      ? resource
+      : new ArrayBuffer(0);
   }
   async readData(path) {
     const data = await get(this.constructor.qualify(path));

app/util/resources.js

@@ -0,0 +1,81 @@
+export async function computeMissing(current, manifest) {
+  const missing = [];
+  for (const path in manifest) {
+    if (!current || !current[path] || current[path].hash !== manifest[path]) {
+      missing.push(path);
+    }
+  }
+  return missing;
+}
+
+const octets = [];
+for (let i = 0; i < 256; ++i) {
+  octets.push(i.toString(16).padStart(2, '0'));
+}
+
+export async function parseResources(resources, buffer) {
+  const view = new DataView(buffer);
+  let caret = 0;
+  const count = view.getUint32(caret);
+  caret += 4;
+  const manifest = {};
+  for (let i = 0; i < count; ++i) {
+    let hash = '';
+    for (let j = 0; j < 20; ++j) {
+      hash += octets[view.getUint8(caret)]
+      caret += 1;
+    }
+    const byteLength = view.getUint32(caret);
+    caret += 4;
+    const asset = new ArrayBuffer(byteLength);
+    const assetView = new DataView(asset);
+    for (let j = 0; j < asset.byteLength; ++j) {
+      assetView.setUint8(j, view.getUint8(caret));
+      caret += 1;
+    }
+    manifest[resources[i]] = {asset, hash};
+  }
+  return manifest;
+}
+
+export async function fetchResources(paths, {signal} = {}) {
+  const response = await fetch('/resources/stream', {
+    body: JSON.stringify(paths),
+    method: 'post',
+    signal,
+  });
+  if (signal.aborted) {
+    return undefined;
+  }
+  return parseResources(paths, await response.arrayBuffer());
+}
+
+export class Cache {
+  async get() {
+    const {get} = await import('idb-keyval');
+    const resources = await get('$$silphius_resources');
+    return resources || {};
+  }
+  async set(manifest) {
+    const {set} = await import('idb-keyval');
+    return set('$$silphius_resources', manifest);
+  }
+}
+
+export async function get() {
+  const {get} = await import('idb-keyval');
+  const resources = await get('$$silphius_resources');
+  return resources || {};
+}
+
+export async function set(resources) {
+  const {set} = await import('idb-keyval');
+  await set('$$silphius_resources', resources);
+}
+
+export async function readAsset(path) {
+  const {pathname} = new URL(path, 'http://example.org');
+  const resourcePath = pathname.slice('/resources/'.length);
+  const resources = await get();
+  return resources[resourcePath]?.asset;
+}

app/util/resources.server.js

@@ -0,0 +1,110 @@
+import {createHash} from 'node:crypto';
+import {readFile, realpath} from 'node:fs/promises';
+
+import chokidar from 'chokidar';
+import {glob} from 'glob';
+
+import {singleton} from './singleton.js';
+
+const {
+  RESOURCES_PATH = [process.cwd(), 'resources'].join('/'),
+} = process.env;
+
+const resources = singleton('resources', {});
+const manifest = singleton('manifest', {});
+const RESOURCES_PATH_REAL = await realpath(RESOURCES_PATH);
+const RESOURCES_GLOB = [RESOURCES_PATH_REAL, '**', '*'].join('/');
+
+async function computeAsset(path) {
+  const buffer = await readFile(path);
+  return {
+    asset: buffer,
+    hash: createHash('sha1').update(buffer).digest(),
+    path: path.slice(RESOURCES_PATH_REAL.length + 1),
+  };
+}
+
+async function assetPaths() {
+  return glob(RESOURCES_GLOB, {nodir: true});
+}
+
+export async function createManifest() {
+  const paths = await assetPaths();
+  const entries = await Promise.all(paths.map(async (path) => {
+    const asset = await computeAsset(path);
+    return [asset.path, asset.hash.toString('hex')];
+  }))
+  return Object.fromEntries(
+    entries.filter(Boolean),
+  );
+}
+
+export async function loadManifest() {
+  emptyCheck: {
+    for (const path in manifest) { // eslint-disable-line no-unused-vars
+      break emptyCheck;
+    }
+    const created = await createManifest();
+    for (const path in created) {
+      manifest[path] = created[path];
+    }
+    const watcher = chokidar.watch(RESOURCES_GLOB);
+    watcher.on('add', async (path) => {
+      const asset = await computeAsset(path);
+      manifest[asset.path] = asset.hash.toString('hex');
+    });
+    watcher.on('change', async (path) => {
+      const asset = await computeAsset(path);
+      manifest[asset.path] = asset.hash.toString('hex');
+    });
+    watcher.on('unlink', async (path) => {
+      delete manifest[path.slice(import.meta.dirname.length - 1)];
+    });
+  }
+  return manifest;
+}
+
+export async function encodeResources(resources) {
+  let byteLength = 4;
+  for (const {asset} of resources) {
+    byteLength += 20;
+    byteLength += 4;
+    byteLength += asset.byteLength;
+  }
+  const buffer = new ArrayBuffer(byteLength);
+  const view = new DataView(buffer);
+  let caret = 0;
+  view.setUint32(caret, resources.length);
+  caret += 4;
+  for (const {asset, hash} of resources) {
+    const hashView = new DataView(hash);
+    for (let j = 0; j < 20; ++j) {
+      view.setUint8(caret, hashView.getUint8(j));
+      caret += 1;
+    }
+    view.setUint32(caret, asset.byteLength);
+    caret += 4;
+    const assetView = new DataView(asset);
+    for (let j = 0; j < asset.byteLength; ++j) {
+      view.setUint8(caret, assetView.getUint8(j));
+      caret += 1;
+    }
+  }
+  return buffer;
+}
+
+export async function loadResources() {
+  emptyCheck: {
+    for (const path in resources) { // eslint-disable-line no-unused-vars
+      break emptyCheck;
+    }
+    const paths = await assetPaths();
+    await Promise.all(
+      paths.map(async (path) => {
+        const {asset, hash, path: assetPath} = await computeAsset(path);
+        resources[assetPath] = {asset: asset.buffer, hash: hash.buffer};
+      }),
+    );
+  }
+  return resources;
+}

app/util/resources.test.js

@@ -0,0 +1,45 @@
+import {expect, test} from 'vitest';
+
+import {computeMissing, parseResources} from './resources.js';
+import {encodeResources} from './resources.server.js';
+
+test('cache', async () => {
+  class TestCache {
+    current;
+    async get() {
+      return new Promise((resolve) => {
+        resolve(this.current);
+      });
+    }
+    async set(manifest) {
+      return new Promise((resolve) => {
+        this.current = manifest;
+        resolve();
+      });
+    }
+  }
+  const cache = new TestCache();
+  expect(await computeMissing(await cache.get(), {foo: 'bar'})).to.deep.equal(['foo']);
+  await cache.set({foo: {hash: 'bar'}});
+  expect(await computeMissing(await cache.get(), {foo: 'bar'})).to.deep.equal([]);
+  expect(await computeMissing(await cache.get(), {foo: 'bar', baz: '32'})).to.deep.equal(['baz']);
+  await cache.set({foo: {hash: 'bar'}, baz: '32'});
+  expect(await computeMissing(await cache.get(), {foo: 'bar', baz: '64'})).to.deep.equal(['baz']);
+});
+
+test('codec', async () => {
+  const asset = {
+    asset: (new Uint8Array('hello'.split('').map((letter) => letter.charCodeAt(0)))).buffer,
+    hash: (new Uint8Array(Array(20).fill(0).map((_, i) => i))).buffer,
+  };
+  const buffer = await encodeResources([
+    asset,
+  ]);
+  const parsed = await parseResources(['hello.txt'], buffer);
+  expect('hello.txt' in parsed).to.equal(true);
+  expect(parsed['hello.txt'].hash).to.equal('000102030405060708090a0b0c0d0e0f10111213');
+  const hello = new Uint8Array(parsed['hello.txt'].asset);
+  for (let i = 0; i < 5; ++i) {
+    expect(hello[i]).to.equal('hello'.charCodeAt(i));
+  }
+});

app/util/singleton.js

@@ -0,0 +1,9 @@
+export function singleton(key, value) {
+  global.__singletons ??= {};
+  global.__singletons[key] ??= value;
+  return global.__singletons[key];
+}
+
+singleton.reset = function (key) {
+  delete global.__singletons[key];
+}