Merge branch 'main' into #141

CONTRIBUTING.md

@@ -51,6 +51,12 @@ We maintain a list of issues that are good for first-time contributors. You can
 
 <br>
 
+## Documentation for Contributors
+
+See [doc/contributors/index.md](./doc/contributors/index.md) for more information.
+
+<br>
+
 ## Code Review
 
 Once you've submitted your pull request, the project maintainers will review your changes. We may suggest some changes or improvements. This is a normal part of the process, and your contributions are greatly appreciated!

README.md

@@ -1,6 +1,6 @@
 <h3 align="center"><img width="80" alt="Puter.com, The Personal Cloud Computer: All your files, apps, and games in one place accessible from anywhere at any time." src="https://assets.puter.site/puter-logo.png"></h3>
 
-<h3 align="center">The Internet OS, Free and Open-Source!</h3>
+<h3 align="center">The Internet OS! Free, Open-Source, and Self-Hostable!</h3>
 
 <p align="center">
     <a href="https://puter.com/"><strong>« LIVE DEMO »</strong></a>
@@ -23,7 +23,7 @@
 
 ## Puter
 
-Puter is an advanced open-source desktop environment in the browser, designed to be feature-rich, exceptionally fast, and highly extensible. It can be used to build remote desktop environments or serve as an interface for cloud storage services, remote servers, web hosting platforms, and more.
+Puter is an advanced, open-source internet operating system designed to be feature-rich, exceptionally fast, and highly extensible. It can be used to build remote desktop environments or serve as an interface for cloud storage services, remote servers, web hosting platforms, and more.
 
 <br/>
 
@@ -52,9 +52,8 @@ docker compose up
 
 <br/>
 
-## Deploy to Production
-
-Detailed guide on how to deploy Puter in production: [docs/prod.md](docs/prod.md)
+## ⚠️ Self-Hosting ⚠️
+The self-hosted version of Puter is currently in alpha stage and should not be used in production yet. It is under active development and may contain bugs, other issues. Please exercise caution and use it for testing and evaluation purposes only.
 
 <br/>
 

doc/contributors/index.md

@@ -0,0 +1,25 @@
+# Contributing to Puter
+
+## Essential / General Knowledge
+
+### Repository Dichotomy
+
+- Puter's GUI is at the root; `/src` is the GUI
+- Puter's backend is a workspace npm package;
+  it resides in `packages/backend(/src)`
+
+The above may seem counter-intuitive; backend and frontend are siblings, right?
+Consider this: by a different intuition, the backend is at a "deeper" level
+of function; this directory structure better adheres to soon-to-be contributors
+sifting around through the files to discover "what's what".
+
+The directory `volatile` exists _for your convenience_ to simplify running
+Puter for development. When Puter is run
+run with the backend with this repository as its working directory, it
+will use `volatile/config` and `volatile/runtime` instead of
+`/etc/puter` and `/var/puter`.
+
+## See Next
+
+- [Backend Documentation](../../packages/backend/doc/contributors/index.md)
+<!-- - [Frontend Documentation](./frontend.md) -->

doc/license_header.txt

@@ -0,0 +1,16 @@
+Copyright (C) 2024 Puter Technologies Inc.
+
+This file is part of Puter.
+
+Puter is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published
+by the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.

exports.js

@@ -0,0 +1,2 @@
+import backend from '@heyputer/backend';
+export default backend;

l_checker_config.json

@@ -0,0 +1,26 @@
+{
+    "ignore": ["**/!(*.js|*.css)", "**/assets/**"],
+    "license": "doc/license_header.txt",
+    "licenseFormats": {
+        "js": {
+            "prepend": "/*",
+            "append": " */",
+            "eachLine": {
+                "prepend": " * "
+            }
+        },
+        "dotfile|^Dockerfile": {
+            "eachLine": {
+                "prepend": "# "
+            }
+        },
+        "css": {
+            "prepend": "/*",
+            "append": " */",
+            "eachLine": {
+                "prepend": " * "
+            }
+        }
+    },
+    "trailingWhitespace": "TRIM"
+}

package.json

@@ -6,6 +6,7 @@
   "description": "Desktop environment in the browser!",
   "homepage": "https://puter.com",
   "type": "module",
+  "main": "exports.js",
   "directories": {
     "lib": "lib"
   },
@@ -15,22 +16,29 @@
     "dotenv": "^16.4.5",
     "express": "^4.18.2",
     "html-entities": "^2.3.3",
-    "nodemon": "^2.0.22",
+    "nodemon": "^3.1.0",
     "uglify-js": "^3.17.4",
     "webpack": "^5.88.2",
     "webpack-cli": "^5.1.1"
   },
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
-    "start": "nodemon --exec \"node dev-server.js\" ",
+    "start=gui": "nodemon --exec \"node dev-server.js\" ",
+    "start": "node run-selfhosted.js",
     "build": "node ./build.js",
     "check-translations": "node tools/check-translations.js"
   },
+  "workspaces": [
+    "packages/*"
+  ],
   "nodemonConfig": {
     "ext": "js, json, mjs, jsx, svg, css",
     "ignore": [
       "./dist/",
       "./node_modules/"
     ]
+  },
+  "dependencies": {
+    "uuid": "^9.0.1"
   }
 }

packages/backend/.gitignore

@@ -0,0 +1,151 @@
+# MAC OS hidden directory settings file
+.DS_Store
+
+# Created by https://www.toptal.com/developers/gitignore/api/node
+# Edit at https://www.toptal.com/developers/gitignore?templates=node
+
+### Node ###
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+web_modules/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+.env.test
+.env.production
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+.parcel-cache
+
+# Next.js build output
+.next
+out
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and not Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+.vscode-test
+
+# yarn v2
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*
+
+# End of https://www.toptal.com/developers/gitignore/api/node
+public/.DS_Store
+*.zip
+*.pem
+public/.DS_Store
+public/.DS_Store
+public/.DS_Store
+./build
+build
+
+# config file
+volatile/
+ssl
+ssl/
+keys
+*.code-workspace
+
+# credentials
+creds*
+
+# thumbnai-service
+thumbnail-service
+
+# init sql generated from ./run.sh
+init.sql

packages/backend/README.md

@@ -0,0 +1,3 @@
+# Puter Backend
+
+- [Documentation for Contributors](./doc/contributors/index.md)

packages/backend/doc/contributors/index.md

@@ -0,0 +1,62 @@
+# Contributing to Puter's Backend
+
+## Initial Reading
+
+- [puter-js-common's README.md](../../packages/puter-js-common/README.md)
+  - Whenever you see `AdvancedBase`, that's from here
+    - Many things in backend extend this. Anything that doesn't only doesn't
+      because it was written before `AdvancedBase` existed.
+  - Allows adding "traits" to classes
+    - Have you ever wanted to wrap every method of a class with
+      common behavior? This can do that!
+
+## Where to Start
+
+- [Kernel](../../src/Kernel.js), despite its intimidating name, is a
+  relatively simple (< 200 LOC) class which loads the modules
+  (modules register services), and then starts all the services.
+- [RuntimeEnvironment](../../src/boot/RuntimeEnvironment.js)
+  sets the configuration and runtime directories. It's invoked by Kernel.
+- The default setup for running a self-hosted Puter loads these modules:
+  - [CoreModule](../../src/CoreModule.js)
+  - [DatabaseModule](../../src/DatabaseModule.js)
+  - [LocalDiskStorageModule](../../src/LocalDiskStorageModule.js)
+- HTTP endpoints are registered with
+  [WebServerService](../../src/services/WebServerService.js)
+  by these services:
+  - [ServeGUIService](../../src/services/ServeGUIService.js)
+  - [PuterAPIService](../../src/services/PuterAPIService.js)
+  - [FilesystemAPIService](../../src/services/FilesystemAPIService.js)
+
+## Development Philosophies
+
+### The copy-paste rule
+
+If you're copying and pasting code, you need to ask this question:
+- am I copying as a reference (i.e. how this function is used),
+- or am I copying an implementation of actual behavior?
+
+If your answer is the first, you should find more than one piece of
+code that's doing the same thing you want to do and see if any of them
+are doing it differently. One of the ways of doing this thing is going
+to be more recent and/or (yes, potentially "or") more correct.
+More correct approaches are ones which reduce
+[coupling](https://en.wikipedia.org/wiki/Coupling_(computer_programming)),
+move from legacy implementations to more recent ones, and are actually
+more convenient for you to use. Whenever ever any of these three things
+are in contention it's very important to communicate this to the
+appropriate maintainers and contributors.
+
+If your answer is the second, you should find a way to
+[DRY that code](https://en.wikipedia.org/wiki/Don%27t_repeat_yourself).
+
+### Architecture Mistakes? You will make them and it will suck.
+
+In my experience, the harder I think about the correct way to implement
+something, the bigger a mistake I'm going to make; ***unless*** a big part
+of the reason I'm thinking so hard is because I want to find a solution
+that reduces complexity and has the right maintanence trade-off.
+There's no easy solution for this so just keep it in mind; there are some
+things we might write 2 times, 3 times, even more times over before we
+really get it right and *that's okay*; sometimes part of doing useful work is
+doing the useless work that reveals what the useful work is.

packages/backend/doc/license_header.txt

@@ -0,0 +1,16 @@
+Copyright (C) 2024 Puter Technologies Inc.
+
+This file is part of Puter.
+
+Puter is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published
+by the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.

packages/backend/exports.js

@@ -0,0 +1,47 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const CoreModule = require("./src/CoreModule.js");
+const { Kernel } = require("./src/Kernel.js");
+const LocalDiskStorageModule = require("./src/LocalDiskStorageModule.js");
+const DatabaseModule = require("./src/DatabaseModule.js");
+const PuterDriversModule = require("./src/PuterDriversModule.js");
+const { testlaunch } = require("./src/index.js");
+const BaseService = require("./src/services/BaseService.js");
+const { Context } = require("./src/util/context.js");
+
+
+module.exports = {
+    helloworld: () => {
+        console.log('Hello, World!');
+        process.exit(0);
+    },
+    testlaunch,
+
+    // Kernel API
+    BaseService,
+    Context,
+
+    Kernel,
+
+    // Pre-built modules
+    CoreModule,
+    DatabaseModule,
+    PuterDriversModule,
+    LocalDiskStorageModule,
+};

packages/backend/package.json

@@ -0,0 +1,83 @@
+{
+  "name": "@heyputer/backend",
+  "version": "1.0.0",
+  "description": "Backend/Kernel for Puter",
+  "main": "exports.js",
+  "scripts": {
+    "test": "npx mocha"
+  },
+  "workspaces": [
+    "packages/*"
+  ],
+  "dependencies": {
+    "@heyputer/kv.js": "^0.1.3",
+    "@heyputer/multest": "^0.0.2",
+    "@opentelemetry/api": "^1.4.1",
+    "@opentelemetry/auto-instrumentations-node": "^0.43.0",
+    "@opentelemetry/exporter-trace-otlp-grpc": "^0.40.0",
+    "@opentelemetry/sdk-metrics": "^1.14.0",
+    "@opentelemetry/sdk-node": "^0.49.1",
+    "@pagerduty/pdjs": "^2.2.4",
+    "@smithy/node-http-handler": "^2.2.2",
+    "args": "^5.0.3",
+    "aws-sdk": "^2.1383.0",
+    "axios": "^1.4.0",
+    "bcrypt": "^5.1.0",
+    "better-sqlite3": "^9.4.3",
+    "busboy": "^1.6.0",
+    "chai-as-promised": "^7.1.1",
+    "clean-css": "^5.3.2",
+    "composite-error": "^1.0.2",
+    "compression": "^1.7.4",
+    "cookie-parser": "^1.4.6",
+    "express": "^4.18.2",
+    "file-type": "^18.5.0",
+    "form-data": "^4.0.0",
+    "handlebars": "^4.7.8",
+    "helmet": "^7.0.0",
+    "html-entities": "^2.3.3",
+    "is-glob": "^4.0.3",
+    "isbot": "^3.7.1",
+    "jimp": "^0.22.8",
+    "js-sha256": "^0.9.0",
+    "json5": "^2.2.3",
+    "jsonwebtoken": "^9.0.0",
+    "knex": "^3.1.0",
+    "lorem-ipsum": "^2.0.8",
+    "micromatch": "^4.0.5",
+    "mime-types": "^2.1.35",
+    "moment": "^2.29.4",
+    "morgan": "^1.10.0",
+    "multer": "^1.4.5-lts.1",
+    "multi-progress": "^4.0.0",
+    "murmurhash": "^2.0.1",
+    "nodemailer": "^6.9.3",
+    "on-finished": "^2.4.1",
+    "openai": "^4.20.1",
+    "prompt-sync": "^4.2.0",
+    "recursive-readdir": "^2.2.3",
+    "response-time": "^2.3.2",
+    "seedrandom": "^3.0.5",
+    "socket.io": "^4.6.2",
+    "ssh2": "^1.13.0",
+    "string-hash": "^1.1.3",
+    "svgo": "^3.0.2",
+    "tiktoken": "^1.0.11",
+    "uglify-js": "^3.17.4",
+    "uuid": "^9.0.0",
+    "validator": "^13.9.0",
+    "winston": "^3.9.0",
+    "winston-daily-rotate-file": "^4.7.1"
+  },
+  "devDependencies": {
+    "@types/node": "^20.5.3",
+    "chai": "^4.3.7",
+    "mocha": "^10.2.0",
+    "nodemon": "^3.1.0",
+    "nyc": "^15.1.0",
+    "sinon": "^15.2.0",
+    "typescript": "^5.1.6"
+  },
+  "author": "Puter Technologies Inc.",
+  "license": "AGPL-3.0-only"
+}

packages/backend/packages/puter-js-common/README.md

@@ -0,0 +1,33 @@
+# Puter - Common Javascript Module
+
+This is a small module for javascript which you might call a
+"langauge tool"; it adds some behavior to make javascript classes
+more flexible, with an aim to avoid any significant complexity.
+
+Each class in this module is best described as an _idea_:
+
+### BasicBase
+
+**BasicBase** is the idea that there should be a common way to
+see the inheretence chain of the current instance, and obtain
+merged objects and arrays from static members of these classes.
+
+### TraitBase
+
+**TraitBase** is the idea that there should be a common way to
+"install" behavior into objects of a particular class, as
+dictated by the class definition. A trait might install a common
+set of methods ("mixins"), decorate all or a specified set of
+methods in the class (performance monitors, sanitization, etc),
+or anything else.
+
+### AdvancedBase
+
+**AdvancedBase** is the idea that, in a node.js environment,
+you always want the ability to add traits to a class and there
+are some default traits you want in all classes, which are:
+
+- `PropertiesTrait` - add lazy factories for instance members
+  instead of always populating them in the constructor.
+- `NodeModuleDITrait` - require node modules in a way that
+  allows unit tests to inject mocks easily.

packages/backend/packages/puter-js-common/index.js

@@ -0,0 +1,23 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { AdvancedBase } = require('./src/AdvancedBase');
+
+module.exports = {
+    AdvancedBase,
+};

packages/backend/packages/puter-js-common/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "puter-js-common",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "Puter Technologies Inc.",
+  "license": "UNLICENSED"
+}

packages/backend/packages/puter-js-common/src/AdvancedBase.js

@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+// This doesn't go in ./bases because it logically depends on
+// both ./bases and ./traits, and ./traits depends on ./bases.
+
+const { TraitBase } = require("./bases/TraitBase");
+
+class AdvancedBase extends TraitBase {
+    static TRAITS = [
+        require('./traits/NodeModuleDITrait'),
+        require('./traits/PropertiesTrait'),
+    ]
+}
+
+module.exports = {
+    AdvancedBase,
+};

packages/backend/packages/puter-js-common/src/bases/BasicBase.js

@@ -0,0 +1,55 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+class BasicBase {
+    _get_inheritance_chain () {
+        const chain = [];
+        let cls = this.constructor;
+        while ( cls && cls !== BasicBase ) {
+            chain.push(cls);
+            cls = cls.__proto__;
+        }
+        return chain.reverse();
+    }
+
+    _get_merged_static_array (key) {
+        const chain = this._get_inheritance_chain();
+        const values = [];
+        for ( const cls of chain ) {
+            if ( cls[key] ) {
+                values.push(...cls[key]);
+            }
+        }
+        return values;
+    }
+
+    _get_merged_static_object (key) {
+        const chain = this._get_inheritance_chain();
+        const values = {};
+        for ( const cls of chain ) {
+            if ( cls[key] ) {
+                Object.assign(values, cls[key]);
+            }
+        }
+        return values;
+    }
+}
+
+module.exports = {
+    BasicBase,
+};

packages/backend/packages/puter-js-common/src/bases/TraitBase.js

@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { BasicBase } = require("./BasicBase");
+
+class TraitBase extends BasicBase {
+    constructor (parameters, ...a) {
+        super(parameters, ...a);
+        for ( const trait of this.traits ) {
+            trait.install_in_instance(
+                this,
+                {
+                    parameters: parameters || {},
+                }
+            )
+        }
+    }
+
+    get traits () {
+        return this._get_merged_static_array('TRAITS');
+    }
+}
+
+module.exports = {
+    TraitBase,
+};

packages/backend/packages/puter-js-common/src/traits/NodeModuleDITrait.js

@@ -0,0 +1,59 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+/**
+ * This trait allows dependency injection of node modules.
+ * This is incredibly useful for passing mock implementations
+ * of modules for unit testing.
+ *
+ * @example
+ * class MyClass extends AdvancedBase {
+ *   static MODULES = {
+ *     axios,
+ *   };
+ * }
+ *
+ * const my_class = new MyClass({
+ *   modules: {
+ *     axios: MY_AXIOS_MOCK,
+ *   }
+ * });
+ */
+module.exports = {
+    install_in_instance: (instance, { parameters }) => {
+        const modules = instance._get_merged_static_object('MODULES');
+
+        if ( parameters.modules ) {
+            for ( const k in parameters.modules ) {
+                modules[k] = parameters.modules[k];
+            }
+        }
+
+        instance.modules = modules;
+
+        // This "require" function can shadow the real one so
+        // that editor tools are aware of the modules that
+        // are being used.
+        instance.require = (name) => {
+            if ( modules[name] ) {
+                return modules[name];
+            }
+            return require(name);
+        }
+    },
+};

packages/backend/packages/puter-js-common/src/traits/PropertiesTrait.js

@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+module.exports = {
+    install_in_instance: (instance) => {
+        const properties = instance._get_merged_static_object('PROPERTIES');
+
+        for ( const k in properties ) {
+            if ( typeof properties[k] === 'function' ) {
+                instance[k] = properties[k]();
+                continue;
+            }
+
+            if ( typeof properties[k] === 'object' ) {
+                // This will be supported in the future.
+                throw new Error(`Property ${k} in ${instance.constructor.name} ` +
+                    `is not a supported property specification.`);
+            }
+
+            instance[k] = properties[k];
+        }
+    }
+}

packages/backend/packages/puter-js-common/test/test.js

@@ -0,0 +1,72 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { expect } = require('chai');
+const { BasicBase } = require('../src/bases/BasicBase');
+const { AdvancedBase } = require('../src/AdvancedBase');
+
+class ClassA extends BasicBase {
+    static STATIC_OBJ = {
+        a: 1,
+        b: 2,
+    };
+    static STATIC_ARR = ['a', 'b'];
+}
+
+class ClassB extends ClassA {
+    static STATIC_OBJ = {
+        c: 3,
+        d: 4,
+    };
+    static STATIC_ARR = ['c', 'd'];
+}
+
+describe('testing', () => {
+    it('does a thing', () => {
+        const b = new ClassB();
+    
+        console.log(b._get_inheritance_chain());
+        console.log([ClassA, ClassB]);
+        expect(b._get_inheritance_chain()).deep.equal([ClassA, ClassB]);
+        expect(b._get_merged_static_array('STATIC_ARR'))
+            .deep.equal(['a', 'b', 'c', 'd']);
+        expect(b._get_merged_static_object('STATIC_OBJ'))
+            .deep.equal({ a: 1, b: 2, c: 3, d: 4 });
+    });
+});
+
+class ClassWithModule extends AdvancedBase {
+    static MODULES = {
+        axios: 'axios',
+    };
+}
+
+describe('AdvancedBase', () => {
+    it('passes DI modules to instance', () => {
+        const c1 = new ClassWithModule();
+        expect(c1.modules.axios).to.equal('axios');
+
+        const c2 = new ClassWithModule({
+            modules: {
+                axios: 'my-axios',
+            },
+        });
+        expect(c2.modules.axios).to.equal('my-axios');
+    });
+});
+

packages/backend/src/CoreModule.js

@@ -0,0 +1,223 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { AdvancedBase } = require("puter-js-common");
+const { Context } = require('./util/context');
+
+
+class CoreModule extends AdvancedBase {
+    async install (context) {
+        const services = context.get('services');
+        const app = context.get('app');
+        await install({ services, app });
+    }
+
+    // Some services were created before the BaseService
+    // class existed. They don't listen to the init event
+    // and the order in which they're instantiated matters.
+    // They all need to be installed after the init event
+    // is dispatched, so they get a separate install method.
+    async install_legacy (context) {
+        const services = context.get('services');
+        await install_legacy({ services });
+    }
+}
+
+module.exports = CoreModule;
+
+const install = async ({ services, app }) => {
+    const config = require('./config');
+
+    const { LogService } = require('./services/runtime-analysis/LogService');
+    const { PagerService } = require('./services/runtime-analysis/PagerService');
+    const { AlarmService } = require('./services/runtime-analysis/AlarmService');
+    const { ErrorService } = require('./services/runtime-analysis/ErrorService');
+    const { CommandService } = require('./services/CommandService');
+    const { ExpectationService } = require('./services/runtime-analysis/ExpectationService');
+    const { HTTPThumbnailService } = require('./services/thumbnails/HTTPThumbnailService');
+    const { PureJSThumbnailService } = require('./services/thumbnails/PureJSThumbnailService');
+    const { NAPIThumbnailService } = require('./services/thumbnails/NAPIThumbnailService');
+    const { DevConsoleService } = require('./services/DevConsoleService');
+    const { RateLimitService } = require('./services/sla/RateLimitService');
+    const { MonthlyUsageService } = require('./services/sla/MonthlyUsageService');
+    const { AuthService } = require('./services/auth/AuthService');
+    const { SLAService } = require('./services/sla/SLAService');
+    const { PermissionService } = require('./services/auth/PermissionService');
+    const { ACLService } = require('./services/auth/ACLService');
+    const { CoercionService } = require('./services/drivers/CoercionService');
+    const { PuterSiteService } = require('./services/PuterSiteService');
+    const { ContextInitService } = require('./services/ContextInitService');
+    const { IdentificationService } = require('./services/abuse-prevention/IdentificationService');
+    const { AuthAuditService } = require('./services/abuse-prevention/AuthAuditService');
+    const { RegistryService } = require('./services/RegistryService');
+    const { RegistrantService } = require('./services/RegistrantService');
+    const { SystemValidationService } = require('./services/SystemValidationService');
+    const { EntityStoreService } = require('./services/EntityStoreService');
+    const SQLES = require('./om/entitystorage/SQLES');
+    const ValidationES = require('./om/entitystorage/ValidationES');
+    const { SetOwnerES } = require('./om/entitystorage/SetOwnerES');
+    const AppES = require('./om/entitystorage/AppES');
+    const WriteByOwnerOnlyES = require('./om/entitystorage/WriteByOwnerOnlyES');
+    const SubdomainES = require('./om/entitystorage/SubdomainES');
+    const { MaxLimitES } = require('./om/entitystorage/MaxLimitES');
+    const { AppLimitedES } = require('./om/entitystorage/AppLimitedES');
+    const { ESBuilder } = require('./om/entitystorage/ESBuilder');
+    const { Eq, Or } = require('./om/query/query');
+    const { TrackSpendingService } = require('./services/TrackSpendingService');
+    const { ServerHealthService } = require('./services/runtime-analysis/ServerHealthService');
+    const { MakeProdDebuggingLessAwfulService } = require('./services/MakeProdDebuggingLessAwfulService');
+    const { ConfigurableCountingService } = require('./services/ConfigurableCountingService');
+    const { FSLockService } = require('./services/fs/FSLockService');
+    const { StrategizedService } = require('./services/StrategizedService');
+    const WebServerService = require('./services/WebServerService');
+    const FilesystemAPIService = require('./services/FilesystemAPIService');
+    const ServeGUIService = require('./services/ServeGUIService');
+    const PuterAPIService = require('./services/PuterAPIService');
+    const { RefreshAssociationsService } = require("./services/RefreshAssociationsService");
+    // Service names beginning with '__' aren't called by other services;
+    // these provide data/functionality to other services or produce
+    // side-effects from the events of other services.
+
+    // === Services which extend BaseService ===
+    services.registerService('system-validation', SystemValidationService);
+    services.registerService('server-health', ServerHealthService);
+    services.registerService('log-service', LogService);
+    services.registerService('commands', CommandService);
+    services.registerService('web-server', WebServerService, { app });
+    services.registerService('__api-filesystem', FilesystemAPIService);
+    services.registerService('__api', PuterAPIService);
+    services.registerService('__gui', ServeGUIService);
+    services.registerService('expectations', ExpectationService);
+    services.registerService('pager', PagerService);
+    services.registerService('alarm', AlarmService);
+    services.registerService('error-service', ErrorService);
+    services.registerService('registry', RegistryService);
+    services.registerService('__registrant', RegistrantService);
+    services.registerService('fslock', FSLockService);
+    services.registerService('es:app', EntityStoreService, {
+        entity: 'app',
+        upstream: ESBuilder.create([
+            SQLES, { table: 'app', debug: true, },
+            AppES,
+            AppLimitedES, {
+                // When apps query es:apps, they're allowed to see apps which
+                // are approved for listing and they're allowed to see their
+                // own entry.
+                exception: async () => {
+                    const actor = Context.get('actor');
+                    return new Or({
+                        children: [
+                            new Eq({
+                                key: 'approved_for_listing',
+                                value: 1,
+                            }),
+                            new Eq({
+                                key: 'uid',
+                                value: actor.type.app.uid,
+                            }),
+                        ]
+                    });
+                },
+            },
+            WriteByOwnerOnlyES,
+            ValidationES,
+            SetOwnerES,
+            MaxLimitES, { max: 50 },
+        ]),
+    });
+    services.registerService('es:subdomain', EntityStoreService, {
+        entity: 'subdomain',
+        upstream: ESBuilder.create([
+            SQLES, { table: 'subdomains', debug: true, },
+            SubdomainES,
+            AppLimitedES,
+            WriteByOwnerOnlyES,
+            ValidationES,
+            SetOwnerES,
+            MaxLimitES, { max: 50 },
+        ]),
+    });
+    services.registerService('rate-limit', RateLimitService);
+    services.registerService('monthly-usage', MonthlyUsageService);
+    services.registerService('auth', AuthService);
+    services.registerService('permission', PermissionService);
+    services.registerService('sla', SLAService);
+    services.registerService('acl', ACLService);
+    services.registerService('coercion', CoercionService);
+    services.registerService('puter-site', PuterSiteService);
+    services.registerService('context-init', ContextInitService);
+    services.registerService('identification', IdentificationService);
+    services.registerService('auth-audit', AuthAuditService);
+    services.registerService('spending', TrackSpendingService);
+    services.registerService('counting', ConfigurableCountingService);
+    services.registerService('thumbnails', StrategizedService, {
+        strategy_key: 'engine',
+        strategies: {
+            napi: [NAPIThumbnailService],
+            purejs: [PureJSThumbnailService],
+            http: [HTTPThumbnailService],
+        }
+    });
+    services.registerService('__refresh-assocs', RefreshAssociationsService);
+    services.registerService('__prod-debugging', MakeProdDebuggingLessAwfulService);
+    if ( config.env == 'dev' ) {
+        services.registerService('dev-console', DevConsoleService);
+    }
+
+    const { EventService } = require('./services/EventService');
+    services.registerService('event', EventService);
+
+}
+
+const install_legacy = async ({ services }) => {
+    const { ProcessEventService } = require('./services/runtime-analysis/ProcessEventService');
+    const { ParameterService } = require('./services/ParameterService');
+    const { InformationService } = require('./services/information/InformationService');
+    const { FilesystemService } = require('./filesystem/FilesystemService');
+    const PerformanceMonitor = require('./monitor/PerformanceMonitor');
+    const { OperationTraceService } = require('./services/OperationTraceService');
+    const { WSPushService } = require('./services/WSPushService');
+    const { PuterVersionService } = require('./services/PuterVersionService');
+    const { ReferralCodeService } = require('./services/ReferralCodeService');
+    const { Emailservice } = require('./services/EmailService');
+    const { ClientOperationService } = require('./services/ClientOperationService');
+    const { EngPortalService } = require('./services/EngPortalService');
+    const { AppInformationService } = require('./services/AppInformationService');
+    const { FileCacheService } = require('./services/file-cache/FileCacheService');
+
+    // === Services which do not yet extend BaseService ===
+    services.registerService('process-event', ProcessEventService);
+    services.registerService('params', ParameterService);
+    services.registerService('information', InformationService)
+    services.registerService('filesystem', FilesystemService);
+    services.registerService('operationTrace', OperationTraceService);
+    services.registerService('__event-push-ws', WSPushService);
+    services.registerService('puter-version', PuterVersionService);
+    services.registerService('referral-code', ReferralCodeService);
+    services.registerService('email', Emailservice);
+    services.registerService('file-cache', FileCacheService);
+    services.registerService('client-operation', ClientOperationService);
+    services.registerService('app-information', AppInformationService);
+    services.registerService('engineering-portal', EngPortalService);
+    // TODO: add to here: ResourceService and DatabaseFSEntryService
+
+    // This singleton was made before services existed,
+    // so we have to pass that to it manually
+    PerformanceMonitor.provideServices(services);
+
+};

packages/backend/src/DatabaseModule.js

@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { AdvancedBase } = require("puter-js-common");
+
+class DatabaseModule extends AdvancedBase {
+    async install (context) {
+        const services = context.get('services');
+
+        const { StrategizedService } = require('./services/StrategizedService');
+        const { SqliteDatabaseAccessService } = require('./services/database/SqliteDatabaseAccessService');
+        services.registerService('database', StrategizedService, {
+            strategy_key: 'engine',
+            strategies: {
+                sqlite: [SqliteDatabaseAccessService],
+            }
+        })
+    }
+}
+
+module.exports = DatabaseModule;

packages/backend/src/Kernel.js

@@ -0,0 +1,189 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { AdvancedBase } = require("puter-js-common");
+const { Context } = require('./util/context');
+
+class Kernel extends AdvancedBase {
+    constructor () {
+        super();
+
+        this.modules = [];
+    }
+
+    add_module (module) {
+        this.modules.push(module);
+    }
+
+    _runtime_init () {
+        const kvjs = require('@heyputer/kv.js');
+        const kv = new kvjs();
+        global.kv = kv;
+        global.cl = console.log;
+
+        const { RuntimeEnvironment } = require('./boot/RuntimeEnvironment');
+        const { BootLogger } = require('./boot/BootLogger');
+
+        // Temporary logger for boot process;
+        // LoggerService will be initialized in app.js
+        const bootLogger = new BootLogger();
+
+        // Determine config and runtime locations
+        const runtimeEnv = new RuntimeEnvironment({
+            logger: bootLogger,
+        });
+        runtimeEnv.init();
+
+        // polyfills
+        require('./polyfill/to-string-higher-radix');
+    }
+
+    boot () {
+        this._runtime_init();
+
+        // const express = require('express')
+        // const app = express();
+        const config = require('./config');
+
+        globalThis.xtra_log = () => {};
+        if ( config.env === 'dev' ) {
+            globalThis.xtra_log = (...args) => {
+                // append to file in temp
+                const fs = require('fs');
+                const path = require('path');
+                const log_path = path.join('/tmp/xtra_log.txt');
+                fs.appendFileSync(log_path, args.join(' ') + '\n');
+            }
+        }
+
+        const { consoleLogManager } = require('./util/consolelog');
+        consoleLogManager.initialize_proxy_methods();
+
+        // TODO: temporary dependency inversion; requires moving:
+        //   - rm, so we can move mv
+        //   - mv, so we can move mkdir
+        //   - generate_default_fsentries, so we can move mkdir
+        //   - mkdir, which needs an fs provider
+
+        // === START: Initialize Service Registry ===
+        const { Container } = require('./services/Container');
+
+        const services = new Container();
+        this.services = services;
+        // app.set('services', services);
+
+        const root_context = Context.create({
+            services,
+            config,
+        }, 'app');
+        globalThis.root_context = root_context;
+
+        root_context.arun(async () => {
+            await this._install_modules();
+        });
+
+        (async () => {
+            await this._boot_services();
+        })();
+
+
+        // Error.stackTraceLimit = Infinity;
+        Error.stackTraceLimit = 200;
+    }
+
+    async _install_modules () {
+        const { services } = this;
+
+        for ( const module of this.modules ) {
+            await module.install(Context.get());
+        }
+
+        try {
+            await services.init();
+        } catch (e) {
+            // First we'll try to mark the system as invalid via
+            // SystemValidationService. This might fail because this service
+            // may not be initialized yet.
+
+            const svc_systemValidation = (() => {
+                try {
+                    return services.get('system-validation');
+                } catch (e) {
+                    return null;
+                }
+            })();
+
+            if ( ! svc_systemValidation ) {
+                // If we can't mark the system as invalid, we'll just have to
+                // throw the error and let the server crash.
+                throw e;
+            }
+
+            await svc_systemValidation.mark_invalid(
+                'failed to initialize services',
+                e,
+            );
+        }
+
+        for ( const module of this.modules ) {
+            await module.install_legacy?.(Context.get());
+        }
+
+        services.ready.resolve();
+        // provide services to helpers
+
+        const { tmp_provide_services } = require('./helpers');
+        tmp_provide_services(services);
+    }
+
+    async _boot_services () {
+        const { services } = this;
+
+        await services.ready;
+        {
+            const app = services.get('web-server').app;
+            app.use(async (req, res, next) => {
+                req.services = services;
+                next();
+            });
+            await services.emit('boot.services-initialized');
+            await services.emit('install.middlewares.context-aware', { app });
+            await services.emit('install.routes', { app });
+            await services.emit('install.routes-gui', { app });
+        }
+
+        // === END: Initialize Service Registry ===
+
+        // self check
+        (async () => {
+            await services.ready;
+            globalThis.services = services;
+            const log = services.get('log-service').create('init');
+            log.info('services ready');
+
+            log.system('server ready', {
+                deployment_type: globalThis.deployment_type,
+            });
+        })();
+
+
+        await services.emit('start.webserver');
+    }
+}
+
+module.exports = { Kernel };

packages/backend/src/LocalDiskStorageModule.js

@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { AdvancedBase } = require("puter-js-common");
+
+class LocalDiskStorageModule extends AdvancedBase {
+    async install (context) {
+        const services = context.get('services');
+        const LocalDiskStorageService = require("./services/LocalDiskStorageService");
+        services.registerService('local-disk-storage', LocalDiskStorageService);
+    }
+}
+
+module.exports = LocalDiskStorageModule;

packages/backend/src/PuterDriversModule.js

@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { AdvancedBase } = require("puter-js-common");
+
+class PuterDriversModule extends AdvancedBase {
+    async install () {}
+    async install_legacy (context) {
+        const services = context.get('services');
+
+        const { DriverService } = require("./services/drivers/DriverService");
+        services.registerService('driver', DriverService);
+    }
+}
+
+module.exports = PuterDriversModule;

packages/backend/src/ThirdPartyDriversModule.js

@@ -0,0 +1,23 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { AdvancedBase } = require("puter-js-common");
+
+class ThirdPartyDriversModule extends AdvancedBase {
+    // constructor () {
+}

packages/backend/src/api/APIError.js

@@ -0,0 +1,459 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { quot } = require("../util/strutil");
+
+/**
+ * APIError represents an error that can be sent to the client.
+ * @class APIError
+ * @property {number} status the HTTP status code
+ * @property {string} message the error message
+ * @property {object} source the source of the error
+ */
+module.exports = class APIError {
+    static codes = {
+        'item_with_same_name_exists': {
+            status: 409,
+            message: ({ entry_name }) => entry_name
+                ? `An item with name ${quot(entry_name)} already exists.`
+                : 'An item with the same name already exists.'
+                ,
+        },
+        'cannot_move_item_into_itself': {
+            status: 422,
+            message: 'Cannot move an item into itself.',
+        },
+        'cannot_copy_item_into_itself': {
+            status: 422,
+            message: 'Cannot copy an item into itself.',
+        },
+        'cannot_move_to_root': {
+            status: 422,
+            message: 'Cannot move an item to the root directory.',
+        },
+        'cannot_copy_to_root': {
+            status: 422,
+            message: 'Cannot copy an item to the root directory.',
+        },
+        'cannot_write_to_root': {
+            status: 422,
+            message: 'Cannot write an item to the root directory.',
+        },
+        'cannot_overwrite_a_directory': {
+            status: 422,
+            message: 'Cannot overwrite a directory.',
+        },
+        'cannot_read_a_directory': {
+            status: 422,
+            message: 'Cannot read a directory.',
+        },
+        'source_and_dest_are_the_same': {
+            status: 422,
+            message: 'Source and destination are the same.',
+        },
+        'dest_is_not_a_directory': {
+            status: 422,
+            message: 'Destination must be a directory.',
+        },
+        'dest_does_not_exist': {
+            status: 422,
+            message: 'Destination was not found.',
+        },
+        'source_does_not_exist': {
+            status: 404,
+            message: 'Source was not found.',
+        },
+        'subject_does_not_exist': {
+            status: 404,
+            message: 'File or directory not found.',
+        },
+        'shortcut_target_not_found': {
+            status: 404,
+            message: 'Shortcut target not found.',
+        },
+        'shortcut_target_is_a_directory': {
+            status: 422,
+            message: 'Shortcut target is a directory; expected a file.',
+        },
+        'shortcut_target_is_a_file': {
+            status: 422,
+            message: 'Shortcut target is a file; expected a directory.',
+        },
+        'forbidden': {
+            status: 403,
+            message: 'Permission denied.',
+        },
+        'immutable': {
+            status: 403,
+            message: 'File is immutable.',
+        },
+        'field_empty': {
+            status: 400,
+            message: ({ key }) => `Field ${quot(key)} is required.`,
+        },
+        'field_missing': {
+            status: 400,
+            message: ({ key }) => `Field ${quot(key)} is required.`,
+        },
+        'xor_field_missing': {
+            status: 400,
+            message: ({ names }) => {
+                let s = 'One of these mutually-exclusive fields is required: ';
+                s += names.map(quot).join(', ');
+                return s;
+            }
+        },
+        'field_only_valid_with_other_field': {
+            status: 400,
+            message: ({ key, other_key }) => `Field ${quot(key)} is only valid when field ${quot(other_key)} is specified.`,
+        },
+        'invalid_id': {
+            status: 400,
+            message: ({ id }) => {
+                return `Invalid id`;
+            }
+        },
+        'field_invalid': {
+            status: 400,
+            message: ({ key, expected, got }) => {
+                return `Field ${quot(key)} is invalid.` +
+                    (expected ? ` Expected ${expected}.` : '') +
+                    (got ? ` Got ${got}.` : '')
+            }
+        },
+        'field_immutable': {
+            status: 400,
+            message: ({ key }) => `Field ${quot(key)} is immutable.`,
+        },
+        'field_too_long': {
+            status: 400,
+            message: ({ key, max_length }) => `Field ${quot(key)} is too long. Max length is ${max_length}.`,
+        },
+        'field_too_short': {
+            status: 400,
+            message: ({ key, min_length }) => `Field ${quot(key)} is too short. Min length is ${min_length}.`,
+        },
+        'already_in_use': {
+            status: 409,
+            message: ({ what, value }) => `The ${what} ${quot(value)} is already in use.`,
+        },
+        'invalid_file_name': {
+            status: 400,
+            message: ({ name, reason }) => `Invalid file name: ${quot(name)}${reason ? `; ${reason}` : '.'}`,
+        },
+        'storage_limit_reached': {
+            status: 400,
+            message: 'Storage capacity limit reached.',
+        },
+        'internal_error': {
+            status: 500,
+            message: 'An internal error occurred.',
+        },
+        'response_timeout': {
+            status: 504,
+            message: 'Response timed out.',
+        },
+        'file_too_large': {
+            status: 413,
+            message: ({ max_size }) => `File too large. Max size is ${max_size} bytes.`,
+        },
+        'thumbnail_too_large': {
+            status: 413,
+            message: ({ max_size }) => `Thumbnail too large. Max size is ${max_size} bytes.`,
+        },
+        'upload_failed': {
+            status: 500,
+            message: 'Upload failed.',
+        },
+        'missing_expected_metadata': {
+            status: 400,
+            message: ({ keys }) => `These fields must come first: ${(keys ?? []).map(quot).join(', ')}.`,
+        },
+        'overwrite_and_dedupe_exclusive': {
+            status: 400,
+            message: 'Cannot specify both overwrite and dedupe_name.',
+        },
+        'not_empty': {
+            status: 422,
+            message: 'Directory is not empty.',
+        },
+
+        // Write
+        'offset_without_existing_file': {
+            status: 404,
+            message: 'An offset was specified, but the file doesn\'t exist.',
+        },
+        'offset_requires_overwrite': {
+            status: 400,
+            message: 'An offset was specified, but overwrite conditions were not met.',
+        },
+        'offset_requires_stream': {
+            status: 400,
+            message: 'The offset option for write is not available for this upload.'
+        },
+
+        // Batch
+        'batch_too_many_files': {
+            status: 400,
+            message: 'Received an extra file with no corresponding operation.',
+        },
+        'batch_missing_file': {
+            status: 400,
+            message: 'Missing fileinfo entry or BLOB for operation.',
+        },
+
+        // Open
+        'no_suitable_app': {
+            status: 422,
+            message: ({ entry_name }) => `No suitable app found for ${quot(entry_name)}.`,
+        },
+        'app_does_not_exist': {
+            status: 422,
+            message: ({ identifier }) => `App ${quot(identifier)} does not exist.`,
+        },
+
+        // Apps
+        'app_name_already_in_use': {
+            status: 409,
+            message: ({ name }) => `App name ${quot(name)} is already in use.`,
+        },
+
+        // Subdomains
+        'subdomain_limit_reached': {
+            status: 400,
+            message: ({ limit }) => `You have exceeded the number of subdomains under your current plan (${limit}).`,
+        },
+        'subdomain_reserved': {
+            status: 400,
+            message: ({ subdomain }) => `Subdomain ${quot(subdomain)} is not available.`,
+        },
+
+        // Users
+        'email_already_in_use': {
+            status: 409,
+            message: ({ email }) => `Email ${quot(email)} is already in use.`,
+        },
+        'username_already_in_use': {
+            status: 409,
+
+            message: ({ username }) => `Username ${quot(username)} is already in use.`,
+        },
+        'too_many_username_changes': {
+            status: 429,
+            message: 'Too many username changes this month.',
+        },
+        'token_invalid': {
+            status: 400,
+            message: () => 'Invalid token.',
+        },
+
+        // drivers
+        'interface_not_found': {
+            status: 404,
+            message: ({ interface_name }) => `Interface not found: ${quot(interface_name)}`,
+        },
+        'no_implementation_available': {
+            status: 502,
+            message: ({ interface_name }) => `No implementation available for interface ${quot(interface_name)}`,
+        },
+        'method_not_found': {
+            status: 404,
+            message: ({ interface_name, method_name }) => `Method not found: ${quot(method_name)} on interface ${quot(interface_name)}`,
+        },
+        'missing_required_argument': {
+            status: 400,
+            message: ({ interface_name, method_name, arg_name }) =>
+                `Missing required argument ${quot(arg_name)} for method ${quot(method_name)} on interface ${quot(interface_name)}`,
+        },
+        'argument_consolidation_failed': {
+            status: 400,
+            message: ({ interface_name, method_name, arg_name, message }) =>
+                `Failed to parse or process argument ${quot(arg_name)} for method ${quot(method_name)} on interface ${quot(interface_name)}: ${message}`,
+        },
+
+        // SLA
+        'rate_limit_exceeded': {
+            status: 429,
+            message: ({ method_name, rate_limit }) =>
+                `Rate limit exceeded for method ${quot(method_name)}: ${rate_limit.max} requests per ${rate_limit.period}ms.`,
+        },
+        'monthly_limit_exceeded': {
+            status: 429,
+            message: ({ method_key, limit }) =>
+                `Monthly limit exceeded for method ${quot(method_key)}: ${limit} requests per month.`,
+        },
+        'server_rate_exceeded': {
+            status: 503,
+            message: 'System-wide rate limit exceeded. Please try again later.',
+        },
+
+        // auth
+        'token_missing': {
+            status: 401,
+            message: 'Missing authentication token.',
+        },
+        'token_auth_failed': {
+            status: 401,
+            message: 'Authentication failed.',
+        },
+        'token_unsupported': {
+            status: 401,
+            message: 'This authentication token is not supported here.',
+        },
+        'account_suspended': {
+            status: 403,
+            message: 'Account suspended.',
+        },
+        'permission_denied': {
+            status: 403,
+            message: 'Permission denied.',
+        },
+        'access_token_empty_permissions': {
+            status: 403,
+            message: 'Attempted to create an access token with no permissions.',
+        },
+
+        // Object Mapping
+        'field_not_allowed_for_create': {
+            status: 400,
+            message: ({ key }) => `Field ${quot(key)} is not allowed for create.`,
+        },
+        'field_required_for_update': {
+            status: 400,
+            message: ({ key }) => `Field ${quot(key)} is required for update.`,
+        },
+        'entity_not_found': {
+            status: 422,
+            message: ({ identifier }) => `Entity not found: ${quot(identifier)}`,
+        },
+
+        // Chat
+        // TODO: specifying these errors here might be a violation
+        // of separation of concerns. Services could register their
+        // own errors with an error registry.
+        'max_tokens_exceeded': {
+            status: 400,
+            message: ({ input_tokens, max_tokens }) =>
+                `Input exceeds maximum token count. ` +
+                `Input has ${input_tokens} tokens, ` +
+                `but the maximum is ${max_tokens}.`,
+        },
+    };
+
+    /**
+     * create() is a factory method for creating APIError instances.
+     * It accepts either a string or an Error object as the second
+     * argument. If a string is passed, it is used as the error message.
+     * If an Error object is passed, its message property is used as the
+     * error message. The Error object itself is stored in the source
+     * property. If no second argument is passed, the source property
+     * is set to null. The first argument is used as the status code.
+     *
+     * @static
+     * @param {number} status
+     * @param {string|Error} message_or_source one of the following:
+     * - a string to use as the error message
+     * - an Error object to use as the source of the error
+     * - an object with a message property to use as the error message
+     * @returns
+     */
+    static create (status, source, fields = {}) {
+        // Just the error code
+        if ( typeof status === 'string' ) {
+            const code = this.codes[status];
+            if ( ! code ) {
+                return new APIError(500, 'Missing error message.', null, {
+                    code: status
+                });
+            }
+            return new APIError(code.status, status, source, fields);
+        }
+
+        // High-level errors like this: APIError.create(400, '...')
+        if ( typeof source === 'string' ) {
+            return new APIError(status, source, null, fields);
+        }
+
+        // Errors from source like this: throw new Error('...')
+        if (
+            typeof source === 'object' &&
+            source instanceof Error
+        ) {
+            return new APIError(status, source?.message, source, fields);
+        }
+
+        // Errors from sources like this: throw { message: '...', ... }
+        if (
+            typeof source === 'object' &&
+            source.constructor.name === 'Object' &&
+            source.hasOwnProperty('message')
+        ) {
+            const allfields = { ...source, ...fields };
+            return new APIError(status, source.message, source, allfields);
+        }
+
+        console.error('Invalid APIError source:', source);
+        return new APIError(500, 'Internal Server Error', null, {});
+    }
+    static adapt (err) {
+        if ( err instanceof APIError ) return err;
+
+        return APIError.create(`internal_error`);
+    }
+    constructor (status, message, source, fields = {}) {
+        this.codes = this.constructor.codes;
+        this.status = status;
+        this._message = message;
+        this.source = source ?? new Error('error for trace');
+        this.fields = fields;
+
+        if ( this.codes.hasOwnProperty(message) ) {
+            this.fields.code = message;
+            this._message = this.codes[message].message;
+        }
+    }
+    write (res) {
+        const message = typeof this.message === 'function'
+            ? this.message(this.fields)
+            : this.message;
+        return res.status(this.status).send({
+            message,
+            ...this.fields,
+        });
+    }
+    serialize () {
+        console.log('MESSAGE FROM ERROR: ' + `|${this.message}|`);
+        return {
+            ...this.fields,
+            $: 'heyputer:api/APIError',
+            message: this.message,
+            status: this.status,
+        };
+    }
+
+    get message () {
+        const message = typeof this._message === 'function'
+            ? this._message(this.fields)
+            : this._message;
+        return message;
+    }
+
+    toString () {
+        return `APIError(${this.status}, ${this.message})`;
+    }
+}

packages/backend/src/api/PathOrUIDValidator.js

@@ -0,0 +1,56 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const APIError = require('./APIError');
+const _path = require('path');
+
+/**
+ * PathOrUIDValidator validates that either `path` or `uid` is present
+ * in the request and requires a valid value for the parameter that was
+ * used. Additionally, resolves the path if a path was provided.
+ * 
+ * @class PathOrUIDValidator
+ * @static
+ * @throws {APIError} if `path` and `uid` are both missing
+ * @throws {APIError} if `path` and `uid` are both present
+ * @throws {APIError} if `path` is not a string
+ * @throws {APIError} if `path` is empty
+ * @throws {APIError} if `uid` is not a valid uuid
+ */
+module.exports = class PathOrUIDValidator {
+    static validate (req) {
+        const params = req.method === 'GET'
+            ? req.query : req.body ;
+
+        if(!params.path && !params.uid)
+            throw new APIError(400, '`path` or `uid` must be provided.');
+        // `path` must be a string
+        else if (params.path && !params.uid && typeof params.path !== 'string')
+            throw new APIError(400, '`path` must be a string.');
+        // `path` cannot be empty
+        else if(params.path && !params.uid && params.path.trim() === '')
+            throw new APIError(400, '`path` cannot be empty');
+        // `uid` must be a valid uuid
+        else if(params.uid && !params.path && !require('uuid').validate(params.uid))
+            throw new APIError(400, '`uid` must be a valid uuid');
+
+        // resolve path if provided
+        if(params.path)
+            params.path = _path.resolve('/', params.path);
+    }
+};

packages/backend/src/api/api_error_handler.js

@@ -0,0 +1,78 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const APIError = require("./APIError");
+
+/**
+ * api_error_handler() is an express error handler for API errors.
+ * It adheres to the express error handler signature and should be
+ * used as the last middleware in an express app.
+ * 
+ * Since Express 5 is not yet released, this function is used by
+ * eggspress() to handle errors instead of as a middleware.
+ * 
+ * @todo remove this function and use express error handling
+ * when Express 5 is released
+ * 
+ * @param {*} err 
+ * @param {*} req 
+ * @param {*} res 
+ * @param {*} next 
+ * @returns 
+ */
+module.exports = function (err, req, res, next) {
+        if (res.headersSent) {
+            console.error('error after headers were sent:', err);
+            return next(err)
+        }
+
+        // API errors might have a response to help the
+        // developer resolve the issue.
+        if ( err instanceof APIError ) {
+            return err.write(res);
+        }
+
+        if (
+            typeof err === 'object' &&
+            ! (err instanceof Error) &&
+            err.hasOwnProperty('message')
+        ) {
+            const apiError = APIError.create(400, err);
+            return apiError.write(res);
+        }
+
+        console.error('internal server error:', err);
+
+        const services = globalThis.services;
+        if ( services && services.has('alarm') ) {
+            const alarm = services.get('alarm');
+            alarm.create('api_error_handler', err.message, {
+                error: err,
+                url: req.url,
+                method: req.method,
+                body: req.body,
+                headers: req.headers,
+            });
+        }
+
+        req.__error_handled = true;
+        
+        // Other errors should provide as little information
+        // to the client as possible for security reasons.
+        return res.send(500, 'Internal Server Error');
+};

packages/backend/src/api/eggspress.js

@@ -0,0 +1,198 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const express = require('express');
+const multer = require('multer');
+const multest = require('@heyputer/multest');
+const api_error_handler = require('../api/api_error_handler.js');
+
+const fsBeforeMW = require('../middleware/fs');
+const APIError = require('./APIError.js');
+const { Context } = require('../util/context.js');
+
+/**
+ * eggspress() is a factory function for creating express routers.
+ *
+ * @param {*} route the route to the router
+ * @param {*} settings the settings for the router. The following
+ *  properties are supported:
+ * - auth: whether or not to use the auth middleware
+ * - fs: whether or not to use the fs middleware
+ * - json: whether or not to use the json middleware
+ * - customArgs: custom arguments to pass to the router
+ * - allowedMethods: the allowed HTTP methods
+ * @param {*} handler the handler for the router
+ * @returns {express.Router} the router
+ */
+module.exports = function eggspress (route, settings, handler) {
+  const router = express.Router();
+  const mw = [];
+  const afterMW = [];
+
+  // These flags enable specific middleware.
+  if ( settings.abuse ) mw.push(require('../middleware/abuse')(settings.abuse));
+  if ( settings.auth ) mw.push(require('../middleware/auth'));
+  if ( settings.auth2 ) mw.push(require('../middleware/auth2'));
+  if ( settings.fs ) {
+    mw.push(fsBeforeMW);
+  }
+  if ( settings.verified ) mw.push(require('../middleware/verified'));
+  if ( settings.json ) mw.push(express.json());
+
+  // The `files` setting is an array of strings. Each string is the name
+  // of a multipart field that contains files. `multer` is used to parse
+  // the multipart request and store the files in `req.files`.
+  if ( settings.files ) {
+    for ( const key of settings.files ) {
+      mw.push(multer().array(key));
+    }
+  }
+
+  if ( settings.multest ) {
+    mw.push(multest());
+  }
+
+  // The `multipart_jsons` setting is an array of strings. Each string
+  // is the name of a multipart field that contains JSON. This middleware
+  // parses the JSON in each field and stores the result in `req.body`.
+  if ( settings.multipart_jsons ) {
+    for ( const key of settings.multipart_jsons ) {
+      mw.push((req, res, next) => {
+        try {
+          if ( ! Array.isArray(req.body[key]) ) {
+            req.body[key] = [JSON.parse(req.body[key])];
+          } else {
+            req.body[key] = req.body[key].map(JSON.parse);
+          }
+        } catch (e) {
+          return res.status(400).send({
+            error: {
+              message: `Invalid JSON in multipart field ${key}`
+            }
+          });
+        }
+        next();
+      });
+    }
+  }
+
+  // The `alias` setting is an object. Each key is the name of a
+  // parameter. Each value is the name of a parameter that should
+  // be aliased to the key.
+  if ( settings.alias ) {
+    for ( const alias in settings.alias ) {
+      const target = settings.alias[alias];
+      mw.push((req, res, next) => {
+        const values = req.method === 'GET' ? req.query : req.body;
+        if ( values[alias] ) {
+          values[target] = values[alias];
+        }
+        next();
+      });
+    }
+  }
+
+  // The `parameters` setting is an object. Each key is the name of a
+  // parameter. Each value is a `Param` object. The `Param` object
+  // specifies how to validate the parameter.
+  if ( settings.parameters ) {
+    for ( const key in settings.parameters ) {
+      const param = settings.parameters[key];
+      mw.push(async (req, res, next) => {
+        if ( ! req.values ) req.values = {};
+
+        const values = req.method === 'GET' ? req.query : req.body;
+        const getParam = (key) => values[key];
+        try {
+          const result = await param.consolidate({ req, getParam });
+          req.values[key] = result;
+        } catch (e) {
+          api_error_handler(e, req, res, next);
+          return;
+        }
+        next();
+      });
+    }
+  }
+
+  // what if I wanted to pass arguments to, for example, `json`?
+  if ( settings.customArgs ) mw.push(settings.customArgs);
+
+  if ( settings.alarm_timeout ) {
+    mw.push((req, res, next) => {
+      setTimeout(() => {
+        if ( ! res.headersSent ) {
+          const log = req.services.get('log-service').create('eggspress:timeout');
+          const errors = req.services.get('error-service').create(log);
+          let id = Array.isArray(route) ? route[0] : route;
+          id = id.replace(/\//g, '_');
+          errors.report(id, {
+            source: new Error('Response timed out.'),
+            message: 'Response timed out.',
+            trace: true,
+            alarm: true,
+          });
+        }
+      }, settings.alarm_timeout);
+      next();
+    });
+  }
+
+  if ( settings.response_timeout ) {
+    mw.push((req, res, next) => {
+      setTimeout(() => {
+        if ( ! res.headersSent ) {
+          api_error_handler(APIError.create('response_timeout'), req, res, next);
+        }
+      }, settings.response_timeout);
+      next();
+    });
+  }
+
+  if ( settings.mw ) mw.push(...settings.mw);
+
+  const errorHandledHandler = async function (req, res, next) {
+    if ( settings.subdomain ) {
+      if ( require('../helpers').subdomain(req) !== settings.subdomain ) {
+        return next();
+      }
+    }
+    try {
+      const expected_ctx = res.locals.ctx;
+      const received_ctx = Context.get(undefined, { allow_fallback: true });
+
+      if ( expected_ctx != received_ctx ) {
+        await expected_ctx.arun(async () => {
+          await handler(req, res, next);
+        });
+      } else await handler(req, res, next);
+    } catch (e) {
+        api_error_handler(e, req, res, next);
+    }
+  };
+
+  if ( settings.allowedMethods.includes('GET') ) {
+    router.get(route, ...mw, errorHandledHandler, ...afterMW);
+  }
+
+  if ( settings.allowedMethods.includes('POST') ) {
+    router.post(route, ...mw, errorHandledHandler, ...afterMW);
+  }
+
+  return router;
+}

packages/backend/src/api/filesystem/FSNodeParam.js

@@ -0,0 +1,78 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { is_valid_path } = require("../../filesystem/validation");
+const { is_valid_uuid4 } = require("../../helpers");
+const { Context } = require("../../util/context");
+const APIError = require("../APIError");
+const _path = require('path');
+
+module.exports = class FSNodeParam {
+    constructor (srckey, options) {
+        this.srckey = srckey;
+        this.options = options ?? {};
+        this.optional = this.options.optional ?? false;
+    }
+
+    async consolidate ({ req, getParam }) {
+        const log = globalThis.services.get('log-service').create('fsnode-param');
+        const fs = req.fs ?? Context.get('services').get('filesystem');
+
+        let uidOrPath = getParam(this.srckey);
+        if ( uidOrPath === undefined ) {
+            if ( this.optional ) return undefined;
+            throw APIError.create('field_missing', null, {
+                key: this.srckey,
+            });
+        }
+
+        if ( uidOrPath.length === 0 ) {
+            if ( this.optional ) return undefined;
+            APIError.create('field_empty', null, {
+                key: this.srckey,
+            });
+        }
+
+        if ( ! ['/','.','~'].includes(uidOrPath[0]) ) {
+            if ( is_valid_uuid4(uidOrPath) ) {
+                return await fs.node({ uid: uidOrPath });
+            }
+
+            log.debug('tried uuid', { uidOrPath })
+            throw APIError.create('field_invalid', null, {
+                key: this.srckey,
+                expected: 'unix-style path or uuid4',
+            });
+        }
+
+        if ( uidOrPath.startsWith('~') && req.user ) {
+            const homedir = `/${req.user.username}`;
+            uidOrPath = homedir + uidOrPath.slice(1);
+        }
+
+        if ( ! is_valid_path(uidOrPath) ) {
+            log.debug('tried path', { uidOrPath })
+            throw APIError.create('field_invalid', null, {
+                key: this.srckey,
+                expected: 'unix-style path or uuid4',
+            });
+        }
+
+        return await fs.node({ path: _path.resolve('/', uidOrPath) });
+    }
+}

packages/backend/src/api/filesystem/FlagParam.js

@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+module.exports = class FlagParam {
+    constructor (srckey, options) {
+        this.srckey = srckey;
+        this.options = options ?? {};
+        this.optional = this.options.optional ?? false;
+        this.default = this.options.default ?? false;
+    }
+
+    async consolidate ({ req, getParam }) {
+        const log = globalThis.services.get('log-service').create('flag-param');
+
+        const value = getParam(this.srckey);
+        if ( value === undefined || value === '' ) {
+            if ( this.optional ) return this.default;
+            throw APIError.create('field_missing', null, {
+                key: this.srckey,
+            });
+        }
+
+        if ( typeof value === 'string' ) {
+            if (
+                value === 'true' || value === '1' || value === 'yes'
+            ) return true;
+
+            if (
+                value === 'false' || value === '0' || value === 'no'
+            ) return false;
+
+            throw APIError.create('field_invalid', null, {
+                key: this.srckey,
+                expected: 'boolean',
+            });
+        }
+
+        if ( typeof value === 'boolean' ) {
+            return value;
+        }
+
+        log.debug('tried boolean', { value })
+        throw APIError.create('field_invalid', null, {
+            key: this.srckey,
+            expected: 'boolean',
+        });
+    }
+}

packages/backend/src/api/filesystem/StringParam.js

@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+module.exports = class StringParam {
+    constructor (srckey, options) {
+        this.srckey = srckey;
+        this.options = options ?? {};
+        this.optional = this.options.optional ?? false;
+    }
+
+    async consolidate ({ req, getParam }) {
+        const log = globalThis.services.get('log-service').create('string-param');
+
+        const value = getParam(this.srckey);
+        if ( value === undefined ) {
+            if ( this.optional ) return undefined;
+            throw APIError.create('field_missing', null, {
+                key: this.srckey,
+            });
+        }
+
+        if ( value.length === 0 ) {
+            if ( this.optional ) return undefined;
+            APIError.create('field_empty', null, {
+                key: this.srckey,
+            });
+        }
+
+        if ( typeof value !== 'string' ) {
+            log.debug('tried string', { value })
+            throw APIError.create('field_invalid', null, {
+                key: this.srckey,
+                expected: 'string',
+            });
+        }
+
+        return value;
+    }
+}

packages/backend/src/api/filesystem/UserParam.js

@@ -0,0 +1,26 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+module.exports = class UserParam {
+    constructor () {
+        //
+    }
+    consolidate ({ req }) {
+        return req.user;
+    }
+}

packages/backend/src/app.js

@@ -0,0 +1,19 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+"use strict"

packages/backend/src/boot/BootLogger.js

@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+class BootLogger {
+    constructor () {
+        console.log(
+            `\x1B[36;1mBoot logger started :)\x1B[0m`,
+        );
+    }
+    info (...args) {
+        console.log(
+            '\x1B[36;1m[BOOT/INFO]\x1B[0m',
+            ...args,
+        );
+    }
+    error (...args) {
+        console.log(
+            '\x1B[31;1m[BOOT/ERROR]\x1B[0m',
+            ...args,
+        );
+    }
+}
+
+module.exports = {
+    BootLogger,
+};

packages/backend/src/boot/RuntimeEnvironment.js

@@ -0,0 +1,304 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { AdvancedBase } = require("puter-js-common");
+const { quot } = require("../util/strutil");
+const { TechnicalError } = require("../errors/TechnicalError");
+const { print_error_help } = require("../errors/error_help_details");
+const default_config = require("./default_config");
+const config = require("../config");
+const { ConfigLoader } = require("../config/ConfigLoader");
+
+// highlights a string
+const hl = s => `\x1b[33;1m${s}\x1b[0m`;
+
+// Save the original working directory
+const original_cwd = process.cwd();
+
+// === [ Puter Runtime Environment ] ===
+// This file contains the RuntimeEnvironment class which is
+// responsible for locating the configuration and runtime
+// directories for the Puter Kernel.
+
+// Depending on which path we're checking for configuration
+// or runtime from config_paths, there will be different
+// requirements. These are all possible requirements.
+//
+// Each check may result in the following:
+// - false: this is not the desired path; skip it
+// - true:  this is the desired path, and it's valid
+// - throw: this is the desired path, but it's invalid
+const path_checks = ({ logger }) => ({ fs, path_ }) => ({
+    require_if_not_undefined: ({ path }) => {
+        if ( path == undefined ) return false;
+
+        const exists = fs.existsSync(path);
+        if ( !exists ) {
+            throw new Error(`Path does not exist: ${path}`);
+        }
+
+        return true;
+    },
+    skip_if_not_exists: ({ path }) => {
+        const exists = fs.existsSync(path);
+        return exists;
+    },
+    skip_if_not_in_repo: ({ path }) => {
+        const exists = fs.existsSync(path_.join(path, '../../.is_puter_repository'));
+        return exists;
+    },
+    require_read_permission: ({ path }) => {
+        try {
+            fs.readdirSync(path);
+        } catch (e) {
+            throw new Error(`Cannot readdir on path: ${path}`);
+        }
+        return true;
+    },
+    require_write_permission: ({ path }) => {
+        try {
+            fs.writeFileSync(path_.join(path, '.tmp_test_write_permission'), 'test');
+            fs.unlinkSync(path_.join(path, '.tmp_test_write_permission'));
+        } catch (e) {
+            throw new Error(`Cannot write to path: ${path}`);
+        }
+        return true;
+    },
+    contains_config_file: ({ path }) => {
+        const valid_config_names = [
+            'config.json',
+            'config.json5',
+        ];
+        for ( const name of valid_config_names ) {
+            const exists = fs.existsSync(path_.join(path, name));
+            if ( exists ) {
+                return true;
+            }
+        }
+        throw new Error(`No valid config file found in path: ${path}`);
+    },
+    env_not_set: name => () => {
+        if ( process.env[name] ) return false;
+        return true;
+    }
+});
+
+// Configuration paths in order of precedence.
+// We will load configuration from the first path that's suitable.
+const config_paths = ({ path_checks }) => ({ path_ }) => [
+    {
+        label: '$CONFIG_PATH',
+        get path () { return process.env.CONFIG_PATH },
+        checks: [
+            path_checks.require_if_not_undefined,
+        ],
+    },
+    {
+        path: '/etc/puter',
+        checks: [ path_checks.skip_if_not_exists ],
+    },
+    {
+        get path () {
+            return path_.join(original_cwd, 'volatile/config');
+        },
+        checks: [ path_checks.skip_if_not_in_repo ],
+    },
+    {
+        get path () {
+            return path_.join(original_cwd, 'config');
+        },
+        checks: [ path_checks.skip_if_not_exists ],
+    },
+];
+
+const valid_config_names = [
+    'config.json',
+    'config.json5',
+];
+
+// Suitable working directories in order of precedence.
+// We will `process.chdir` to the first path that's suitable.
+const runtime_paths = ({ path_checks }) => ({ path_ }) => [
+    {
+        label: '$RUNTIME_PATH',
+        get path () { return process.env.RUNTIME_PATH },
+        checks: [
+            path_checks.require_if_not_undefined,
+        ],
+    },
+    {
+        path: '/var/puter',
+        checks: [
+            path_checks.skip_if_not_exists,
+            path_checks.env_not_set('NO_VAR_RUNTIME'),
+        ],
+    },
+    {
+        get path () {
+            return path_.join(original_cwd, 'volatile/runtime');
+        },
+        checks: [ path_checks.skip_if_not_in_repo ],
+    },
+    {
+        get path () {
+            return path_.join(original_cwd, 'runtime');
+        },
+        checks: [ path_checks.skip_if_not_exists ],
+    },
+];
+
+class RuntimeEnvironment extends AdvancedBase {
+    static MODULES = {
+        fs: require('node:fs'),
+        path_: require('node:path'),
+        crypto: require('node:crypto'),
+    }
+
+    constructor ({ logger }) {
+        super();
+        this.logger = logger;
+        this.path_checks = path_checks(this)(this.modules);
+        this.config_paths = config_paths(this)(this.modules);
+        this.runtime_paths = runtime_paths(this)(this.modules);
+    }
+
+    init () {
+        try {
+            this.init_();
+        } catch (e) {
+            this.logger.error(e);
+            print_error_help(e);
+            process.exit(1);
+        }
+    }
+
+    init_ () {
+        const config_path_entry = this.get_first_suitable_path_(
+            { pathFor: 'configuration' },
+            this.config_paths,
+            [
+                this.path_checks.require_read_permission,
+                // this.path_checks.contains_config_file,
+            ]
+        );
+
+        const pwd_path_entry = this.get_first_suitable_path_(
+            { pathFor: 'working directory' },
+            this.runtime_paths,
+            [ this.path_checks.require_write_permission ]
+        );
+
+        process.chdir(pwd_path_entry.path);
+
+        // Check for a valid config file in the config path
+        let using_config;
+        for ( const name of valid_config_names ) {
+            const exists = this.modules.fs.existsSync(
+                this.modules.path_.join(config_path_entry.path, name)
+            );
+            if ( exists ) {
+                using_config = name;
+                break;
+            }
+        }
+
+        const { fs, path_, crypto } = this.modules;
+        let config_values = {};
+        if ( !using_config ) {
+            const generated_config = {
+                ...default_config,
+            };
+            generated_config.cookie_name = crypto.randomUUID();
+            generated_config.jwt_secret = crypto.randomUUID();
+            generated_config.url_signature_secret = crypto.randomUUID();
+            generated_config[""] = null; // for trailing comma
+            fs.writeFileSync(
+                path_.join(config_path_entry.path, 'config.json'),
+                JSON.stringify(generated_config, null, 4) + '\n',
+            );
+            using_config = 'config.json';
+        }
+
+        let config_to_load = 'config.json';
+        if ( process.env.PUTER_CONFIG_PROFILE ) {
+            this.logger.info(
+                hl('PROFILE') + ' ' +
+                quot(process.env.PUTER_CONFIG_PROFILE) + ' ' +
+                `because $PUTER_CONFIG_PROFILE is set`
+            );
+            config_to_load = `${process.env.PUTER_CONFIG_PROFILE}.json`
+            const exists = fs.existsSync(
+                path_.join(config_path_entry.path, config_to_load)
+            );
+            if ( ! exists ) {
+                fs.writeFileSync(
+                    path_.join(config_path_entry.path, config_to_load),
+                    JSON.stringify({
+                        config_name: process.env.PUTER_CONFIG_PROFILE,
+                        $imports: ['config.json'],
+                    }, null, 4) + '\n',
+                );
+            }
+        }
+
+        const loader = new ConfigLoader(this.logger, config_path_entry.path, config);
+        loader.enable(config_to_load);
+
+        if ( ! config.config_name ) {
+            throw new Error('config_name is required');
+        }
+        this.logger.info(hl(`config name`) + ` ${quot(config.config_name)}`);
+        // console.log(config.services);
+        // console.log(Object.keys(config.services));
+        // console.log({ ...config.services });
+    }
+
+    get_first_suitable_path_ (meta, paths, last_checks) {
+        iter_paths:
+        for ( const entry of paths ) {
+            const checks = [...(entry.checks ?? []), ...last_checks];
+            this.logger.info(
+                `Checking path ${quot(entry.label ?? entry.path)} for ${meta.pathFor}...`
+            );
+            for ( const check of checks ) {
+                this.logger.info(
+                    `-> doing ${quot(check.name)} on path ${quot(entry.path)}...`
+                );
+                const result = check(entry);
+                if ( result === false ) {
+                    this.logger.info(
+                        `-> ${quot(check.name)} doesn't like this path`
+                    );
+                    continue iter_paths;
+                }
+            }
+
+            this.logger.info(
+                `${hl('USING')} ${quot(entry.path)} for ${meta.pathFor}.`
+            )
+
+            return entry;
+        }
+
+        throw new TechnicalError(`No suitable path found for ${meta.pathFor}.`);
+    }
+}
+
+module.exports = {
+    RuntimeEnvironment,
+};

packages/backend/src/boot/default_config.js

@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+module.exports = {
+    config_name: 'generated default config',
+    env: 'dev',
+    nginx_mode: true, // really means "serve http instead of https"
+    server_id: 'localhost',
+    http_port: 'auto',
+    domain: 'puter.localhost',
+    protocol: 'http',
+    contact_email: 'hey@example.com',
+
+    services: {
+        database: {
+            engine: 'sqlite',
+            path: 'puter-database.sqlite',
+        },
+        thumbnails: {
+            engine: 'purejs'
+        },
+        'file-cache': {
+            disk_limit: 16384,
+            disk_max_size: 16384,
+            precache_size: 16384,
+            path: './file-cache',
+
+        }
+    },
+};

packages/backend/src/codex/CodeModel.js

@@ -0,0 +1,25 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+class CodeModel {
+    static create () {}
+}
+
+module.exports = {
+    CodeModel,
+};

packages/backend/src/codex/CodeUtil.js

@@ -0,0 +1,53 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+class CodeUtil {
+    /**
+     * Wrap a method*[1] with an implementation of a runnable class.
+     * The wrapper must be a class that implements `async run(values)`,
+     * and `run` should delegate to `this._run()` after setting this.values.
+     * The `BaseOperation` class is an example of such a class.
+     *
+     * [1]: since our runnable interface expects named parameters, this
+     *      wrapping behavior is only useful for methods that accept a single
+     *      object argument.
+     * @param {*} method
+     * @param {*} wrapper
+     */
+    static mrwrap (method, wrapper, options = {}) {
+        const cls_name = options.name || method.name;
+
+        const cls = class extends wrapper {
+            async _run () {
+                return await method.call(this.self, this.values);
+            }
+        }
+
+        Object.defineProperty(cls, 'name', { value: cls_name });
+
+        return async function (...a) {
+            const op = new cls();
+            op.self = this;
+            return await op.run(...a);
+        }
+    }
+}
+
+module.exports = {
+    CodeUtil,
+};

packages/backend/src/codex/README.md

@@ -0,0 +1,10 @@
+# What is this?
+
+ChatGPT told me to call this codex and that sounds really cool so
+I couldn't resist.
+
+This directory contains utilities for modelling code as data, so that
+we can use static analysis techniques and prevent detectable errors
+from reaching produciton. This is an attempt at making things more robust,
+but it's not guarenteed to work or even be useful; we need to try it and
+collect data about its effectiveness.

packages/backend/src/codex/Sequence.js

@@ -0,0 +1,252 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+/**
+ * Sequence is a callable object that executes a series of functions in order.
+ * The functions are expected to be asynchronous; if they're not it might still
+ * work, but it's neither tested nor supported.
+ *
+ * Note: arrow functions are supported, but they are not recommended;
+ * using keyword functions allows each step to be named.
+ *
+ * Example usage:
+ *
+ *     const seq = new Sequence([
+ *         async function set_foo (a) {
+ *             a.set('foo', 'bar')
+ *         },
+ *         async function print_foo (a) {
+    *          console.log(a.get('foo'));
+ *         },
+ *         async function third_step (a) {
+ *             // do something
+ *         },
+ *     ]);
+ *
+ *     await seq();
+ *
+ * Example with controlled conditional branches:
+ *
+ *     const seq = new Sequence([
+ *         async function first_step (a) {
+ *             // do something
+ *         },
+ *         {
+ *             condition: async a => a.get('foo') === 'bar',
+ *             fn: async function second_step (a) {
+ *                 // do something
+ *             }
+ *         },
+ *         async function third_step (a) {
+ *             // do something
+ *         },
+ *     ]);
+ *
+ * If it is called with an argument, it must be an object containing values
+ * which will populate the "sequence scope".
+ *
+ * If it is called on an instance with a member called `values`
+ * (i.e. if `this.values` is defined), then these values will populate the
+ * sequence scope. This is to maintain compatibility for Sequence to be used
+ * as an implementation of a runnable class. (See CodeUtil.mrwrap or BaseOperation)
+ *
+ * The object returned by the constructor is a function, which is used to
+ * make the object callable. The callable object will execute the sequence
+ * when called. The return value of the sequence is the return value of the
+ * last function in the sequence.
+ *
+ * Each function in the sequence is passed a SequenceState object
+ * as its first argument. Conventionally, this argument is called `a`,
+ * which is short for either "API", "access", or "the `a` variable"
+ * depending on which you prefer. Sequence provides methods for accessing
+ * the sequence scope.
+ *
+ * By accessing the sequence scope through the `a` variable, changes to the
+ * sequence scope can be monitored and recorded. (TODO: implement observe methods)
+ */
+class Sequence {
+    /**
+     * A SequenceState is created each time a Sequence is called.
+     */
+    static SequenceState = class SequenceState {
+        constructor (sequence, thisArg) {
+            if ( typeof sequence === 'function' ) {
+                sequence = sequence.sequence;
+            }
+
+            this.sequence_ = sequence;
+            this.thisArg = thisArg;
+            this.steps_ = null;
+            this.value_history_ = [];
+            this.scope_ = {};
+            this.last_return_ = undefined;
+            this.i = 0;
+            this.stopped_ = false;
+
+            this.defer_ptr_ = undefined;
+            this.defer = this.constructor.defer_0;
+        }
+
+        get steps () {
+            return this.steps_ ?? this.sequence_.steps_;
+        }
+
+        async run (values) {
+            // Initialize scope
+            values = values || this.thisArg?.values || {};
+            Object.assign(this.scope_, values);
+
+            // Run sequence
+            for ( ; this.i < this.steps.length ; this.i++ ) {
+                let step = this.steps[this.i];
+                if ( typeof step !== 'object' ) step = {
+                    name: step.name,
+                    fn: step,
+                };
+
+                if ( step.condition && ! await step.condition(this) ) {
+                    continue;
+                }
+
+                const parent_scope = this.scope_;
+                this.scope_ = {};
+                // We could do Object.assign(this.scope_, parent_scope), but
+                // setting __proto__ is faster because it leverages the optimizations
+                // of the JS engine for the prototype chain.
+                this.scope_.__proto__ = parent_scope;
+
+                if ( this.sequence_.options_.record_history ) {
+                    this.value_history_.push(this.scope_);
+                }
+
+                if ( this.sequence_.options_.before_each ) {
+                    await this.sequence_.options_.before_each(this, step);
+                }
+
+                this.last_return_ = await step.fn(this);
+                if ( this.stopped_ ) {
+                    break;
+                }
+            }
+        }
+
+        // Why check a condition every time code is called,
+        // when we can check it once and then replace the code?
+
+        // defer_0: the first time defer is called, a copy of the sequence
+        // steps is made, and defer is replaced with defer_1 and called.
+        static defer_0 = function (fn) {
+            this.steps_ = [...this.sequence_.steps_];
+            this.defer = this.constructor.defer_1;
+            this.defer_ptr_ = this.steps_.length;
+            this.defer(fn);
+        }
+        // defer_1: subsequent calls to defer are delegated to defer_2,
+        // which pushes the given value to the sequence state's steps.
+        static defer_1 = function (fn) {
+            // Deferred functions don't affect the return value
+            const real_fn = fn;
+            fn = async () => {
+                await real_fn(this);
+                return this.last_return_;
+            };
+
+            // Suppose we want to defer a function called `g'`
+            // given the following state of the sequence:
+            //     [a, b, c, d, f', e']
+            // where `'` indicates a deferred step, `f'` is the item pointed
+            // to by `defer_ptr_`;
+            // We want to insert `g'` immediately before `f'`
+
+            this.steps_.splice(this.defer_ptr_, 0, fn);
+        }
+
+        get (k) {
+            // TODO: record read1
+            return this.scope_[k];
+        }
+
+        set (k, v) {
+            // TODO: record mutation
+            this.scope_[k] = v;
+        }
+
+        values () {
+            return new Proxy(this.scope_, {
+                get: (target, property) => {
+                    if (property in target) {
+                        // TODO: record read
+                        return target[property];
+                    }
+                    return undefined;
+                }
+            });
+        }
+
+        iget (k) {
+            if ( k === undefined ) return this.thisArg;
+            return this.thisArg?.[k];
+        }
+
+        get log () {
+            return this.iget('log');
+        }
+
+        stop (return_value) {
+            this.stopped_ = true;
+            return return_value;
+        }
+    }
+
+    constructor(...args) {
+        const sequence = this;
+
+        const steps = [];
+        const options = {};
+
+        for ( const arg of args ) {
+            if ( Array.isArray(arg) ) {
+                steps.push(...arg);
+            } else if ( typeof arg === 'object' ) {
+                Object.assign(options, arg);
+            } else if ( typeof arg === 'function' ) {
+                steps.push(arg);
+            } else {
+                throw new TypeError(`Invalid argument to Sequence constructor: ${arg}`);
+            }
+        }
+
+        const fn = async function () {
+            const state = new Sequence.SequenceState(sequence, this);
+            await state.run();
+            return state.last_return_;
+        }
+
+        this.steps_ = steps;
+        this.options_ = options || {};
+
+        Object.defineProperty(fn, 'name', { value: 'Sequence' });
+        Object.defineProperty(fn, 'sequence', { value: this });
+
+        return fn;
+    }
+}
+
+module.exports = {
+    Sequence
+};

packages/backend/src/config.js

@@ -0,0 +1,169 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+"use strict"
+const deep_proto_merge = require('./config/deep_proto_merge');
+// const reserved_words = require('./config/reserved_words');
+
+let config = {};
+
+// Static defaults
+config.servers = [];
+
+config.max_file_size = 100_000_000_000,
+config.max_thumb_size = 1_000,
+config.max_fsentry_name_length = 767,
+
+config.username_regex = /^\w{1,}$/;
+config.username_max_length = 45;
+config.subdomain_regex = /^[a-zA-Z0-9-_-]+$/;
+config.subdomain_max_length = 60;
+config.app_name_regex = /^[a-zA-Z0-9-_-]+$/;
+config.app_name_max_length = 60;
+config.app_title_max_length = 60;
+config.min_pass_length = 6;
+
+config.strict_email_verification_required = false,
+config.require_email_verification_to_publish_website = false,
+
+config.kv_max_key_size = 1024,
+config.kv_max_value_size = 400 * 1024,
+
+config.monitor = {
+    metricsInterval: 60000,
+    windowSize: 30,
+};
+
+config.max_subdomains_per_user = 2000;
+config.storage_capacity = 1*1024*1024*1024;
+config.static_hosting_domain = '-static.puter.local';
+
+config.thumb_width = 80;
+config.thumb_height = 80;
+config.app_max_icon_size = 5*1024*1024;
+
+// config.origin = config.protocol + '://' + config.domain;
+// config.api_base_url = config.protocol + '://api.' + config.domain;
+// config.social_card = `${config.origin}/assets/img/screenshot.png`;
+config.short_description = `Puter is a privacy-first personal cloud that houses all your files, apps, and games in one private and secure place, accessible from anywhere at any time.`;
+config.title = 'Puter';
+config.company = 'Puter Technologies Inc.';
+
+config.puter_hosted_data = {
+    puter_versions: 'https://version.puter.site/puter_versions.json',
+};
+
+{
+    const path_ = require('path');
+    config.assets = {
+        gui: path_.join(__dirname, '../../..'),
+    };
+}
+
+// words that cannot be used by others as subdomains or app names
+// config.reserved_words = reserved_words;
+config.reserved_words = [];
+
+// set default S3 settings for this server, if any
+if (config.server_id) {
+	// see if this server has a specific bucket
+	for (let index = 0; index < config.servers.length; index++) {
+		if (config.servers[index].id === config.server_id && config.servers[index].s3_bucket){
+			config.s3_bucket = config.servers[index].s3_bucket;
+            config.s3_region = config.servers[index].region;
+        }
+	}
+}
+
+config.contact_email = 'hey@' + config.domain;
+
+module.exports = config;
+
+// NEW_CONFIG_LOADING
+
+const computed_defaults = {
+    pub_port: config => config.http_port,
+    origin: config => config.protocol + '://' + config.domain +
+        (config.pub_port !== 80 && config.pub_port !== 443 ? ':' + config.pub_port : ''),
+    api_base_url: config => config.protocol + '://api.' + config.domain +
+        (config.pub_port !== 80 && config.pub_port !== 443 ? ':' + config.pub_port : ''),
+    social_card: config => `${config.origin}/assets/img/screenshot.png`,
+};
+
+// We're going to export a config object that's decorated
+// with additional behavior
+let config_to_export;
+
+// We have a pointer to some config object which
+// load_config() may replace
+const config_pointer = {};
+{
+    config_pointer.__proto__ = config;
+    config_to_export = config_pointer;
+}
+
+
+// We have some methods that can be called on `config`
+{
+    // Add configuration values with precedence over the current config
+    const load_config = o => {
+        let replacement_config = {
+            ...o,
+        };
+        // replacement_config.__proto__ = config_pointer.__proto__;
+        replacement_config = deep_proto_merge(replacement_config, config_pointer.__proto__, {
+            preserve_flag: true,
+        })
+        config_pointer.__proto__ = replacement_config;
+    };
+
+    const config_api = { load_config };
+    config_api.__proto__ = config_to_export;
+    config_to_export = config_api;
+}
+
+// We have some values with computed defaults
+{
+    const get_implied = (target, prop) => {
+        if (prop in computed_defaults) {
+            return computed_defaults[prop](target);
+        }
+        return undefined;
+    };
+    config_to_export = new Proxy(config_to_export, {
+        get: (target, prop, receiver) => {
+            if (prop in target) {
+                return target[prop];
+            } else {
+                // console.log('implied', prop,
+                //     'to', get_implied(config_to_export, prop));
+                return get_implied(config_to_export, prop);
+            }
+        }
+    })
+}
+
+// We'd like to store values changed at runtime separately
+// for easier runtime debugging
+{
+    const config_runtime_values = {};
+    config_runtime_values.__proto__ = config_to_export;
+    config_to_export = config_runtime_values
+}
+
+module.exports = config_to_export;

packages/backend/src/config/ConfigLoader.js

@@ -0,0 +1,72 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { AdvancedBase } = require("puter-js-common");
+const { quot } = require("../util/strutil");
+
+class ConfigLoader extends AdvancedBase {
+    static MODULES = {
+        path_: require("path"),
+        fs: require("fs"),
+    }
+
+    constructor (logger, path, config) {
+        super();
+        this.logger = logger;
+        this.path = path;
+        this.config = config;
+    }
+
+    enable (name, meta = {}) {
+        const { path_, fs } = this.modules;
+
+        const config_path = path_.join(this.path, name);
+
+        if ( ! fs.existsSync(config_path) ) {
+            throw new Error(`Config file not found: ${config_path}`);
+        }
+
+        const config_values = JSON.parse(fs.readFileSync(config_path, 'utf8'));
+        if ( config_values.$requires ) {
+            const config_list = config_values.$requires;
+            delete config_values.$requires;
+            this.apply_requires(this.path, config_list, { by: name });
+        }
+        this.logger.info(
+            `Applying config: ${path_.relative(this.path, config_path)}` +
+            (meta.by ? ` (required by ${meta.by})` : '')
+        );
+        this.config.load_config(config_values);
+
+    }
+
+    apply_requires (dir, config_list, { by } = {}) {
+        const { path_, fs } = this.modules;
+
+        for ( const name of config_list ) {
+            const config_path = path_.join(dir, name);
+            if ( ! fs.existsSync(config_path) ) {
+                throw new Error(`could not find ${quot(config_path)} ` +
+                    `required by ${quot(by)}`);
+            }
+            this.enable(name, { by });
+        }
+    }
+}
+
+module.exports = { ConfigLoader };

packages/backend/src/config/deep_proto_merge.js

@@ -0,0 +1,95 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+/**
+ * Sets replacement.__proto__ to `delegate`
+ * then iterates over members of `replacement` looking for
+ * objects that are not arrays.
+ *
+ * When an object is found, a recursive call is made to
+ * `deep_proto_merge` with the corresponding object in `delegate`.
+ *
+ * If `preserve_flag` is set to true, only objects containing
+ * a truthy property named `$preserve` will be merged.
+ *
+ * @param {*} replacement
+ * @param {*} delegate
+ */
+const deep_proto_merge = (replacement, delegate, options) => {
+    const is_object = (obj) => obj &&
+        typeof obj === 'object' && !Array.isArray(obj);
+
+    replacement.__proto__ = delegate;
+
+    for ( const key in replacement ) {
+        if ( ! is_object(replacement[key]) ) continue;
+
+        if ( options?.preserve_flag && ! replacement[key].$preserve ) {
+            continue;
+        }
+        if ( ! is_object(delegate[key]) ) {
+            continue;
+        }
+        replacement[key] = deep_proto_merge(
+            replacement[key], delegate[key], options,
+        );
+    }
+
+    // use a Proxy object to ensure all keys are present
+    // when listing keys of `replacement`
+    replacement = new Proxy(replacement, {
+        // no get needed
+        // no set needed
+        ownKeys: (target) => {
+            const ownProps = Reflect.ownKeys(target); // Get own property names and symbols, including non-enumerable
+            const protoProps = Reflect.ownKeys(Object.getPrototypeOf(target)); // Get prototype's properties
+
+            // Combine and deduplicate properties using a Set, then convert back to an array
+            const s = new Set([
+                ...protoProps,
+                ...ownProps
+            ]);
+
+            if (options?.preserve_flag) {
+                // remove $preserve if it exists
+                s.delete('$preserve');
+            }
+
+            return Array.from(s);
+        },
+        getOwnPropertyDescriptor: (target, prop) => {
+            // Real descriptor
+            let descriptor = Object.getOwnPropertyDescriptor(target, prop);
+
+            if (descriptor) return descriptor;
+
+            // Immediate prototype descriptor
+            const proto = Object.getPrototypeOf(target);
+            descriptor = Object.getOwnPropertyDescriptor(proto, prop);
+
+            if (descriptor) return descriptor;
+
+            return undefined;
+        }
+
+    });
+
+    return replacement;
+};
+
+module.exports = deep_proto_merge;

packages/backend/src/config/reserved_words.js

@@ -0,0 +1,203 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+module.exports = [
+    // system and apps
+    'about',
+    'api',
+    'camera',
+    'changelog',
+    'cloudjs',
+    'cloud.js',
+    'code',
+    'dev-center',
+    'draw',
+    'editor',
+    'markus',
+    'pdf',
+    'photopea',
+    'player',
+    'terminal',
+    'viewer',
+    'www',
+
+    // others
+    'admin',
+    'ads',
+    'alt',
+    'api',
+    'app',
+    'apps',
+    'audio',
+    'auth',
+    'badge',
+    'beta',
+    'business',
+    'buy',
+    'cdn',
+    'cli',
+    'cloud',
+    'cmd',
+    'community',
+    'careers',
+    'config',
+    'db',
+    'demo',
+    'dev',
+    'developers',
+    'dns1',
+    'dns2',
+    'dns3',
+    'dns4',
+    'dns5',
+    'dns6',
+    'dns7',
+    'dns8',
+    'dns9',
+    'dns0',
+    'doc',
+    'docs',
+    'email',
+    'eng',
+    'engineering',
+    'exchange',
+    'faq',
+    'feeds',
+    'files',
+    'forum',
+    'fs',
+    'ftp',
+    'gov',
+    'groups',
+    'help',
+    'hq',
+    'images',
+    'img',
+    'in',
+    'inbound',
+    'info',
+    'jobs',
+    'js',
+    'lab',
+    'learn',
+    'live',
+    'login',
+    'mail',
+    'media',
+    'mobile',
+    'mx',
+    'mx1',
+    'mx2',
+    'mx3',
+    'mx4',
+    'mx5',
+    'mx6',
+    'mx7',
+    'mx8',
+    'mx9',
+    'mx0',
+    'my',
+    'mysql',
+    'news',
+    'newsletter',
+    'ns1',
+    'ns2',
+    'ns3',
+    'ns4',
+    'ns5',
+    'ns6',
+    'ns7',
+    'ns8',
+    'ns9',
+    'ns0',
+    'office',
+    'out',
+    'owa',
+    'pop',
+    'pop3',
+    'portal',
+    'private',
+    'public',
+    'remote',
+    'sandbox',
+    'sdk',
+    'search',
+    'secure',
+    'service',
+    'shell',
+    'shop',
+    'signin',
+    'signup',
+    'smtp',
+    'smtpin',
+    'socket',
+    'ssl',
+    'start',
+    'static',
+    'status',
+    'store',
+    'support',
+    'test',
+    'tutorials',
+    'upload',
+    'video',
+    'videos',
+    'vpn',
+    'vps',
+    'web',
+    'wiki',
+    'www',
+
+    '1',
+    '2',
+    '3',
+    '4',
+    '5',
+    '6',
+    '7',
+    '8',
+    '9',
+    '0',
+
+    'a',
+    'b',
+    'c',
+    'd',
+    'e',
+    'f',
+    'g',
+    'h',
+    'i',
+    'j',
+    'k',
+    'l',
+    'm',
+    'n',
+    'o',
+    'p',
+    'q',
+    'r',
+    's',
+    't',
+    'u',
+    'v',
+    'w',
+    'x',
+    'y',
+    'z',
+];

packages/backend/src/connective/data_model.js

@@ -0,0 +1,25 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+{
+    registries: {
+        type: {
+            description: 'high-level types'
+        }
+    }
+}

packages/backend/src/connective/registry.js

@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+class Registry {
+    constructor() {
+        this._registry = {};
+    }
+
+    put(name, value) {
+        this._registry[name] = value;
+    }
+
+    get(name) {
+        return this._registry[name];
+    }
+}

packages/backend/src/env

@@ -0,0 +1 @@
+dev

packages/backend/src/errors/TechnicalError.js

@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+/**
+ * @class TechnicalError
+ * @extends Error
+ *
+ * This error type is used for errors that may be presented in a
+ * technical context, such as a terminal or log file.
+ *
+ * @todo This could be a trait errors can have rather than a class.
+ */
+class TechnicalError extends Error {
+    constructor (message, ...details) {
+        super(message);
+
+        for ( const detail of details ) {
+            detail(this);
+        }
+    }
+}
+
+const ERR_HINT_NOSTACK = e => {
+    e.toString = () => e.message;
+}
+
+module.exports = {
+    TechnicalError,
+    ERR_HINT_NOSTACK,
+};

packages/backend/src/errors/error_help_details.js

@@ -0,0 +1,244 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { quot, osclink } = require("../util/strutil");
+
+const reused = {
+    runtime_env_references: [
+        {
+            subject: 'ENVIRONMENT.md file',
+            location: 'root of the repository',
+            use: 'describes which paths are checked',
+        },
+        {
+            subject: 'boot logger',
+            location: 'above this text',
+            use: 'shows what checks were performed',
+        },
+        {
+            subject: 'RuntimeEnvironment.js',
+            location: 'src/boot/ in repository',
+            use: 'code that performs the checks',
+        }
+    ]
+};
+
+const programmer_errors = [
+    'Assignment to constant variable.'
+];
+
+const error_help_details = [
+    {
+        match: ({ message }) => (
+            message.startsWith('No suitable path found for')
+        ),
+        apply (more) {
+            more.references = [
+                ...reused.runtime_env_references,
+            ]
+        }
+    },
+    {
+        match: ({ message }) => (
+            message.match(/^No (read|write) permission for/)
+        ),
+        apply (more) {
+            more.solutions = [
+                {
+                    title: 'Change permissions with chmod',
+                },
+                {
+                    title: 'Remove the path to use working directory',
+                },
+                {
+                    title: 'Set CONFIG_PATH or RUNTIME_PATH environment variable',
+                },
+            ],
+            more.references = [
+                ...reused.runtime_env_references,
+            ]
+        }
+    },
+    {
+        match: ({ message }) => (
+            message.startsWith('No valid config file found in path')
+        ),
+        apply (more) {
+            more.solutions = [
+                {
+                    title: 'Create a valid config file',
+                },
+            ]
+        }
+    },
+    {
+        match: ({ message }) => (
+            message === `config_name is required`
+        ),
+        apply (more) {
+            more.solutions = [
+                'ensure config_name is present in your config file',
+                'Seek help on ' + osclink(
+                    'https://discord.gg/PQcx7Teh8u',
+                    'our Discord server'
+                ),
+            ];
+        }
+    },
+    {
+        match: ({ message }) => (
+            message == 'Assignment to constant variable.'
+        ),
+        apply (more) {
+            more.references = [
+                {
+                    subject: 'MDN Reference for this error',
+                    location: 'on the internet',
+                    use: 'describes why this error occurs',
+                    url: 'https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Errors/Invalid_const_assignment'
+                },
+            ]
+        }
+    },
+    {
+        match: ({ message }) => (
+            programmer_errors.includes(message)
+        ),
+        apply (more) {
+            more.notes = [
+                'It looks like this might be our fault.',
+            ]
+            more.solutions = [
+                {
+                    title: `Check for an issue on ` +
+                        osclink('https://github.com/HeyPuter/puter/issues')
+                },
+                {
+                    title: `If there's no issue, please ` +
+                        osclink(
+                            'https://github.com/HeyPuter/puter/issues/new',
+                            'create one'
+                        ) + '.'
+                }
+            ]
+        }
+    },
+    {
+        match: ({ message }) => (
+            message.startsWith('Expected double-quoted property')
+        ),
+        apply (more) {
+            more.notes = [
+                'There might be a trailing-comma in your config',
+            ];
+        }
+    }
+];
+
+/**
+ * Print error help information to a stream in a human-readable format.
+ *
+ * @param {Error} err - The error to print help for.
+ * @param {*} out - The stream to print to; defaults to process.stdout.
+ * @returns {undefined}
+ */
+const print_error_help = (err, out = process.stdout) => {
+    if ( ! err.more ) {
+        err.more = {};
+        err.more.references = [];
+        err.more.solutions = [];
+        for ( const detail of error_help_details ) {
+            if ( detail.match(err) ) {
+                detail.apply(err.more);
+            }
+        }
+    }
+
+    let write = out.write.bind(out);
+
+    write('\n');
+
+    const wrap_msg = s =>
+        `\x1B[31;1m┏━━ [ HELP:\x1B[0m ${quot(s)} \x1B[31;1m]\x1B[0m`;
+    const wrap_list_title = s =>
+        `\x1B[36;1m${s}:\x1B[0m`;
+
+    write(wrap_msg(err.message) + '\n');
+
+    write = (s) => out.write('\x1B[31;1m┃\x1B[0m ' + s);
+
+    const vis = (stok, etok, str) => {
+        return `\x1B[36;1m${stok}\x1B[0m${str}\x1B[36;1m${etok}\x1B[0m`;
+    }
+
+    let lf_sep = false;
+
+    write('Whoops! Looks like something isn\'t working!\n');
+    let any_help = false;
+
+    if ( err.more.notes ) {
+        write('\n');
+        lf_sep = true;
+        any_help = true;
+        for ( const note of err.more.notes ) {
+            write(`\x1B[33;1m * ${note}\x1B[0m\n`);
+        }
+    }
+
+    if ( err.more.solutions?.length > 0 ) {
+        if ( lf_sep ) write('\n');
+        lf_sep = true;
+        any_help = true;
+        write('The suggestions below may help resolve this issue.\n')
+        write('\n');
+        write(wrap_list_title('Possible Solutions') + '\n');
+        for ( const sol of err.more.solutions ) {
+            write(`  - ${sol.title}\n`);
+        }
+    }
+
+    if ( err.more.references?.length > 0 ) {
+        if ( lf_sep ) write('\n');
+        lf_sep = true;
+        any_help = true;
+        write('The references below may be related to this issue.\n')
+        write('\n');
+        write(wrap_list_title('References') + '\n');
+        for ( const ref of err.more.references ) {
+            write(`  - ${vis('[', ']', ref.subject)} ` +
+                `${vis('(', ')', ref.location)};\n`);
+            write(`      ${ref.use}\n`);
+            if ( ref.url ) {
+                write(`      ${osclink(ref.url)}\n`);
+            }
+        }
+    }
+
+    if ( ! any_help ) {
+        write('No help is available for this error.\n');
+        write('Help can be added in src/errors/error_help_details.\n');
+    }
+
+    out.write(`\x1B[31;1m┗━━ [ END HELP ]\x1B[0m\n`)
+    out.write('\n');
+}
+
+module.exports = {
+    error_help_details,
+    print_error_help,
+};

packages/backend/src/filesystem/FSAccessContext.js

@@ -0,0 +1,117 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const FSNodeContext = require("./FSNodeContext");
+
+const { NodePathSelector, NodeUIDSelector, NodeInternalIDSelector } = require("./node/selectors");
+
+/**
+ * Container for access implementations.
+ * 
+ * Access implementations may vary depending on region,
+ * user privileges, and other factors.
+ * 
+ * @class FSAccessContext
+ */
+module.exports = class FSAccessContext {
+    constructor () {
+        this.fsEntryFetcher = null;
+    }
+
+    /**
+     * get_entry_by_path() returns a filesystem entry using
+     * the path to the entry. Use this method when you need
+     * to get a filesystem entry but don't need to collect
+     * any other information about the entry.
+     * 
+     * @warning The entry returned by this method is not
+     * client-safe. Use FSNodeContext to get a client-safe
+     * entry by calling it's fetchEntry() method.
+     * 
+     * @param {*} path 
+     * @returns 
+     * @deprecated use get_entry({ path }) instead
+     */
+    async get_entry_by_path (path) {
+        return await this.get_entry({ path });
+    }
+
+    /**
+     * get_entry() returns a filesystem entry using
+     * path, uid, or id associated with a filesystem
+     * node. Use this method when you need to get a
+     * filesystem entry but don't need to collect any
+     * other information about the entry.
+     * 
+     * @warning The entry returned by this method is not
+     * client-safe. Use FSNodeContext to get a client-safe
+     * entry by calling it's fetchEntry() method.
+     * 
+     * @param {*} param0 options for getting the entry
+     * @param {*} param0.path
+     * @param {*} param0.uid
+     * @param {*} param0.id please use mysql_id instead
+     * @param {*} param0.mysql_id
+     */
+    async get_entry ({ path, uid, id, mysql_id, ...options }) {
+        let fsNode = await this.node({ path, uid, id, mysql_id });
+        await fsNode.fetchEntry(options);
+        return fsNode.entry;
+    }
+
+    /**
+     * node() returns a filesystem node using path, uid,
+     * or id associated with a filesystem node. Use this
+     * method when you need to get a filesystem node and
+     * need to collect information about the entry.
+     * 
+     * @param {*} location - path, uid, or id associated with a filesystem node
+     * @returns 
+     */
+    async node (selector) {
+        if ( typeof selector === 'string' ) {
+            if ( selector.startsWith('/') ) {
+                selector = new NodePathSelector(selector);
+            } else {
+                selector = new NodeUIDSelector(selector);
+            }
+        }
+
+        // TEMP: remove when these objects aren't used anymore
+        if (
+            typeof selector === 'object' &&
+            selector.constructor.name === 'Object'
+        ) {
+            if ( selector.path ) {
+                selector = new NodePathSelector(selector.path);
+            } else if ( selector.uid ) {
+                selector = new NodeUIDSelector(selector.uid);
+            } else {
+                selector = new NodeInternalIDSelector(
+                    'mysql', selector.mysql_id);
+            }
+        }
+
+        let fsNode = new FSNodeContext({
+            services: this.services,
+            selector,
+            fs: this
+        });
+        return fsNode;
+    }
+};

packages/backend/src/filesystem/FSNodeContext.js

@@ -0,0 +1,820 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { get_user, get_dir_size, id2path, id2uuid, is_empty, is_shared_with_anyone, suggest_app_for_fsentry, get_app } = require("../helpers");
+
+const config = require("../config");
+const _path = require('path');
+const { NodeInternalIDSelector, NodeChildSelector, NodeUIDSelector, RootNodeSelector, NodePathSelector } = require("./node/selectors");
+const { Context } = require("../util/context");
+const { MultiDetachable } = require("../util/listenerutil");
+const { NodeRawEntrySelector } = require("./node/selectors");
+const { DB_READ } = require("../services/database/consts");
+
+/**
+ * Container for information collected about a node
+ * on the filesystem.
+ *
+ * Examples of such information include:
+ * - data collected by querying an fsentry
+ * - the location of a file's contents
+ *
+ * This is an implementation of the Facade design pattern,
+ * so information about a filesystem node should be collected
+ * via the methods on this class and not mutated directly.
+ *
+ * @class FSNodeContext
+ * @property {object} entry the filesystem entry
+ * @property {string} path the path to the filesystem entry
+ * @property {string} uid the UUID of the filesystem entry
+ */
+module.exports = class FSNodeContext {
+    static TYPE_FILE = { label: 'File' };
+    static TYPE_DIRECTORY = { label: 'Directory' };
+    static TYPE_SYMLINK = {};
+    static TYPE_SHORTCUT = {};
+    static TYPE_UNDETERMINED = {};
+
+    static SELECTOR_PRIORITY_ORDER = [
+        NodeRawEntrySelector,
+        RootNodeSelector,
+        NodeInternalIDSelector,
+        NodeUIDSelector,
+        NodeChildSelector,
+        NodePathSelector,
+    ];
+
+    /**
+     * Creates an instance of FSNodeContext.
+     * @param {*} opt_identifier
+     * @param {*} opt_identifier.path a path to the filesystem entry
+     * @param {*} opt_identifier.uid a UUID of the filesystem entry
+     * @param {*} opt_identifier.id please pass mysql_id instead
+     * @param {*} opt_identifier.mysql_id a MySQL ID of the filesystem entry
+     */
+    constructor ({ services, selector, fs }) {
+        this.log = services.get('log-service').create('fsnode-context');
+        this.selector_ = null;
+        this.selectors_ = [];
+        this.selector = selector;
+        this.entry = {};
+        this.found = undefined;
+        this.found_thumbnail = undefined;
+
+        selector.setPropertiesKnownBySelector(this);
+
+        this.services = services;
+
+        this.fileContentsFetcher = null;
+
+        this.fs = fs;
+
+        // Decorate all fetch methods with otel span
+        // TODO: language tool for traits; this is a trait
+        const fetch_methods = [
+            'fetchEntry',
+            'fetchPath',
+            'fetchSubdomains',
+            'fetchOwner',
+            'fetchShares',
+            'fetchVersions',
+            'fetchSize',
+            'fetchSuggestedApps',
+            'fetchIsEmpty',
+        ];
+        for ( const method of fetch_methods ) {
+            const original_method = this[method];
+            this[method] = async (...args) => {
+                const tracer = this.services.get('traceService').tracer;
+                let result;
+                await tracer.startActiveSpan(`fs:nodectx:fetch:${method}`, async span => {
+                    result = await original_method.call(this, ...args);
+                    span.end();
+                });
+                return result;
+            }
+        }
+    }
+
+    set selector (new_selector) {
+        // Only add the selector if we don't already have it
+        for ( const selector of this.selectors_ ) {
+            if ( selector instanceof new_selector.constructor ) return;
+        }
+        this.selectors_.push(new_selector);
+        this.selector_ = new_selector;
+    }
+
+    get selector () {
+        return this.get_optimal_selector();
+    }
+
+    get_selector_of_type (cls) {
+        // Reverse iterate over selectors
+        for ( let i = this.selectors_.length - 1; i >= 0; i-- ) {
+            const selector = this.selectors_[i];
+            if ( selector instanceof cls ) {
+                return selector;
+            }
+        }
+
+        if ( cls.implyFromFetchedData ) {
+            return cls.implyFromFetchedData(this);
+        }
+
+        return null;
+    }
+
+    get_optimal_selector () {
+        for ( const cls of FSNodeContext.SELECTOR_PRIORITY_ORDER ) {
+            const selector = this.get_selector_of_type(cls);
+            if ( selector ) return selector;
+        }
+        this.log.warn('Failed to get optimal selector');
+        return this.selector_;
+    }
+
+    get isRoot () {
+        return this.path === '/';
+    }
+
+    async isUserDirectory () {
+        if ( this.isRoot ) return false;
+        if ( this.found === undefined ) {
+            await this.fetchEntry();
+        }
+        if ( this.isRoot ) return false;
+        if ( this.found === false ) return undefined;
+        return ! this.entry.parent_uid;
+    }
+
+    async exists (fetch_options = {}) {
+        await this.fetchEntry();
+        if ( ! this.found ) {
+            this.log.debug(
+                'here\'s why it doesn\'t exist: ' +
+                this.selector.describe() + ' -> ' +
+                this.uid + ' ' +
+                JSON.stringify(this.entry, null, '  ')
+            );
+        }
+        return this.found;
+    }
+
+    async fetchPath () {
+        if ( this.path ) return;
+
+        this.path = await this.services.get('information')
+            .with('fs.fsentry')
+            .obtain('fs.fsentry:path')
+            .exec(this.entry);
+    }
+
+    /**
+     * Fetches the filesystem entry associated with a
+     * filesystem node identified by a path or UID.
+     *
+     * If a UID exists, the path is ignored.
+     * If neither a UID nor a path is set, an error is thrown.
+     *
+     * @param {*} fsEntryFetcher fetches the filesystem entry
+     * @void
+     */
+    async fetchEntry (fetch_entry_options = {}) {
+        if (
+            this.found === true &&
+            ! fetch_entry_options.force &&
+            (
+                // thumbnail already fetched, or not asked for
+                ! fetch_entry_options.thumbnail || this.entry?.thumbnail ||
+                this.found_thumbnail !== undefined
+            )
+        ) {
+            return;
+        }
+
+        // NOTE: commented out for now because it's too verbose
+        this.log.info('fetching entry: ' + this.selector.describe(true));
+        // All services at the top (DEVLOG-401)
+        const {
+            traceService,
+            fsEntryService,
+            fsEntryFetcher,
+            resourceService,
+        } = Context.get('services').values;
+
+        // await this.fs.resourceService
+        //     .waitForResource(this.selector);
+        if ( fetch_entry_options.tracer == null ) {
+            fetch_entry_options.tracer = traceService.tracer;
+        }
+
+        if ( fetch_entry_options.op ) {
+            fetch_entry_options.trace_options = {
+                parent: fetch_entry_options.op.span,
+            };
+        }
+
+        let entry;
+
+        await new Promise (rslv => {
+            const detachables = new MultiDetachable();
+
+            let resolved = false;
+
+            const callback = (resolver) => {
+                // NOTE: commented out for now because it's too verbose
+                this.log.noticeme(`resolved by ${resolver}`, {
+                    debug: fetch_entry_options.debug,
+                });
+                resolved = true;
+                detachables.detach();
+                rslv();
+            }
+
+            // either the resource is free
+            {
+                // no detachale because waitForResource returns a
+                // Promise that will be resolved when the resource
+                // is free no matter what, and then it will be
+                // garbage collected.
+                resourceService.waitForResource(
+                    this.selector
+                ).then(callback.bind(null, 'resourceService'));
+            }
+
+            // or pending information about the resource
+            // becomes available
+            {
+                // detachable is needed here because waitForEntry keeps
+                // a map of listeners in memory, and this event may
+                // never occur. If this never occurs, waitForResource
+                // is guaranteed to resolve eventually, and then this
+                // detachable will be detached by `callback` so the
+                // listener can be garbage collected.
+                const det = fsEntryService.waitForEntry(
+                    this, callback.bind(null, 'fsEntryService'));
+                if ( det ) detachables.add(det);
+            }
+        });
+
+        this.log.debug('got past the promise')
+
+        if ( resourceService.getResourceInfo(this.uid) ) {
+            entry = await fsEntryService.get(this.uid, fetch_entry_options);
+            this.log.debug('got an entry from the future');
+        } else {
+            this.log.debug('resource is already free');
+            entry = await fsEntryFetcher.find(
+                this.selector, fetch_entry_options);
+        }
+
+        if ( ! entry ) {
+            this.log.info(`entry not found: ${this.selector.describe(true)}`);
+        }
+
+        if ( entry === null || typeof entry !== 'object' ) {
+            // TODO: this property shouldn't be set to false -
+            //   this is set to false to avoid regressions with
+            //   existing code.
+            this.entry = false;
+
+            this.found = false;
+            return;
+        }
+
+        this.found = true;
+
+        if ( entry.id ) {
+            this.selector = new NodeInternalIDSelector('mysql', entry.id, {
+                source: 'FSNodeContext optimization'
+            });
+        }
+
+        if ( ! this.uid && entry.uuid ) {
+            this.uid = entry.uuid;
+        }
+
+        if ( ! this.mysql_id && entry.id ) {
+            this.mysql_id = entry.id;
+        }
+
+        if ( ! this.path && entry.path ) {
+            this.path = entry.path;
+        }
+
+        if ( ! this.name && entry.name ) this.name = entry.name;
+
+        Object.assign(this.entry, entry);
+    }
+
+    /**
+     * Wait for an fsentry which might be enqueued for insertion
+     * into the database.
+     *
+     * This just calls ResourceService under the hood.
+     */
+    async awaitStableEntry () {
+        const resourceService = Context.get('services').get('resourceService');
+        await resourceService.waitForResource(this.selector);
+    }
+
+    /**
+     * Fetches the subdomains associated with a directory or file
+     * and stores them on the `subdomains` property of the fsentry.
+     * @param {object} user the user is needed to query subdomains
+     * @param {bool} force fetch subdomains if they were already fetched
+     *
+     * @param fs:decouple-subdomains
+     */
+    async fetchSubdomains (user, force) {
+        if ( ! this.entry.is_dir ) return;
+
+        const db = this.services.get('database').get(DB_READ, 'filesystem');
+
+        this.entry.subdomains = []
+        let subdomains = await db.read(
+            `SELECT * FROM subdomains WHERE root_dir_id = ? AND user_id = ?`,
+            [this.entry.id, user.id]
+        );
+        if(subdomains.length > 0){
+            subdomains.forEach((sd)=>{
+                this.entry.subdomains.push({
+                    subdomain: sd.subdomain,
+                    address: config.protocol + '://' + sd.subdomain + "." + 'puter.site',
+                    uuid: sd.uuid,
+                })
+            })
+            this.entry.has_website = true;
+        }
+    }
+
+    /**
+     * Fetches the owner of a directory or file and stores it on the
+     * `owner` property of the fsentry.
+     * @param {bool} force fetch owner if it was already fetched
+     */
+    async fetchOwner (force) {
+        if ( this.isRoot ) return;
+        const owner = await get_user({ id: this.entry.user_id });
+        this.entry.owner = {
+            username: owner.username,
+            email: owner.email,
+        };
+    }
+
+    /**
+     * Fetches shares, AKA "permissions", for a directory or file;
+     * then, stores them on the `permissions` property
+     * of the fsentry.
+     * @param {bool} force fetch shares if they were already fetched
+     *
+     * @deprecated sharing will use user-to-user permissions
+     */
+    async fetchShares (force) {
+        if ( this.entry.permissions && ! force ) return;
+
+        const db = this.services.get('database').get(DB_READ, 'filesystem');
+
+        let shares = await db.read(
+            `SELECT share.id as share_id, user.* FROM share
+                INNER JOIN user ON share.recipient_user_id = user.id
+                WHERE share.fsentry_id = ?`,
+            [this.entry.id]
+        );
+        const shares_tidy = [];
+        for ( const share of shares ) {
+            shares_tidy.push({uid: share.share_id, username: share.username, email: share.email});
+        }
+        this.entry.permissions = shares_tidy;
+    }
+
+    /**
+     * Fetches versions associated with a filesystem entry,
+     * then stores them on the `versions` property of
+     * the fsentry.
+     * @param {bool} force fetch versions if they were already fetched
+     *
+     * @todo fs:decouple-versions
+     */
+    async fetchVersions (force) {
+        if ( this.entry.versions && ! force ) return;
+
+        const db = this.services.get('database').get(DB_READ, 'filesystem');
+
+        let versions = await db.read(
+            `SELECT * FROM fsentry_versions WHERE fsentry_id = ?`,
+            [this.entry.id]
+        );
+        const versions_tidy = [];
+        for (let index = 0; index < versions.length; index++) {
+            const version = versions[index];
+            let username = version.user_id ? (await get_user({id: version.user_id})).username : null;
+            versions_tidy.push({
+                id: version.version_id,
+                message: version.message,
+                timestamp: version.ts_epoch,
+                user: {
+                    username: username,
+                }
+            })
+        }
+
+        this.entry.versions = versions_tidy;
+    }
+
+    /**
+     * Fetches the size of a file or directory if it was not
+     * already fetched.
+     * @param {object} user the user is needed to fetch the size
+     */
+    async fetchSize (user) {
+        const { fsEntryService } = Context.get('services').values;
+
+        // we already have the size for files
+        if ( ! this.entry.is_dir ) return;
+
+        this.entry.size = await fsEntryService.get_recursive_size(
+            this.entry.uuid,
+        );
+
+        return this.entry.size;
+    }
+
+    async fetchSuggestedApps (user, force) {
+        if ( this.entry.suggested_apps && ! force ) return;
+
+        await this.fetchEntry();
+        if ( ! this.entry ) return;
+
+        this.entry.suggested_apps =
+            await suggest_app_for_fsentry(this.entry, { user });
+    }
+
+    async fetchIsEmpty () {
+        if ( ! this.entry ) return;
+        if ( ! this.entry.is_dir ) return;
+        if ( ! this.uid ) return;
+
+        this.entry.is_empty = await is_empty(this.uid);
+    }
+
+    // TODO: this is currently not called anywhere; for now it
+    //   will never be fetched since sharing is not a priority.
+    async fetchIsShared () {
+        if ( ! this.mysql_id ) return;
+
+        this.entry.is_shared = await is_shared_with_anyone(this.mysql_id);
+    }
+
+    async fetchAll(fsEntryFetcher, user, force) {
+        await this.fetchEntry({ thumbnail: true });
+        await this.fetchSubdomains(user);
+        await this.fetchOwner();
+        await this.fetchShares();
+        await this.fetchVersions();
+        await this.fetchSize(user);
+        await this.fetchSuggestedApps(user);
+        await this.fetchIsEmpty();
+    }
+
+    async get (key) {
+        /*
+            This isn't supposed to stay like this!
+
+            """ if ( key === something ) return this """
+
+                         ^ we should use a map of getters instead
+
+            Ideally I'd like to make a class trait for classes like
+            FSNodeContext that provide a key-value facade to access
+            information about some entity.
+        */
+
+        if ( this.found === false ) {
+            throw new Error(
+                `Tried to get ${key} of non-existent fsentry: ` +
+                this.selector.describe(true)
+            );
+        }
+
+        if ( key === 'entry' ) {
+            await this.fetchEntry();
+            if ( this.found === false ) {
+                throw new Error(
+                    `Tried to get entry of non-existent fsentry: ` +
+                    this.selector.describe(true)
+                );
+            }
+            return this.entry;
+        }
+
+        if ( key === 'path' ) {
+            if ( ! this.path ) await this.fetchEntry();
+            if ( this.found === false ) {
+                throw new Error(
+                    `Tried to get path of non-existent fsentry: ` +
+                    this.selector.describe(true)
+                );
+            }
+            if ( ! this.path ) {
+                // console.log('PATH WAS NOT ON ENTRY', this);
+                await this.fetchPath();
+            }
+            if ( ! this.path ) {
+                throw new Error(`failed to get path`);
+            }
+            return this.path;
+        }
+
+        if ( key === 'uid' ) {
+            await this.fetchEntry();
+            return this.uid;
+        }
+
+        if ( key === 'mysql-id' ) {
+            await this.fetchEntry();
+            return this.mysql_id;
+        }
+
+        const values_from_entry = ['immutable', 'user_id', 'name', 'size', 'parent_uid', 'metadata'];
+        for ( const k of values_from_entry ) {
+            if ( key === k ) {
+                await this.fetchEntry();
+                if ( this.found === false ) {
+                    throw new Error(
+                        `Tried to get ${key} of non-existent fsentry: ` +
+                        this.selector.describe(true)
+                    );
+                }
+                return this.entry[k];
+            }
+        }
+
+        if ( key === 'type' ) {
+            await this.fetchEntry();
+
+            // Longest ternary operator chain I've ever written?
+            return this.entry.is_shortcut
+                ? FSNodeContext.TYPE_SHORTCUT
+                : this.entry.is_symlink
+                    ? FSNodeContext.TYPE_SYMLINK
+                    : this.entry.is_dir
+                        ? FSNodeContext.TYPE_DIRECTORY
+                        : FSNodeContext.TYPE_FILE;
+        }
+
+        if ( key === 'has-s3' ) {
+            await this.fetchEntry();
+            if ( this.entry.is_dir ) return false;
+            if ( this.entry.is_shortcut ) return false;
+            return true;
+        }
+
+        if ( key === 's3:location' ) {
+            await this.fetchEntry();
+            if ( ! await this.exists() ) {
+                throw new Error('file does not exist');
+            }
+            return {
+                bucket: this.entry.bucket,
+                bucket_region: this.entry.bucket_region,
+                key: this.entry.uuid,
+            };
+        }
+
+        if ( key === 'is-root' ) {
+            await this.fetchEntry();
+            return this.isRoot;
+        }
+
+        throw new Error(`unrecognize key for FSNodeContext.get: ${key}`);
+    }
+
+    async getParent () {
+        if ( this.isRoot ) {
+            throw new Error('tried to get parent of root');
+        }
+
+        if ( this.path ) {
+            const parent_fsNode = await this.fs.node({
+                path: _path.dirname(this.path),
+            })
+            return parent_fsNode;
+        }
+
+        if ( this.selector instanceof NodeChildSelector ) {
+            return this.fs.node(this.selector.parent);
+        }
+
+        if ( ! await this.exists() ) {
+            throw new Error('unable to get parent');
+        }
+
+        const parent_uid = this.entry.parent_uid;
+
+        if ( ! parent_uid ) {
+            return this.fs.node(new RootNodeSelector());
+        }
+
+        return this.fs.node(new NodeUIDSelector(parent_uid));
+    }
+
+    async getChild (name) {
+        // If we have a path, we can get an FSNodeContext for the child
+        // without fetching anything.
+        if ( this.path ) {
+            const child_fsNode = await this.fs.node({
+                path: _path.join(this.path, name),
+            })
+            return child_fsNode;
+        }
+
+        return await this.fs.node(new NodeChildSelector(
+            this.selector, name));
+    }
+
+    async getTarget () {
+        await this.fetchEntry();
+        const type = await this.get('type');
+
+        if ( type === FSNodeContext.TYPE_SYMLINK ) {
+            const path = await this.entry.symlink_path;
+            return await this.fs.node({ path });
+        }
+
+        if ( type === FSNodeContext.TYPE_SHORTCUT ) {
+            const target_id = await this.entry.shortcut_to;
+            return await this.fs.node({ mysql_id: target_id });
+        }
+
+        return this;
+    }
+
+    async is_above (child_fsNode) {
+        if ( this.isRoot ) return true;
+
+        const path_this = await this.get('path');
+        const path_child = await child_fsNode.get('path');
+
+        return path_child.startsWith(path_this + '/');
+    }
+
+    async is (fsNode) {
+        if ( this.mysql_id && fsNode.mysql_id ) {
+            return this.mysql_id === fsNode.mysql_id;
+        }
+
+        if ( this.uid && fsNode.uid ) {
+            return this.uid === fsNode.uid;
+        }
+
+        await this.fetchEntry();
+        await fsNode.fetchEntry();
+        return this.uid === fsNode.uid;
+    }
+
+    async getSafeEntry (fetch_options = {}) {
+        if ( this.found === false ) {
+            throw new Error(
+                `Tried to get entry of non-existent fsentry: ` +
+                this.selector.describe(true)
+            );
+        }
+        await this.fetchEntry(fetch_options);
+
+        const res = this.entry;
+        const fsentry = {};
+
+        // This property will not be serialized, but it can be checked
+        // by other code to verify that API calls do not send
+        // unsanitized filsystem entries.
+        Object.defineProperty(fsentry, '__is_safe__', {
+            enumerable: false,
+            value: true,
+        });
+
+        for ( const k in res ) {
+            fsentry[k] = res[k];
+        }
+
+        const info = this.services.get('information');
+
+        if ( ! this.uid && ! this.entry.uuid ) {
+            this.log.noticeme(
+                'whats even happening!?!? ' +
+                    this.selector.describe() + ' ' +
+                    JSON.stringify(this.entry, null, '  ')
+            );
+        }
+
+        // If fsentry was found by a path but the entry doesn't
+        // have a path, use the path that was used to find it.
+        fsentry.path = res.path ?? this.path ?? await info
+            .with('fs.fsentry:uuid')
+            .obtain('fs.fsentry:path')
+            .exec(this.uid ?? this.entry.uuid);
+
+        fsentry.dirname = _path.dirname(fsentry.path);
+        fsentry.dirpath = fsentry.dirname;
+
+        // Do not send internal IDs to clients
+        fsentry.id = res.uuid;
+        fsentry.parent_id = res.parent_uid;
+        // The client calls it uid, not uuid.
+        fsentry.uid = res.uuid;
+        delete fsentry.uuid;
+        delete fsentry.user_id;
+        if ( fsentry.suggested_apps ) {
+            for ( const app of fsentry.suggested_apps ) {
+                if ( app === null ) {
+                    this.log.warn('null app');
+                    continue;
+                }
+                this.log.debug('app?', { value:  app });
+                delete app.owner_user_id;
+            }
+        }
+
+        // Do not send S3 bucket information to clients
+        delete fsentry.bucket;
+        delete fsentry.bucket_region;
+
+        // Use client-friendly IDs for shortcut_to
+        fsentry.shortcut_to = (res.shortcut_to
+            ? await id2uuid(res.shortcut_to) : undefined);
+        fsentry.shortcut_to_path = (res.shortcut_to
+            ? await id2path(res.shortcut_to) : undefined);
+
+        // Add file_request_url
+        if(res.file_request_token && res.file_request_token !== ''){
+            fsentry.file_request_url  = config.origin +
+                '/upload?token=' + res.file_request_token;
+        }
+
+        if ( fsentry.associated_app_id ) {
+            const app = await get_app({ id: fsentry.associated_app_id });
+            fsentry.associated_app = app;
+        }
+
+        fsentry.is_dir = !! fsentry.is_dir;
+
+        // Ensure `size` is numeric
+        if ( fsentry.size ) {
+            fsentry.size = parseInt(fsentry.size);
+        }
+
+        return fsentry;
+    }
+
+    static sanitize_pending_entry_info (res) {
+        const fsentry = {};
+
+        // This property will not be serialized, but it can be checked
+        // by other code to verify that API calls do not send
+        // unsanitized filsystem entries.
+        Object.defineProperty(fsentry, '__is_safe__', {
+            enumerable: false,
+            value: true,
+        });
+
+        for ( const k in res ) {
+            fsentry[k] = res[k];
+        }
+
+        fsentry.dirname = _path.dirname(fsentry.path);
+
+        // Do not send internal IDs to clients
+        fsentry.id = res.uuid;
+        fsentry.parent_id = res.parent_uid;
+        // The client calls it uid, not uuid.
+        fsentry.uid = res.uuid;
+
+        delete fsentry.uuid;
+        delete fsentry.user_id;
+
+        // Do not send S3 bucket information to clients
+        delete fsentry.bucket;
+        delete fsentry.bucket_region;
+
+        delete fsentry.shortcut_to;
+        delete fsentry.shortcut_to_path;
+
+        return fsentry;
+    }
+}

packages/backend/src/filesystem/FSOperationContext.js

@@ -0,0 +1,324 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const PerformanceMonitor = require('../monitor/PerformanceMonitor');
+
+const FSNodeContext = require('./FSNodeContext');
+const FSAccessContext = require('./FSAccessContext');
+const { Context } = require('../util/context');
+
+/**
+ * FSOperationContext represents a single operation on the filesystem.
+ * 
+ * FSOperationContext is used to record events such as side-effects
+ * which occur during a high-level filesystem operation. It is also
+ * responsible for generating a client-safe result which describes
+ * the operation.
+ */
+module.exports = class FSOperationContext {
+    // TODO: rename this.fs to this.access
+    constructor (op_name, context, options) {
+        // TRACK: fs:create-service
+        // TODO: rename this.fs to this.access
+        // NOTE: the 2nd parameter of this constructor
+        //   was called `fs` and was expected to be FSAccessContext.
+        //   Now it should be a context object holding the services
+        //   container. context.access is the FSAccessContext.
+        if ( context instanceof FSAccessContext ) {
+            this.fs = context;
+        } else if ( context ) {
+            this.context = context;
+            this.fs = context.access;
+        } else {
+            const x = Context.get();
+            this.fs = {};
+            this.fs.traceService = x.get('services').get('traceService');
+        }
+
+        this.name = op_name;
+        this.events = [];
+        this.parent_dirs_created = [];
+        this.created = [];
+        this.fields = {};
+        this.safeFields = {};
+
+        this.valueListeners_ = {};
+        this.valueFactories_ = {};
+        this.values_ = {};
+        this.rejections_ = {};
+
+        this.tasks_ = [];
+
+        this.currentCheckpoint_ = 'checkpoint not set';
+
+        if ( options.parent_operation ) {
+            this.parent = options.parent_operation;
+        }
+
+        this.donePromise = new Promise((resolve, reject) => {
+            this.doneResolve = resolve;
+            this.doneReject = reject;
+        });
+
+        // TRACK: arch:trace-service:move-outta-fs
+        if ( this.fs.traceService ) {
+            // Set 'span_' to current active span
+            const { context, trace } = require('@opentelemetry/api');
+            this.span_ = trace.getSpan(context.active());
+        }
+
+        this.monitor = PerformanceMonitor.createContext(`fs.${op_name}`);
+    }
+
+    checkpoint (label) {
+        this.currentCheckpoint_ = label;
+    }
+
+    async addTask (name, fn) {
+        const task = {
+            name,
+            operations: [],
+            promise: Promise.resolve(),
+        };
+
+        const taskContext = {
+            registerOperation: op => {
+                task.operations.push(op);
+                task.promise = task.promise.then(() => op.awaitDone());
+            }
+        };
+
+        const monitor = PerformanceMonitor.createContext('fs.rm');
+        monitor.label(`task:${name}`);
+        task.promise = task.promise.then(() => fn(taskContext));
+        this.tasks_.push(task);
+
+        let last_promise = null;
+        while ( task.promise !== last_promise ) {
+            last_promise = task.promise;
+            await task.promise;
+        }
+        // await task.promise;
+
+        monitor.stamp();
+        monitor.end();
+    }
+
+    get span () { return this.span_; }
+
+    recordParentDirCreated (fsNode) {
+        if ( ! fsNode ) {
+            throw new Error(
+                'falsy value to recordParentDirCreated',
+                fsNode,
+            );
+        }
+        this.parent_dirs_created.push(fsNode);
+    }
+
+    recordCreated (fsNode) {
+        this.created.push(fsNode);
+    }
+
+    set (field, value) {
+        this.fields[field] = value;
+    }
+
+    async set_now (field, value) {
+        this.fields[field] = value;
+        if ( value instanceof FSNodeContext ) {
+            this.safeFields[field] = await value.getSafeEntry();
+        }
+    }
+
+    get (field) {
+        return this.fields[field];
+    }
+
+    complete (options) {
+        options = options ?? {};
+
+        if ( this.parent ) {
+            for ( const fsNode of this.parent_dirs_created ) {
+                this.parent.recordParentDirCreated(fsNode);
+            }
+
+            for ( const fsNode of this.created ) {
+                this.parent.recordCreated(fsNode);
+            }
+        }
+
+        if ( this.tasks_.length > 0 ) {
+            // TODO: it's mutating input options, which is not ideal
+            if ( ! options.after ) options.after = [];
+            
+            options.after.push(
+                this.tasks_.map(task => task.promise)
+            );
+        }
+
+        if ( options.after ) {
+            const thingsToWaitFor = options.after.map(item => {
+                if ( item.awaitDone ) return item.awaitDone;
+                return item;
+            });
+            (async () => {
+                await Promise.all(thingsToWaitFor);
+                this.doneResolve();
+            })();
+            return;
+        }
+
+        this.doneResolve();
+    }
+
+    onComplete(fn) {
+        this.donePromise.then(fn);
+    }
+
+    awaitDone () {
+        return this.donePromise;
+    }
+
+    provideValue (key, value) {
+        this.values_[key] = value;
+
+        let listeners = this.valueListeners_[key];
+        if ( ! listeners ) return;
+
+        delete this.valueListeners_[key];
+
+        for ( let listener of listeners ) {
+            if ( Array.isArray(listener) ) listener = listener[0];
+            listener(value);
+        }
+    }
+
+    rejectValue (key, err) {
+        this.rejections_[key] = err;
+
+        let listeners = this.valueListeners_[key];
+        if ( ! listeners ) return;
+
+        delete this.valueListeners_[key];
+
+        for ( let listener of listeners ) {
+            if ( ! Array.isArray(listener) ) continue;
+            if ( ! listener[1] ) continue;
+            listener = listener[1];
+
+            listener(err);
+        }
+    }
+
+    awaitValue (key) {
+        return new Promise ((rslv, rjct) => {
+            this.onValue(key, rslv, rjct);
+        });
+    }
+
+    onValue (key, fn, rjct) {
+        if ( this.values_[key] ) {
+            fn(this.values_[key]);
+            return;
+        }
+
+        if ( this.rejections_[key] ) {
+            if ( rjct ) {
+                rjct(this.rejections_[key]);
+            } else throw this.rejections_[key];
+            return;
+        }
+
+        if ( ! this.valueListeners_[key] ) {
+            this.valueListeners_[key] = [];
+        }
+        this.valueListeners_[key].push([fn, rjct]);
+
+        if ( this.valueFactories_[key] ) {
+            const fn = this.valueFactories_[key];
+            delete this.valueFactories_[key];
+            (async () => {
+                try {
+                    const value = await fn();
+                    this.provideValue(key, value);
+                } catch (e) {
+                    this.rejectValue(key, e);
+                }
+            })();
+        }
+    }
+
+    async setFactory (key, factoryFn) {
+        if ( this.valueListeners_[key] ) {
+            let v;
+            try {
+                v = await factoryFn();
+            } catch (e) {
+                this.rejectValue(key, e);
+            }
+            this.provideValue(key, v);
+            return;
+        }
+
+        this.valueFactories_[key] = factoryFn;
+    }
+
+    /**
+     * Listen for another operation to complete, and then
+     * complete this operation. This is useful for operations
+     * which delegate to other operations.
+     * 
+     * @param {FSOperationContext} other 
+     * @returns {FSOperationContext} this
+     */
+    completedBy (other) {
+        other.onComplete(() => {
+            this.complete();
+        });
+
+        return this;
+    }
+
+    /**
+     * Produces an object which describes the operation in a
+     * way that is intended to be sent to the client.
+     * 
+     * @returns {Promise<Object>}
+     */
+    async getClientSafeResult () {
+        const result = {};
+        for ( const field in this.fields ) {
+            if ( this.fields[field] instanceof FSNodeContext ) {
+                result[field] = this.safeFields[field] ??
+                    await this.fields[field].getSafeEntry();
+                continue;
+            }
+
+            result[field] = this.fields[field];
+        }
+
+        result.parent_dirs_created = [];
+        for ( const fsNode of this.parent_dirs_created ) {
+            const fsNodeResult = await fsNode.getSafeEntry();
+            result.parent_dirs_created.push(fsNodeResult);
+        }
+
+        return result;
+    }
+}

packages/backend/src/filesystem/FilesystemService.js

@@ -0,0 +1,404 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+// TODO: database access can be a service
+const { ResourceService, RESOURCE_STATUS_PENDING_CREATE } = require('./storage/ResourceService');
+const DatabaseFSEntryFetcher = require("./storage/DatabaseFSEntryFetcher");
+const { DatabaseFSEntryService } = require('./storage/DatabaseFSEntryService');
+const { SizeService } = require('./storage/SizeService');
+const { TraceService } = require('../services/TraceService.js');
+const FSAccessContext = require('./FSAccessContext.js');
+const SystemFSEntryService = require('./storage/SystemFSEntryService.js');
+const PerformanceMonitor = require('../monitor/PerformanceMonitor.js');
+const { NodePathSelector, NodeUIDSelector, NodeInternalIDSelector } = require('./node/selectors.js');
+const FSNodeContext = require('./FSNodeContext.js');
+const { AdvancedBase } = require('puter-js-common');
+const { Context } = require('../util/context.js');
+const { simple_retry } = require('../util/retryutil.js');
+const APIError = require('../api/APIError.js');
+const { LLMkdir } = require('./ll_operations/ll_mkdir.js');
+const { LLCWrite, LLOWrite } = require('./ll_operations/ll_write.js');
+const { LLCopy } = require('./ll_operations/ll_copy.js');
+const { PermissionUtil, PermissionRewriter } = require('../services/auth/PermissionService.js');
+const { DB_WRITE } = require("../services/database/consts");
+
+class FilesystemService extends AdvancedBase {
+    static MODULES = {
+        _path: require('path'),
+        uuidv4: require('uuid').v4,
+        socketio: require('../socketio.js'),
+        config: require('../config.js'),
+    }
+
+    constructor (args) {
+        super(args);
+        const { services } = args;
+
+        this.services = services;
+
+        services.registerService('resourceService', ResourceService);
+        services.registerService('sizeService', SizeService);
+        services.registerService('traceService', TraceService);
+
+        // TODO: [fs:remove-separate-updater-and-fetcher]
+        services.set('fsEntryFetcher', new DatabaseFSEntryFetcher({
+            services: services,
+        }));
+        services.registerService('fsEntryService', DatabaseFSEntryService);
+
+        // The new fs entry service
+        services.registerService('systemFSEntryService', SystemFSEntryService);
+
+        this.log = services.get('log-service').create('filesystem-service');
+
+        // used by update_child_paths
+        this.db = services.get('database').get(DB_WRITE, 'filesystem');
+
+        const info = services.get('information');
+        info.given('fs.fsentry').provide('fs.fsentry:path')
+            .addStrategy('entry-or-delegate', async entry => {
+                if ( entry.path ) return entry.path;
+                return await info
+                    .with('fs.fsentry:uuid')
+                    .obtain('fs.fsentry:path')
+                    .exec(entry.uuid);
+            });
+
+
+        // Decorate methods with otel span
+        // TODO: language tool for traits; this is a trait
+        const span_methods = [
+            'write', 'mkdir', 'rm', 'mv', 'cp', 'read', 'stat',
+            'mkdir_2',
+            'update_child_paths',
+        ];
+        for ( const method of span_methods ) {
+            const original_method = this[method];
+            this[method] = async (...args) => {
+                const tracer = services.get('traceService').tracer;
+                let result;
+                await tracer.startActiveSpan(`fs-svc:${method}`, async span => {
+                    result = await original_method.call(this, ...args);
+                    span.end();
+                });
+                return result;
+            }
+        }
+
+        // TODO: eventually FilesystemService will extend BaseService
+        // and _init() will be called (and awaited) automatically
+        this._init();
+    }
+
+    async _init () {
+        const svc_permission = this.services.get('permission');
+        svc_permission.register_rewriter(PermissionRewriter.create({
+            matcher: permission => {
+                if ( ! permission.startsWith('fs:') ) return false;
+                const [_, specifier] = PermissionUtil.split(permission);
+                if ( ! specifier.startsWith('/') ) return false;
+                return true;
+            },
+            rewriter: async permission => {
+                const [_, path, ...rest] = PermissionUtil.split(permission);
+                console.log('checking path: ', path);
+                const node = await this.node(new NodePathSelector(path));
+                if ( ! await node.exists() ) {
+                    // TOOD: we need a general-purpose error that can have
+                    // a user-safe message, instead of using APIError
+                    // which is for API errors.
+                    throw APIError.create('subject_does_not_exist');
+                }
+                const uid = await node.get('uid');
+                if ( uid === undefined || uid === 'undefined' ) {
+                    throw new Error(`uid is undefined for path ${path}`);
+                }
+                return `fs:${uid}:${rest.join(':')}`;
+            },
+        }));
+    }
+
+    /**
+     * @deprecated - temporary migration method
+     */
+    get_systemfs () {
+        if ( ! this.systemfs_ ) {
+            this.systemfs_ = new FSAccessContext();
+            this.systemfs_.fsEntryFetcher = this.services.get('fsEntryFetcher');
+            this.systemfs_.fsEntryService = this.services.get('fsEntryService');
+            this.systemfs_.resourceService = this.services.get('resourceService');
+            this.systemfs_.sizeService = this.services.get('sizeService');
+            this.systemfs_.traceService = this.services.get('traceService');
+            this.systemfs_.services = this.services;
+        }
+        return this.systemfs_;
+    }
+
+    async owrite ({
+        node, user, immutable,
+        file, tmp, fsentry_tmp,
+        message,
+    }) {
+        const ll_owrite = new LLOWrite();
+        return await ll_owrite.run({
+            node, user, immutable,
+            file, tmp, fsentry_tmp,
+            message,
+        });
+    }
+
+    // REMINDER: There was an idea that FilesystemService implements
+    // an interface, and if that ever happens these arguments are
+    // important:
+    // parent, name, user, immutable, file, message
+    async cwrite (parameters) {
+        const ll_cwrite = new LLCWrite();
+        return await ll_cwrite.run(parameters);
+    }
+
+    async mkdir_2 ({parent, name, user, immutable}) {
+        const ll_mkdir = new LLMkdir();
+        return await ll_mkdir.run({ parent, name, user, immutable });
+    }
+
+    async mkshortcut ({ parent, name, user, target }) {
+
+        // Access Control
+        {
+            const svc_acl = this.services.get('acl');
+
+            if ( ! await svc_acl.check(user, target, 'read') ) {
+                throw await svc_acl.get_safe_acl_error(user, target, 'read');
+            }
+
+            if ( ! await svc_acl.check(user, parent, 'write') ) {
+                throw await svc_acl.get_safe_acl_error(user, parent, 'write');
+            }
+        }
+
+        if ( ! await target.exists() ) {
+            throw APIError.create('shortcut_to_does_not_exist');
+        }
+
+        await target.fetchEntry({ thumbnail: true });
+
+        const { _path, uuidv4 } = this.modules;
+        const resourceService = this.services.get('resourceService');
+        const systemFSEntryService = this.services.get('systemFSEntryService');
+
+        const ts = Math.round(Date.now() / 1000);
+        const uid = uuidv4();
+
+        resourceService.register({
+            uid,
+            status: RESOURCE_STATUS_PENDING_CREATE,
+        });
+
+        console.log('registered entry')
+
+        const raw_fsentry = {
+            is_shortcut: 1,
+            shortcut_to: target.mysql_id,
+            is_dir: target.entry.is_dir,
+            thumbnail: target.entry.thumbnail,
+            uuid: uid,
+            parent_uid: await parent.get('uid'),
+            path: _path.join(await parent.get('path'), name),
+            user_id: user.id,
+            name,
+            created: ts,
+            updated: ts,
+            modified: ts,
+            immutable: false,
+        };
+
+        this.log.debug('creating fsentry', { fsentry: raw_fsentry })
+
+        const entryOp = await systemFSEntryService.insert(raw_fsentry);
+
+        console.log('entry op', entryOp);
+
+        (async () => {
+            await entryOp.awaitDone();
+            this.log.debug('finished creating fsentry', { uid })
+            resourceService.free(uid);
+        })();
+
+        const node = await this.node(new NodeUIDSelector(uid));
+
+        const svc_event = this.services.get('event');
+        svc_event.emit('fs.create.shortcut', {
+            node,
+            context: Context.get(),
+        });
+
+        return node;
+    }
+
+    async mklink ({ parent, name, user, target }) {
+
+        // Access Control
+        {
+            const svc_acl = this.services.get('acl');
+
+            if ( ! await svc_acl.check(user, parent, 'write') ) {
+                throw await svc_acl.get_safe_acl_error(user, parent, 'write');
+            }
+        }
+
+        // We don't check if the target exists because broken links
+        // are allowed.
+
+        const { _path, uuidv4 } = this.modules;
+        const resourceService = this.services.get('resourceService');
+        const systemFSEntryService = this.services.get('systemFSEntryService');
+
+        const ts = Math.round(Date.now() / 1000);
+        const uid = uuidv4();
+
+        resourceService.register({
+            uid,
+            status: RESOURCE_STATUS_PENDING_CREATE,
+        });
+
+        const raw_fsentry = {
+            is_symlink: 1,
+            symlink_path: target,
+            is_dir: 0,
+            uuid: uid,
+            parent_uid: await parent.get('uid'),
+            path: _path.join(await parent.get('path'), name),
+            user_id: user.id,
+            name,
+            created: ts,
+            updated: ts,
+            modified: ts,
+            immutable: false,
+        };
+
+        this.log.debug('creating symlink', { fsentry: raw_fsentry })
+
+        const entryOp = await systemFSEntryService.insert(raw_fsentry);
+
+        (async () => {
+            await entryOp.awaitDone();
+            this.log.debug('finished creating symlink', { uid })
+            resourceService.free(uid);
+        })();
+
+        const node = await this.node(new NodeUIDSelector(uid));
+
+        const svc_event = this.services.get('event');
+        svc_event.emit('fs.create.symlink', {
+            node,
+            context: Context.get(),
+        });
+
+        return node;
+    }
+
+    async copy_2 (...a) {
+        const ll_copy = new LLCopy();
+        return await ll_copy.run(...a);
+    }
+
+    async update_child_paths (old_path, new_path, user_id) {
+        const monitor = PerformanceMonitor.createContext('update_child_paths');
+
+        if ( ! old_path.endsWith('/') ) old_path += '/';
+        if ( ! new_path.endsWith('/') ) new_path += '/';
+        // TODO: fs:decouple-tree-storage
+        await this.db.write(
+            `UPDATE fsentries SET path = CONCAT(?, SUBSTRING(path, ?)) WHERE path LIKE ? AND user_id = ?`,
+            [new_path, old_path.length + 1, old_path + '%', user_id]
+        );
+
+        const log = services.get('log-service').create('update_child_paths');
+        log.info(`updated ${old_path} -> ${new_path}`);
+
+        monitor.end();
+    }
+
+    /**
+     * node() returns a filesystem node using path, uid,
+     * or id associated with a filesystem node. Use this
+     * method when you need to get a filesystem node and
+     * need to collect information about the entry.
+     *
+     * @param {*} location - path, uid, or id associated with a filesystem node
+     * @returns
+     */
+    async node (selector) {
+        if ( typeof selector === 'string' ) {
+            if ( selector.startsWith('/') ) {
+                selector = new NodePathSelector(selector);
+            } else {
+                selector = new NodeUIDSelector(selector);
+            }
+        }
+
+        // TEMP: remove when these objects aren't used anymore
+        if (
+            typeof selector === 'object' &&
+            selector.constructor.name === 'Object'
+        ) {
+            if ( selector.path ) {
+                selector = new NodePathSelector(selector.path);
+            } else if ( selector.uid ) {
+                selector = new NodeUIDSelector(selector.uid);
+            } else {
+                selector = new NodeInternalIDSelector(
+                    'mysql', selector.mysql_id);
+            }
+        }
+
+        let fsNode = new FSNodeContext({
+            services: this.services,
+            selector,
+            fs: this
+        });
+        return fsNode;
+    }
+
+    /**
+     * get_entry() returns a filesystem entry using
+     * path, uid, or id associated with a filesystem
+     * node. Use this method when you need to get a
+     * filesystem entry but don't need to collect any
+     * other information about the entry.
+     *
+     * @warning The entry returned by this method is not
+     * client-safe. Use FSNodeContext to get a client-safe
+     * entry by calling it's fetchEntry() method.
+     *
+     * @param {*} param0 options for getting the entry
+     * @param {*} param0.path
+     * @param {*} param0.uid
+     * @param {*} param0.id please use mysql_id instead
+     * @param {*} param0.mysql_id
+     */
+    async get_entry ({ path, uid, id, mysql_id, ...options }) {
+        let fsNode = await this.node({ path, uid, id, mysql_id });
+        await fsNode.fetchEntry(options);
+        return fsNode.entry;
+    }
+}
+
+module.exports = {
+    FilesystemService
+};

packages/backend/src/filesystem/backends/Test.js

@@ -0,0 +1,24 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Test = void 0;
+class Test {
+}
+exports.Test = Test;

packages/backend/src/filesystem/backends/Test.ts

@@ -0,0 +1,3 @@
+export class Test {
+    //
+}

packages/backend/src/filesystem/batch/BatchExecutor.js

@@ -0,0 +1,178 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { AdvancedBase } = require('puter-js-common');
+const PathResolver = require('../../routers/filesystem_api/batch/PathResolver');
+const commands = require('./commands').commands;
+const { WorkUnit } = require('../../services/runtime-analysis/ExpectationService');
+const APIError = require('../../api/APIError');
+const { Context } = require('../../util/context');
+const config = require('../../config');
+const { TeePromise } = require('../../util/promise');
+
+class BatchExecutor extends AdvancedBase {
+    constructor (x, { user, log, errors }) {
+        super();
+        this.x = x;
+        this.user = user;
+        this.pathResolver = new PathResolver({ user });
+        this.expectations = x.get('services').get('expectations');
+        this.log = log;
+        this.errors = errors;
+        this.responsePromises = [];
+        this.hasError = false;
+
+        this.total_tbd = true;
+        this.total = 0;
+        this.counter = 0;
+
+        this.concurrent_ops = 0;
+        this.max_concurrent_ops = 20;
+        this.ops_promise = null;
+    }
+
+    async ready_for_more () {
+        if ( this.ops_promise === null ) {
+            this.ops_promise = new TeePromise();
+        }
+        await this.ops_promise;
+    }
+
+    async exec_op (req, op, file) {
+        while ( this.concurrent_ops >= this.max_concurrent_ops ) {
+            await this.ready_for_more();
+        }
+
+        this.concurrent_ops++;
+        if ( config.env == 'dev' ) {
+            const wid = this.x.get('dev_batch-widget');
+            wid.ops++;
+        }
+
+        const { expectations } = this;
+        const command_cls = commands[op.op];
+        console.log(command_cls, JSON.stringify(op, null, 2));
+        delete op.op;
+
+        const workUnit = WorkUnit.create();
+        expectations.expect_eventually({
+            workUnit,
+            checkpoint: 'operation responded'
+        });
+
+        // TEMP: event service will handle this
+        op.original_client_socket_id = req.body.original_client_socket_id;
+        op.socket_id = req.body.socket_id;
+
+        // run the operation
+        let p = this.x.arun(async () => {
+            const x= Context.get();
+            if ( ! x ) throw new Error('no context');
+
+            try {
+                if ( file ) workUnit.checkpoint(
+                    'about to run << ' +
+                    (file.originalname ?? file.name) +
+                    ' >> ' +
+                    JSON.stringify(op)
+                );
+                const command_ins = await command_cls.run({
+                    getFile: () => file,
+                    pathResolver: this.pathResolver,
+                    user: this.user
+                }, op);
+                workUnit.checkpoint('operation invoked');
+
+                const res = await command_ins.awaitValue('result');
+                // const res = await opctx.awaitValue('response');
+                workUnit.checkpoint('operation responded');
+                return res;
+            } catch (e) {
+                this.hasError = true;
+                if ( ! ( e instanceof APIError ) ) {
+                    // TODO: alarm condition
+                    this.errors.report('batch-operation', {
+                        source: e,
+                        trace: true,
+                        alarm: true,
+                    });
+
+                    e = APIError.adapt(e);
+                }
+
+                // Consume stream if there's a file
+                if ( file ) {
+                    try {
+                        // read entire stream
+                        await new Promise((resolve, reject) => {
+                            file.stream.on('end', resolve);
+                            file.stream.on('error', reject);
+                            file.stream.resume();
+                        });
+                    } catch (e) {
+                        this.errors.report('batch-operation-2', {
+                            source: e,
+                            trace: true,
+                            alarm: true,
+                        });
+                    }
+                }
+
+                if ( config.env == 'dev' ) {
+                    console.error(e);
+                    // process.exit(1);
+                }
+
+                const serialized_error = e.serialize();
+                return serialized_error;
+            } finally {
+                if ( config.env == 'dev' ) {
+                    const wid = x.get('dev_batch-widget');
+                    wid.ops--;
+                }
+                this.concurrent_ops--;
+                if ( this.ops_promise && this.concurrent_ops < this.max_concurrent_ops ) {
+                    this.ops_promise.resolve();
+                    this.ops_promise = null;
+                }
+            }
+        });
+
+        // decorate with logging
+        p = p.then(result => {
+            this.counter++;
+            const { log, total, total_tbd, counter } = this;
+            const total_str = total_tbd ? `TBD(>${total})` : `${total}`;
+            log.noticeme(`Batch Progress: ${counter} / ${total_str} operations`);
+            return result;
+        });
+
+        // this.responsePromises.push(p);
+
+        // It doesn't really matter whether or not `await` is here
+        // (that's a design flaw in the Promise API; what if you
+        // want a promise that returns a promise?)
+        const result = await p;
+        return result;
+
+    }
+}
+
+module.exports = {
+    BatchExecutor,
+};

packages/backend/src/filesystem/batch/commands.js

@@ -0,0 +1,285 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { AdvancedBase } = require("puter-js-common");
+const { AsyncProviderTrait } = require("../../traits/AsyncProviderTrait");
+const { HLMkdir, QuickMkdir } = require("../hl_operations/hl_mkdir");
+const { Context } = require("../../util/context");
+const { HLWrite } = require("../hl_operations/hl_write");
+const { get_app } = require("../../helpers");
+const { OperationFrame } = require("../../services/OperationTraceService");
+const { AppUnderUserActorType } = require("../../services/auth/Actor");
+const FSNodeParam = require("../../api/filesystem/FSNodeParam");
+const { TYPE_DIRECTORY } = require("../FSNodeContext");
+const { HLMkShortcut } = require("../hl_operations/hl_mkshortcut");
+const { HLMkLink } = require("../hl_operations/hl_mklink");
+const { HLRemove } = require("../hl_operations/hl_remove");
+
+
+class BatchCommand extends AdvancedBase {
+    static TRAITS = [
+        new AsyncProviderTrait(),
+    ]
+    static async run (executor, parameters) {
+        const instance = new this();
+        let x = Context.get();
+        const operationTraceSvc = x.get('services').get('operationTrace');
+        const frame = await operationTraceSvc.add_frame('batch:' + this.name);
+        if ( parameters.hasOwnProperty('item_upload_id') ) {
+            frame.attr('gui_metadata', {
+                ...(frame.get_attr('gui_metadata') || {}),
+                item_upload_id: parameters.item_upload_id,
+            });
+        }
+        x = x.sub({ [operationTraceSvc.ckey('frame')]: frame });
+        await x.arun(async () => {
+            await instance.run(executor, parameters);
+        });
+        frame.status = OperationFrame.FRAME_STATUS_DONE;
+        return instance;
+    }
+}
+
+class MkdirCommand extends BatchCommand {
+    async run (executor, parameters) {
+        const context = Context.get();
+        const fs = context.get('services').get('filesystem');
+
+        const parent = parameters.parent
+            ? await fs.node(await executor.pathResolver.awaitSelector(parameters.parent))
+            : undefined ;
+
+        const meta = parameters.parent
+            ? executor.pathResolver.getMeta(parameters.parent)
+            : undefined ;
+
+        if ( meta?.conflict_free ) {
+            // No potential conflict; just create the directory
+            const q_mkdir = new QuickMkdir();
+            await q_mkdir.run({
+                parent,
+                path: parameters.path,
+            });
+            if ( parameters.as ) {
+                executor.pathResolver.putSelector(
+                    parameters.as,
+                    q_mkdir.created.selector,
+                    { conflict_free: true }
+                );
+            }
+            this.setFactory('result', async () => {
+                await q_mkdir.created.awaitStableEntry();
+                const response = await q_mkdir.created.getSafeEntry();
+                return response;
+            });
+            return;
+        }
+        console.log('USING SLOW MKDIR');
+
+        const hl_mkdir = new HLMkdir();
+        const response = await hl_mkdir.run({
+            parent,
+            path: parameters.path,
+            overwrite: parameters.overwrite,
+            dedupe_name: parameters.dedupe_name,
+            create_missing_parents:
+                parameters.create_missing_ancestors ??
+                parameters.create_missing_parents ??
+                false,
+            shortcut_to: parameters.shortcut_to,
+            user: executor.user,
+        });
+        if ( parameters.as ) {
+            executor.pathResolver.putSelector(
+                parameters.as,
+                hl_mkdir.created.selector,
+                hl_mkdir.used_existing
+                    ? undefined
+                    : { conflict_free: true }
+            );
+        }
+        this.provideValue('result', response)
+    }
+}
+
+class WriteCommand extends BatchCommand {
+    async run (executor, parameters) {
+        const context = Context.get();
+        const fs = context.get('services').get('filesystem');
+
+        const uploaded_file = executor.getFile();
+
+        const destinationOrParent =
+            await fs.node(await executor.pathResolver.awaitSelector(parameters.path));
+
+        let app;
+        if ( parameters.app_uid ) {
+            app = await get_app({uid: parameters.app_uid})
+        }
+
+        const hl_write = new HLWrite();
+        const response = await hl_write.run({
+            destination_or_parent: destinationOrParent,
+            specified_name: parameters.name,
+            fallback_name: uploaded_file.originalname,
+
+            overwrite: parameters.overwrite,
+            dedupe_name: parameters.dedupe_name,
+
+            create_missing_parents:
+                parameters.create_missing_ancestors ??
+                parameters.create_missing_parents ??
+                false,
+            user: executor.user,
+
+            file: uploaded_file,
+            offset: parameters.offset,
+
+            // TODO: handle these with event service instead
+            socket_id: parameters.socket_id,
+            operation_id: parameters.operation_id,
+            item_upload_id: parameters.item_upload_id,
+            app_id: app ? app.id : null,
+        });
+
+        this.provideValue('result', response);
+
+
+        // const opctx = await fs.write(fs, {
+        //     // --- per file ---
+        //     name: parameters.name,
+        //     fallbackName: uploaded_file.originalname,
+        //     destinationOrParent,
+        //     // app_id: app ? app.id : null,
+        //     overwrite: parameters.overwrite,
+        //     dedupe_name: parameters.dedupe_name,
+        //     file: uploaded_file,
+        //     thumbnail: parameters.thumbnail,
+        //     target: parameters.target ? await req.fs.node(parameters.shortcut_to) : null,
+        //     symlink_path: parameters.symlink_path,
+        //     operation_id: parameters.operation_id,
+        //     item_upload_id: parameters.item_upload_id,
+        //     user: executor.user,
+
+        //     // --- per batch ---
+        //     socket_id: parameters.socket_id,
+        //     original_client_socket_id: parameters.original_client_socket_id,
+        // });
+
+        // opctx.onValue('response', v => this.provideValue('result', v));
+    }
+}
+
+class ShortcutCommand extends BatchCommand {
+    async run (executor, parameters) {
+        const context = Context.get();
+        const fs = context.get('services').get('filesystem');
+
+        const destinationOrParent =
+            await fs.node(await executor.pathResolver.awaitSelector(parameters.path));
+
+        const shortcut_to =
+            await fs.node(await executor.pathResolver.awaitSelector(parameters.shortcut_to));
+
+        let app;
+        if ( parameters.app_uid ) {
+            app = await get_app({uid: parameters.app_uid})
+        }
+
+        await destinationOrParent.fetchEntry({ thumbnail: true });
+        await shortcut_to.fetchEntry({ thumbnail: true });
+
+        const hl_mkShortcut = new HLMkShortcut();
+        const response = await hl_mkShortcut.run({
+            parent: destinationOrParent,
+            name: parameters.name,
+            user: executor.user,
+            target: shortcut_to,
+
+            // TODO: handle these with event service instead
+            socket_id: parameters.socket_id,
+            operation_id: parameters.operation_id,
+            item_upload_id: parameters.item_upload_id,
+            app_id: app ? app.id : null,
+        });
+
+        this.provideValue('result', response);
+    }
+}
+
+class SymlinkCommand extends BatchCommand {
+    async run (executor, parameters) {
+        const context = Context.get();
+        const fs = context.get('services').get('filesystem');
+
+        const destinationOrParent =
+            await fs.node(await executor.pathResolver.awaitSelector(parameters.path));
+
+        let app;
+        if ( parameters.app_uid ) {
+            app = await get_app({uid: parameters.app_uid})
+        }
+
+        await destinationOrParent.fetchEntry({ thumbnail: true });
+
+        const hl_mkLink = new HLMkLink();
+        const response = await hl_mkLink.run({
+            parent: destinationOrParent,
+            name: parameters.name,
+            user: executor.user,
+            target: parameters.target,
+
+            // TODO: handle these with event service instead
+            socket_id: parameters.socket_id,
+            operation_id: parameters.operation_id,
+            item_upload_id: parameters.item_upload_id,
+            app_id: app ? app.id : null,
+        });
+
+        this.provideValue('result', response);
+    }
+}
+
+class DeleteCommand extends BatchCommand {
+    async run (executor, parameters) {
+        const context = Context.get();
+        const fs = context.get('services').get('filesystem');
+
+        const target =
+            await fs.node(await executor.pathResolver.awaitSelector(parameters.path));
+
+        const hl_remove = new HLRemove();
+        const response = await hl_remove.run({
+            target,
+            user: executor.user,
+            recursive: parameters.recursive ?? false,
+            descendants_only: parameters.descendants_only ?? false,
+        });
+        this.provideValue('result', response);
+    }
+}
+
+module.exports = {
+    commands: {
+        mkdir: MkdirCommand,
+        write: WriteCommand,
+        shortcut: ShortcutCommand,
+        symlink: SymlinkCommand,
+        delete: DeleteCommand,
+    }
+};

packages/backend/src/filesystem/core/.gitignore

@@ -0,0 +1,2 @@
+# Typescript directory
+*.js

packages/backend/src/filesystem/core/BackendAPI.ts

@@ -0,0 +1,85 @@
+import { ISelector } from "./Selector";
+
+type PuterUserID = number;
+
+export const enum FSBackendSupportFlags {
+    None = 0,
+    
+    // Platform-related flags
+    PlatformCaseSensitive = 1 << 1,
+
+    // Puter support flags
+    // PuterStatOwner indicates the backend can store `user_id`
+    PuterStatOwner = 1 << 2,
+    // PuterStatApp indicates the backend can store `associated_app_id`
+    PuterStatApp = 1 << 3,
+
+    // DetailVerboseReaddir indicates the backend will provide a full
+    // stat() result for each entry in readdir().
+    DetailVerboseReaddir = 1 << 4,
+}
+
+export const enum FSNodeType {
+    File,
+    Directory,
+    PuterShortcut,
+    SymbolicLink,
+    KVStore,
+    Socket,
+}
+
+export interface IOverwriteOptions {
+    readonly overwrite: boolean;
+    UserID: PuterUserID,
+}
+
+export interface IWriteOptions extends IOverwriteOptions {
+    readonly create: boolean;
+}
+
+export interface IDeleteOptions {
+    readonly recursive: boolean;
+}
+
+export interface IStatOptions {
+    followSymlinks?: boolean;
+}
+
+export interface IStatResult {
+    uuid: string;
+    name: string;
+    type: FSNodeType;
+    size: number;
+    mtime: Date;
+    ctime: Date;
+    atime: Date;
+    immutable: boolean;
+}
+
+export interface IMiniStatResult {
+    uuid: string;
+    name: string;
+    type: FSNodeType;
+}
+
+type ReaddirResult = IMiniStatResult | IStatResult;
+
+export interface IMkdirOptions {
+    // Not for permission checks by the storage backend.
+    // A supporting storage backend will simply store this and
+    // return it in the stat() call.
+    UserID: PuterUserID,
+}
+
+export interface BackendAPI {
+    stat (selector: ISelector, options: IStatOptions): Promise<IStatResult>;
+    readdir (selector: ISelector): Promise<[string, ReaddirResult][]>;
+
+    mkdir (selector: ISelector, name: string): Promise<void>;
+    copy (from: ISelector, to: ISelector, options: IOverwriteOptions): Promise<void>;
+    rename (from: ISelector, to: ISelector, options: IOverwriteOptions): Promise<void>;
+    delete (selector: ISelector, options: IDeleteOptions): Promise<void>;
+
+    read_file (selector: ISelector): Promise<Buffer>;
+    write_file (selector: ISelector, data: Buffer, options: IOverwriteOptions): Promise<void>;
+}

packages/backend/src/filesystem/core/Selector.ts

@@ -0,0 +1,65 @@
+import * as _path from 'path';
+import * as _util from 'util';
+
+type TemporeryNodeType = any;
+
+export interface ISelector {
+    describe (showDebug?: boolean): string;
+    setPropertiesKnownBySelector (node: object): void;
+}
+
+export class NodePathSelector {
+    public value: string;
+
+    constructor (path: string) {
+        this.value = path;
+    }
+
+    public describe (showDebug?: boolean): string {
+        return this.value;
+    }
+
+    public setPropertiesKnownBySelector (node: TemporeryNodeType): void {
+        node.path = this.value;
+        node.name = _path.basename(this.value);
+    }
+}
+
+export class NodeInternalUIDSelector {
+    public value: string;
+
+    constructor (uid: string) {
+        this.value = uid;
+    }
+
+    public describe (showDebug?: boolean): string {
+        return `[uid:${this.value}]`;
+    }
+
+    public setPropertiesKnownBySelector (node: TemporeryNodeType): void {
+        node.uid = this.value;
+    }
+}
+
+export class NodeInternalIDSelector {
+    constructor (
+        public service: string,
+        public id: number,
+        public debugInfo: any
+    ) { }
+
+    public describe (showDebug?: boolean): string {
+        if ( showDebug ) {
+            return `[db:${this.id}] (${
+                _util.inspect(this.debugInfo)
+            })`;
+        }
+        return `[db:${this.id}]`;
+    }
+
+    public setPropertiesKnownBySelector (node: TemporeryNodeType): void {
+        if ( this.service === 'mysql' ) {
+            node.id = this.id;
+        }
+    }
+}

packages/backend/src/filesystem/hl_operations/definitions.js

@@ -0,0 +1,25 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { BaseOperation } = require('../../services/OperationTraceService');
+
+class HLFilesystemOperation extends BaseOperation {}
+
+module.exports = {
+    HLFilesystemOperation
+};

packages/backend/src/filesystem/hl_operations/hl_copy.js

@@ -0,0 +1,221 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const APIError = require("../../api/APIError");
+const { chkperm, validate_fsentry_name, get_user, is_ancestor_of } = require("../../helpers");
+const { TYPE_DIRECTORY } = require("../FSNodeContext");
+const { NodePathSelector, RootNodeSelector } = require("../node/selectors");
+const { HLFilesystemOperation } = require("./definitions");
+const { MkTree } = require("./hl_mkdir");
+const { HLRemove } = require("./hl_remove");
+
+class HLCopy extends HLFilesystemOperation {
+    static DESCRIPTION = `
+        High-level copy operation.
+
+        This operation is a wrapper around the low-level copy operation.
+        It provides the following features:
+        - create missing parent directories
+        - overwrite existing files or directories
+        - deduplicate files/directories with the same name
+    `
+
+    static MODULES = {
+        _path: require('path'),
+    }
+
+    static PARAMETERS = {
+        source: {},
+        destionation_or_parent: {},
+        new_name: {},
+
+        overwrite: {},
+        dedupe_name: {},
+
+        create_missing_parents: {},
+
+        user: {},
+    }
+
+    async _run () {
+        const { _path } = this.modules;
+
+        const { values, context } = this;
+        const svc = context.get('services');
+        const fs = svc.get('filesystem');
+
+        let parent = values.destination_or_parent;
+        let dest = null;
+
+        const source = values.source;
+
+        if ( values.overwrite && values.dedupe_name ) {
+            throw APIError.create('overwrite_and_dedupe_exclusive');
+        }
+
+        if ( ! await source.exists() ) {
+            throw APIError.create('source_does_not_exist');
+        }
+
+        if ( ! await chkperm(source.entry, values.user.id, 'cp') ) {
+            throw APIError.create('forbidden');
+        }
+
+        if ( await parent.get('is-root') ) {
+            throw APIError.create('cannot_copy_to_root');
+        }
+
+        // If parent exists and is a file, and a new name wasn't
+        // specified, the intention must be to overwrite the file.
+        if (
+            ! values.new_name &&
+            await parent.exists() &&
+            await parent.get('type') !== TYPE_DIRECTORY
+        ) {
+            dest = parent;
+            parent = await dest.getParent();
+            await parent.fetchEntry();
+        }
+
+        // If parent is not found either throw an error or create
+        // the parent directory as specified by parameters.
+        if ( ! await parent.exists() ) {
+            if ( ! (parent.selector instanceof NodePathSelector) ) {
+                throw APIError.create('dest_does_not_exist', null, {
+                    parent: parent.selector,
+                });
+            }
+            const path = parent.selector.value;
+            const tree_op = new MkTree();
+            await tree_op.run({
+                parent: await fs.node(new RootNodeSelector()),
+                tree: [path],
+            });
+            await parent.fetchEntry({ force: true });
+        }
+
+        if (
+            await parent.get('type') !== TYPE_DIRECTORY
+        ) {
+            throw APIError.create('dest_is_not_a_directory');
+        }
+
+        if ( ! await chkperm(parent.entry, values.user.id, 'write') ) {
+            throw APIError.create('forbidden');
+        }
+
+        let target_name = values.new_name ?? await source.get('name');
+
+        try {
+            validate_fsentry_name(target_name);
+        } catch (e) {
+            throw APIError.create(400, e);
+        }
+
+        // NEXT: implement _verify_room with profiling
+        const tracer = svc.get('traceService').tracer;
+        await tracer.startActiveSpan(`fs:cp:verify-size-constraints`, async span => {
+            const source_file = source.entry;
+            const dest_fsentry = parent.entry;
+
+            let source_user = await get_user({id: source_file.user_id});
+            let dest_user = source_user.id !== dest_fsentry.user_id
+                ? await get_user({id: dest_fsentry.user_id})
+                : source_user ;
+            const sizeService = svc.get('sizeService');
+            let deset_usage = await sizeService.get_usage(dest_user.id);
+
+            const size = await source.fetchSize(values.user);
+            let capacity = (dest_user.free_storage === undefined || dest_user.free_storage === null) ? config.storage_capacity : dest_user.free_storage
+            if(capacity - deset_usage - size < 0){
+                throw APIError.create('storage_limit_reached');
+            }
+            span.end();
+        });
+
+        if ( dest === null ) {
+            dest = await parent.getChild(target_name);
+        }
+
+        // Ensure copy operation is legal
+        // TODO: maybe this is better in the low-level operation
+        if ( await source.get('uid') == await parent.get('uid') ) {
+            throw APIError.create('source_and_dest_are_the_same');
+        }
+
+        if ( await is_ancestor_of(source.mysql_id, parent.mysql_id) ) {
+            throw APIError('cannot_copy_item_into_itself');
+        }
+
+        if ( await dest.exists() ) {
+            // condition: no overwrite behaviour specified
+            if ( ! values.overwrite && ! values.dedupe_name ) {
+                throw APIError.create('item_with_same_name_exists', null, {
+                    entry_name: dest.entry.name
+                });
+            }
+
+            if ( values.dedupe_name ) {
+                const fsEntryFetcher = context.get('services').get('fsEntryFetcher');
+                const target_ext = _path.extname(target_name);
+                const target_noext = _path.basename(target_name, target_ext);
+                for ( let i=1 ;; i++ ) {
+                    const try_new_name = `${target_noext} (${i})${target_ext}`;
+                    const exists = await fsEntryFetcher.nameExistsUnderParent(
+                        parent.uid, try_new_name
+                    );
+                    if ( ! exists ) {
+                        target_name = try_new_name;
+                        break;
+                    }
+                }
+
+                dest = await parent.getChild(target_name);
+            }
+            else if ( values.overwrite ) {
+                if ( ! await chkperm(dest.entry, options.user.id, 'rm') ) {
+                    throw APIError.create('forbidden');
+                }
+
+                // TODO: This will be LLRemove
+                // TODO: what to do with parent_operation?
+                const hl_remove = new HLRemove();
+                await hl_remove.run({
+                    target: dest,
+                    user: values.user,
+                    recursive: true,
+                });
+            }
+        }
+
+        this.copied = await fs.copy_2({
+            source,
+            parent,
+            user: values.user,
+            target_name,
+        })
+
+        await this.copied.awaitStableEntry();
+        const response = await this.copied.getSafeEntry({ thumbnail: true });
+        return response;
+    }
+}
+
+module.exports = {
+    HLCopy,
+};

packages/backend/src/filesystem/hl_operations/hl_data_read.js

@@ -0,0 +1,101 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { stream_to_buffer } = require("../../util/streamutil");
+const { HLFilesystemOperation } = require("./definitions");
+
+/**
+ * HLDataRead reads a stream of objects from a file containing structured data.
+ * For .jsonl files, the stream will product multiple objects.
+ * For .json files, the stream will produce a single object.
+ */
+class HLDataRead extends HLFilesystemOperation {
+    static MODULES = {
+        'stream': require('stream'),
+    }
+
+    async _run () {
+        const { context } = this;
+
+        // We get the user from context so that an elevated system context
+        // can read files under the system user.
+        const user = await context.get('user');
+
+        const {
+            fsNode,
+        } = this.values;
+
+        if ( ! await fsNode.exists() ) {
+            throw APIError.create('subject_does_not_exist');
+        }
+
+        if ( ! await chkperm(fsNode.entry, user.id, 'read') ) {
+            throw APIError.create('forbidden');
+        }
+
+        const ll_read = new LLRead();
+        let stream = await ll_read.run({
+            fsNode, user,
+            version_id,
+        });
+
+        stream = this._stream_bytes_to_lines(stream);
+        stream = this._stream_jsonl_lines_to_objects(stream);
+
+        return stream;
+    }
+
+    _stream_bytes_to_lines (stream) {
+        const readline = require('readline');
+        const rl = readline.createInterface({
+          input: stream,
+          terminal: false
+        });
+
+        const { PassThrough } = this.modules.stream;
+
+        const output_stream = new PassThrough();
+
+        new Promise((resolve, reject) => {
+            rl.on('line', (line) => {
+                output_stream.write(line);
+            });
+            rl.on('close', () => {
+                output_stream.end();
+                resolve();
+            });
+        });
+
+        return output_stream;
+    }
+
+    _stream_jsonl_lines_to_objects (stream) {
+        const output_stream = new PassThrough();
+        (async () => {
+            for await (const line of stream) {
+                output_stream.write(JSON.parse(line));
+            }
+            output_stream.end();
+        })();
+        return output_stream;
+    }
+}
+
+module.exports = {
+    HLDataRead
+};

packages/backend/src/filesystem/hl_operations/hl_mkdir.js

@@ -0,0 +1,476 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { chkperm } = require('../../helpers');
+
+const { RootNodeSelector, NodeChildSelector, NodePathSelector } = require("../node/selectors");
+const APIError = require('../../api/APIError');
+
+const FSNodeParam = require('../../api/filesystem/FSNodeParam');
+const StringParam = require('../../api/filesystem/StringParam');
+const FlagParam = require("../../api/filesystem/FlagParam");
+const UserParam = require('../../api/filesystem/UserParam');
+const FSNodeContext = require('../FSNodeContext');
+const { ContextAwareTrait } = require('../../traits/ContextAwareTrait');
+const { OtelTrait } = require('../../traits/OtelTrait');
+const { HLFilesystemOperation } = require('./definitions');
+const { is_valid_path } = require('../validation');
+const { HLRemove } = require('./hl_remove');
+
+class MkTree extends HLFilesystemOperation {
+    static DESCRIPTION = `
+        High-level operation for making directory trees
+
+        The following input for 'tree':
+        ['a/b/c', ['i/j/k'], ['p', ['q'], ['r/s']]]]
+
+        Would create a directory tree like this:
+        a
+        └── b
+            └── c
+                ├── i
+                │   └── j
+                │       └── k
+                └── p
+                    ├── q
+                    └── r
+                        └── s
+    `
+
+    static PARAMETERS = {
+        parent: new FSNodeParam('parent', { optional: true }),
+    }
+
+    static PROPERTIES = {
+        leaves: () => [],
+        directories_created: () => [],
+    }
+
+    async _run () {
+        const { values, context } = this;
+
+        await this.create_branch_({
+            parent_node: values.parent || await fs.node(new RootNodeSelector()),
+            tree: values.tree,
+            parent_exists: true,
+        });
+    }
+
+    async create_branch_ ({ parent_node, tree, parent_exists }) {
+        const { context, values } = this;
+        const { _path } = this.modules;
+        const fs = context.get('services').get('filesystem');
+        const user = context.get('user');
+
+        const trunk = tree[0];
+        const branches = tree.slice(1);
+
+        let current = parent_node.selector;
+        let lastCreatedSelector = parent_node.selector;
+
+        // trunk = a/b/c
+
+        const dirs = trunk === '.' ? []
+            : trunk.split('/').filter(Boolean);
+
+        console.log('DIRS', dirs, parent_node.selector.describe())
+
+        // dirs = [a, b, c]
+
+        let parent_did_exist = parent_exists;
+
+        // This is just a loop that goes through each part of the path
+        // until it finds the first directory that doesn't exist yet.
+        let i = 0;
+        if ( parent_exists ) for ( ; i < dirs.length ; i++ ) {
+            const dir = dirs[i];
+            const currentParent = current;
+            current = new NodeChildSelector(current, dir);
+
+            const maybe_dir = await fs.node(current);
+
+            if ( maybe_dir.isRoot ) continue;
+            if ( await maybe_dir.isUserDirectory() ) continue;
+
+            if ( await maybe_dir.exists() ) {
+
+                if ( await maybe_dir.get('type') !== FSNodeContext.TYPE_DIRECTORY ) {
+                    throw APIError.create('dest_is_not_a_directory');
+                }
+
+                continue;
+            }
+
+            current = currentParent;
+            parent_exists = false;
+            break;
+        }
+
+        if ( parent_did_exist && ! parent_exists ) {
+            const node = await fs.node(current);
+            const has_perm = await chkperm(await node.get('entry'), user.id, 'write');
+            if ( ! has_perm ) throw APIError.create('permission_denied');
+        }
+
+        // This next loop creates the new directories
+
+        // We break into a second loop because we know none of these directories
+        // exist yet. If we continued those checks each child operation would
+        // wait for the previous one to complete because FSNodeContext::fetchEntry
+        // will notice ResourceService has a lock on the previous operation
+        // we started.
+
+        // In this way it goes nyyyoooom because all the database inserts
+        // happen concurrently (and probably end up in the same batch).
+
+        for ( ; i < dirs.length ; i++ ) {
+            const dir = dirs[i];
+            const currentParent = current;
+            current = new NodeChildSelector(current, dir);
+
+            const node = await fs.mkdir_2({
+                parent: await fs.node(currentParent),
+                name: current.name,
+                user,
+            })
+
+            current = node.selector;
+
+            this.directories_created.push(node);
+        }
+
+        const bottom_parent = await fs.node(current);
+
+        console.log('BOTTOM PARENT', bottom_parent.selector.describe());
+
+        if ( branches.length === 0 ) {
+            this.leaves.push(bottom_parent);
+        }
+
+        for ( const branch of branches ) {
+            await this.create_branch_({
+                parent_node: bottom_parent,
+                tree: branch,
+                parent_exists,
+            });
+        }
+    }
+}
+
+class QuickMkdir extends HLFilesystemOperation {
+    async _run () {
+        const { context, values } = this;
+        let { parent, path } = values;
+        const { _path } = this.modules;
+        const fs = context.get('services').get('filesystem');
+        const user = context.get('user');
+
+        parent = parent || await fs.node(new RootNodeSelector());
+
+        let current = parent.selector;
+
+        const dirs = path === '.' ? []
+            : path.split('/').filter(Boolean);
+
+        const api = require('@opentelemetry/api');
+        const currentSpan = api.trace.getSpan(api.context.active());
+        if ( currentSpan ) {
+            currentSpan.setAttribute('path', path);
+            currentSpan.setAttribute('dirs', dirs.join('/'));
+            currentSpan.setAttribute('parent', parent.selector.describe());
+        }
+
+
+
+        for ( let i=0 ; i < dirs.length ; i++ ) {
+            const dir = dirs[i];
+            const currentParent = current;
+            current = new NodeChildSelector(current, dir);
+
+            const node = await fs.mkdir_2({
+                parent: await fs.node(currentParent),
+                name: current.name,
+                user,
+            })
+
+            current = node.selector;
+
+            // this.directories_created.push(node);
+        }
+
+        this.created = await fs.node(current);
+    }
+}
+
+class HLMkdir extends HLFilesystemOperation {
+    static DESCRIPTION = `
+        High-level mkdir operation.
+
+        This operation is a wrapper around the low-level mkdir operation.
+        It provides the following features:
+        - create missing parent directories
+        - overwrite existing files
+        - dedupe names
+        - create shortcuts
+    `
+
+    static PARAMETERS = {
+        parent: new FSNodeParam('parent', { optional: true }),
+        path: new StringParam('path'),
+        overwrite: new FlagParam('overwrite', { optional: true }),
+        create_missing_parents: new FlagParam('create_missing_parents', { optional: true }),
+        user: new UserParam(),
+
+        shortcut_to: new FSNodeParam('shortcut_to', { optional: true }),
+    };
+
+    static MODULES = {
+        _path: require('path'),
+        socketio: require('../../socketio.js'),
+    }
+
+    static PROPERTIES = {
+        parent_directories_created: () => [],
+    }
+
+    static TRAITS = [
+        new OtelTrait([
+            '_get_existing_parent',
+            '_create_parents',
+        ]),
+    ]
+
+    async _run () {
+        const { context, values } = this;
+        const { _path, socketio } = this.modules;
+        const fs = context.get('services').get('filesystem');
+
+        if ( ! is_valid_path(values.path, {
+            no_relative_components: true,
+            allow_path_fragment: true,
+        }) ) {
+            throw APIError.create('field_invalid', null, {
+                key: 'path',
+                expected: 'valid path',
+                got: 'invalid path',
+            });
+        }
+
+        let parent_node = values.parent || await fs.node(new RootNodeSelector());
+        console.log('USING PARENT', parent_node.selector.describe());
+        let target_basename = _path.basename(values.path);
+
+        const top_parent = values.create_missing_parents
+            ? await this._create_top_parent({ top_parent: parent_node })
+            : await this._get_existing_top_parent({ top_parent: parent_node })
+            ;
+
+        // `parent_node` becomes the parent of the last directory name
+        // specified under `path`.
+        parent_node = await this._create_parents({
+            parent_node: top_parent,
+            user: values.user,
+        });
+
+        const has_perm = await chkperm(await parent_node.get('entry'), values.user.id, 'write');
+        if ( ! has_perm ) throw APIError.create('permission_denied');
+
+        const existing = await fs.node(
+            new NodeChildSelector(parent_node.selector, target_basename)
+        );
+
+        await existing.fetchEntry();
+
+        if ( existing.found ) {
+            const { overwrite, dedupe_name, create_missing_parents } = values;
+            if ( overwrite ) {
+                // TODO: tag rm operation somehow
+                const has_perm = await chkperm(await existing.get('entry'), values.user.id, 'write');
+                if ( ! has_perm ) throw APIError.create('permission_denied');
+                const hl_remove = new HLRemove();
+                await hl_remove.run({
+                    target: existing,
+                    user: values.user,
+                    recursive: true,
+                });
+            }
+            else if ( dedupe_name ) {
+                const fsEntryFetcher = context.get('services').get('fsEntryFetcher');
+                for ( let i=1 ;; i++ ) {
+                    let try_new_name = `${target_basename} (${i})`;
+                    const exists = await fsEntryFetcher.nameExistsUnderParent(
+                        existing.entry.parent_uid, try_new_name
+                    );
+                    if ( ! exists ) {
+                        target_basename = try_new_name;
+                        break;
+                    }
+                }
+            }
+            else if ( create_missing_parents ) {
+                if ( ! existing.entry.is_dir ) {
+                    throw APIError.create('dest_is_not_a_directory');
+                }
+                this.created = existing;
+                this.used_existing = true;
+                return {};
+            } else {
+                throw APIError.create('item_with_same_name_exists', null, {
+                    entry_name: target_basename,
+                });
+            }
+        }
+
+        if ( values.shortcut_to ) {
+            const shortcut_to = values.shortcut_to;
+            if ( ! await shortcut_to.exists() ) {
+                throw APIError.create('shortcut_to_does_not_exist');
+            }
+            if ( ! shortcut_to.entry.is_dir ) {
+                throw APIError.create('shortcut_target_is_a_directory');
+            }
+            const has_perm = await chkperm(shortcut_to.entry, values.user.id, 'read');
+            if ( ! has_perm ) throw APIError.create('forbidden');
+
+            this.created = await fs.mkshortcut({
+                parent: parent_node,
+                name: target_basename,
+                user: values.user,
+                target: shortcut_to,
+            });
+
+            await this.created.awaitStableEntry();
+            return await this.created.getSafeEntry();
+        }
+
+        this.created = await fs.mkdir_2({
+            parent: parent_node,
+            name: target_basename,
+            user: values.user,
+        });
+
+        const all_nodes = [
+            ...this.parent_directories_created,
+            this.created,
+        ];
+
+        await Promise.all(all_nodes.map(node => node.awaitStableEntry()));
+
+        const response = await this.created.getSafeEntry();
+        response.parent_dirs_created = [];
+        for ( const node of this.parent_directories_created ) {
+            response.parent_dirs_created.push(await node.getSafeEntry());
+        }
+        response.requested_path = values.path;
+
+        return response;
+    }
+
+    async _create_parents ({ parent_node, user }) {
+        const { context, values } = this;
+        const { _path } = this.modules;
+        const fs = context.get('services').get('filesystem');
+
+        let current = parent_node.selector;
+        let lastCreatedSelector = null;
+
+        const tree_op = new MkTree();
+        await tree_op.run({
+            parent: parent_node,
+            tree: [_path.dirname(values.path)],
+        });
+
+        this.parent_directories_created = tree_op.directories_created;
+
+        return tree_op.leaves[0];
+    }
+
+    async _get_existing_parent ({ parent_node }) {
+        const { context, values } = this;
+        const { _path } = this.modules;
+        const fs = context.get('services').get('filesystem');
+
+        const target_dirname = _path.dirname(values.path);
+        const dirs = target_dirname === '.' ? []
+            : target_dirname.split('/').filter(Boolean);
+
+        let current = parent_node.selector;
+        for ( let i=0 ; i < dirs.length ; i++ ) {
+            current = new NodeChildSelector(current, dirs[i]);
+        }
+
+        const node = await fs.node(current);
+
+        if ( ! await node.exists() ) {
+            console.log('HERE FROM', node.selector.describe(), parent_node.selector.describe());
+            throw APIError.create('dest_does_not_exist');
+        }
+
+        if ( ! node.entry.is_dir ) {
+            throw APIError.create('dest_is_not_a_directory');
+        }
+
+        return node;
+    }
+
+    async _create_top_parent ({ top_parent }) {
+        if ( await top_parent.exists() ) {
+            if ( ! top_parent.entry.is_dir ) {
+                throw APIError.create('dest_is_not_a_directory');
+            }
+            return top_parent;
+        }
+
+        const maybe_path_selector =
+            top_parent.get_selector_of_type(NodePathSelector);
+
+        if ( ! maybe_path_selector ) {
+            throw APIError.create('dest_does_not_exist');
+        }
+
+        const path = maybe_path_selector.value;
+
+        const fs = this.context.get('services').get('filesystem');
+
+        const tree_op = new MkTree();
+        await tree_op.run({
+            parent: await fs.node(new RootNodeSelector()),
+            tree: [path],
+        });
+
+        return tree_op.leaves[0];
+    }
+
+    async _get_existing_top_parent ({ top_parent }) {
+        if ( ! await top_parent.exists() ) {
+            throw APIError.create('dest_does_not_exist');
+        }
+
+        if ( ! top_parent.entry.is_dir ) {
+            throw APIError.create('dest_is_not_a_directory');
+        }
+
+        return top_parent;
+    }
+}
+
+module.exports = {
+    QuickMkdir,
+    HLMkdir,
+    MkTree,
+};

packages/backend/src/filesystem/hl_operations/hl_mklink.js

@@ -0,0 +1,80 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const FSNodeParam = require("../../api/filesystem/FSNodeParam");
+const StringParam = require("../../api/filesystem/StringParam");
+const { HLFilesystemOperation } = require("./definitions");
+const { chkperm } = require("../../helpers");
+const APIError = require("../../api/APIError");
+const { TYPE_DIRECTORY } = require("../FSNodeContext");
+
+class HLMkLink extends HLFilesystemOperation {
+    static PARAMETERS = {
+        parent: new FSNodeParam('symlink'),
+        name: new StringParam('name'),
+        target: new StringParam('target'),
+    }
+
+    static MODULES = {
+        path: require('node:path'),
+    }
+
+    async _run () {
+        const { context, values } = this;
+        const { _path } = this.modules;
+        const fs = context.get('services').get('filesystem');
+
+        const { target, parent, user } = values;
+        let { name, dedupe_name } = values;
+
+        if ( ! name ) {
+            throw APIError.create('field_empty', null, { key: 'name' });
+        }
+
+        if ( ! await parent.exists() ) {
+            throw APIError.create('dest_does_not_exist');
+        }
+
+        if ( await parent.get('type') !== TYPE_DIRECTORY ) {
+            throw APIError.create('dest_is_not_a_directory');
+        }
+
+        {
+            const dest = await parent.getChild(name);
+            if ( await dest.exists() ) {
+                throw APIError.create('item_with_same_name_exists', null, {
+                    entry_name: name,
+                });
+            }
+        }
+
+        const created = await fs.mklink({
+            target,
+            parent,
+            name,
+            user,
+        });
+
+        await created.awaitStableEntry();
+        return await created.getSafeEntry();
+    }
+}
+
+module.exports = {
+    HLMkLink,
+};

packages/backend/src/filesystem/hl_operations/hl_mkshortcut.js

@@ -0,0 +1,107 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const APIError = require("../../api/APIError");
+const FSNodeParam = require("../../api/filesystem/FSNodeParam");
+const FlagParam = require("../../api/filesystem/FlagParam");
+const StringParam = require("../../api/filesystem/StringParam");
+const { chkperm } = require("../../helpers");
+const { TYPE_DIRECTORY } = require("../FSNodeContext");
+const { HLFilesystemOperation } = require("./definitions");
+
+class HLMkShortcut extends HLFilesystemOperation {
+    static PARAMETERS = {
+        parent: new FSNodeParam('shortcut'),
+        name: new StringParam('name'),
+        target: new FSNodeParam('target'),
+
+        dedupe_name: new FlagParam('dedupe_name', { optional: true }),
+    }
+
+    static MODULES = {
+        path: require('node:path'),
+    }
+
+    async _run () {
+        console.log('HLMKSHORTCUT IS HAPPENING')
+        const { context, values } = this;
+        const { _path, socketio } = this.modules;
+        const fs = context.get('services').get('filesystem');
+
+        const { target, parent, user } = values;
+        let { name, dedupe_name } = values;
+
+        if ( ! await target.exists() ) {
+            throw APIError.create('shortcut_to_does_not_exist');
+        }
+
+        if ( ! name ) {
+            dedupe_name = true;
+            name = 'Shortcut to ' + await target.get('name');
+        }
+
+        {
+            const has_perm = await chkperm(target.entry, values.user.id, 'read');
+            if ( ! has_perm ) throw APIError.create('permission_denied');
+        }
+
+        if ( ! await parent.exists() ) {
+            throw APIError.create('dest_does_not_exist');
+        }
+
+        if ( await parent.get('type') !== TYPE_DIRECTORY ) {
+            throw APIError.create('dest_is_not_a_directory');
+        }
+
+        {
+            const dest = await parent.getChild(name);
+            if ( await dest.exists() ) {
+                if ( ! dedupe_name ) {
+                    throw APIError.create('item_with_same_name_exists', null, {
+                        entry_name: name,
+                    });
+                }
+
+                const name_ext = this.modules.path.extname(name);
+                const name_noext = this.modules.path.basename(name, name_ext);
+                for ( let i=1 ;; i++ ) {
+                    const try_new_name = `${name_noext} (${i})${name_ext}`;
+                    const try_dest = await parent.getChild(try_new_name);
+                    if ( ! await try_dest.exists() ) {
+                        name = try_new_name;
+                        break;
+                    }
+                }
+            }
+        }
+
+        const created = await fs.mkshortcut({
+            target,
+            parent,
+            name,
+            user,
+        });
+
+        await created.awaitStableEntry();
+        return await created.getSafeEntry();
+    }
+}
+
+module.exports = {
+    HLMkShortcut,
+};

packages/backend/src/filesystem/hl_operations/hl_move.js

@@ -0,0 +1,198 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const APIError = require("../../api/APIError");
+const { chkperm, validate_fsentry_name, is_ancestor_of } = require("../../helpers");
+const { LLMove } = require("../ll_operations/ll_move");
+const { RootNodeSelector } = require("../node/selectors");
+const { HLFilesystemOperation } = require("./definitions");
+const { MkTree } = require("./hl_mkdir");
+const { HLRemove } = require("./hl_remove");
+const { TYPE_DIRECTORY } = require("../FSNodeContext");
+
+class HLMove extends HLFilesystemOperation {
+    static MODULES = {
+        _path: require('path'),
+    }
+    async _run () {
+        const { _path } = this.modules;
+
+        const { context, values } = this;
+        const svc = context.get('services');
+        const fs = svc.get('filesystem');
+
+        const new_metadata  = typeof values.new_metadata === 'string'
+            ? values.new_metadata : JSON.stringify(values.new_metadata);
+
+        // !! new_name, create_missing_parents, overwrite, dedupe_name
+
+        let parent = values.destination_or_parent;
+        let dest = null;
+        const source = values.source;
+
+        if ( await source.get('is-root') ) {
+            throw APIError.create('immutable');
+        }
+        if ( await parent.get('is-root') ) {
+            throw APIError.create('cannot_copy_to_root');
+        }
+
+        if ( ! await source.exists() ) {
+            throw APIError.create('source_does_not_exist');
+        }
+
+        if ( ! await chkperm(source.entry, values.user.id, 'cp') ) {
+            throw APIError.create('forbidden');
+        }
+
+        if ( source.entry.immutable ) {
+            throw APIError.create('immutable');
+        }
+
+        // If the "parent" is a file, then it's actually our destination; not the parent.
+        if ( ! values.new_name && await parent.exists() && await parent.get('type') !== TYPE_DIRECTORY ) {
+            dest = parent;
+            parent = await dest.getParent();
+        }
+
+        if ( ! await parent.exists() ) {
+            if ( ! parent.path || ! values.create_missing_parents ) {
+                throw APIError.create('dest_does_not_exist');
+            }
+
+            const tree_op = new MkTree();
+            await tree_op.run({
+                parent: await fs.node(new RootNodeSelector()),
+                tree: [parent.path],
+            });
+
+            parent = tree_op.leaves[0];
+        }
+
+        await parent.fetchEntry();
+        if ( ! await chkperm(parent.entry, values.user.id, 'write') ) {
+            throw APIError.create('forbidden');
+        }
+        if ( await parent.get('type') !== TYPE_DIRECTORY ) {
+            throw APIError.create('dest_is_not_a_directory');
+        }
+
+        let source_user, dest_user;
+
+        // 3. Verify cross-user size constraints
+        const src_user_id = await source.get('user_id');
+        const par_user_id = await parent.get('user_id');
+        if ( src_user_id !== par_user_id ) {
+            source_user = await get_user({id: src_user_id});
+            if(source_user.id !== par_user_id)
+                dest_user = await get_user({id: par_user_id});
+            else
+                dest_user = source_user;
+            await source.fetchSize();
+            const item_size = source.entry.size;
+            let capacity = (dest_user.free_storage === undefined || dest_user.free_storage === null) ? config.storage_capacity : dest_user.free_storage
+            if(capacity - await df(dest_user.id) - item_size < 0){
+                throw APIError.create('storage_limit_reached');
+            }
+        }
+
+        let target_name = values.new_name ?? await source.get('name');
+        const metadata = new_metadata ?? await source.get('metadata');
+
+        try {
+            validate_fsentry_name(target_name);
+        } catch (e) {
+            throw APIError.create(400, e);
+        }
+
+        if ( dest === null ) {
+            dest = await parent.getChild(target_name);
+        }
+
+        const src_uid = await source.get('uid');
+        // const dst_uid = await dest.get('uid');
+        const par_uid = await parent.get('uid');
+
+        if ( src_uid === par_uid ) {
+            throw APIError.create('source_and_dest_are_the_same');
+        }
+        if ( await is_ancestor_of(src_uid, par_uid) ) {
+            throw APIError('cannot_move_item_into_itself');
+        }
+
+        let overwritten;
+        if ( await dest.exists() ) {
+            if ( ! values.overwrite && ! values.dedupe_name ) {
+                throw APIError.create('item_with_same_name_exists', null, {
+                    entry_name: target_name,
+                });
+            }
+
+            if ( values.dedupe_name ) {
+                const svc_fsEntryFetcher = svc.get('fsEntryFetcher');
+                const target_ext = _path.extname(target_name);
+                const target_noext = _path.basename(target_name, target_ext);
+                for ( let i=1 ;; i++ ) {
+                    const try_new_name = `${target_noext} (${i})${target_ext}`;
+                    const exists = await svc_fsEntryFetcher.nameExistsUnderParent(
+                        parent.uid, try_new_name
+                    );
+                    if ( ! exists ) {
+                        target_name = try_new_name;
+                        break;
+                    }
+                }
+
+                dest = await parent.getChild(target_name);
+            }
+            else if ( values.overwrite ) {
+                overwritten = await dest.getSafeEntry();
+                const hl_remove = new HLRemove();
+                await hl_remove.run({
+                    target: dest,
+                    user: values.user,
+                });
+            }
+            else { throw new Error('unreachable'); }
+        }
+
+        const old_path = await source.get('path');
+
+        const ll_move = new LLMove();
+        const source_new = await ll_move.run({
+            source,
+            parent,
+            target_name,
+            user: values.user,
+            metadata: metadata,
+        });
+
+        await source_new.awaitStableEntry();
+        await source_new.fetchSuggestedApps();
+        await source_new.fetchOwner();
+        return {
+            moved: await source_new.getSafeEntry({ thumbnail: true }),
+            overwritten,
+            old_path,
+        }
+    }
+}
+
+module.exports = {
+    HLMove,
+};

packages/backend/src/filesystem/hl_operations/hl_read.js

@@ -0,0 +1,99 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const APIError = require("../../api/APIError");
+const { chkperm } = require("../../helpers");
+const { TYPE_SYMLINK } = require("../FSNodeContext");
+const { LLRead } = require("../ll_operations/ll_read");
+const { HLFilesystemOperation } = require("./definitions");
+
+class HLRead extends HLFilesystemOperation {
+    static MODULES = {
+        'stream': require('stream'),
+    }
+
+    async _run () {
+        const { context } = this;
+        const {
+            fsNode, actor,
+            line_count, byte_count,
+            offset,
+            version_id,
+        } = this.values;
+
+        if ( ! await fsNode.exists() ) {
+            throw APIError.create('subject_does_not_exist');
+        }
+
+        const ll_read = new LLRead();
+        const stream = await ll_read.run({
+            fsNode, actor,
+            version_id,
+            ...(byte_count !== undefined ? {
+                offset: offset ?? 0,
+                length: byte_count
+            } : {}),
+        });
+
+        if ( line_count !== undefined ) {
+            stream = this._wrap_stream_line_count(stream, line_count);
+        }
+
+        return stream;
+    }
+
+    /**
+     * returns a new stream that will only produce the first `line_count` lines
+     * @param {*} stream - input stream
+     * @param {*} line_count - number of lines to produce
+     */
+    _wrap_stream_line_count (stream, line_count) {
+        const readline = require('readline');
+        const rl = readline.createInterface({
+          input: stream,
+          terminal: false
+        });
+
+        const { PassThrough } = this.modules.stream;
+
+        const output_stream = new PassThrough();
+
+        let lines_read = 0;
+        new Promise((resolve, reject) => {
+            rl.on('line', (line) => {
+                if(lines_read++ >= line_count){
+                    return rl.close();
+                }
+
+                output_stream.write(lines_read > 1 ? '\r\n' + line : line);
+            });
+            rl.on('error', () => {
+                console.log('error');
+            });
+            rl.on('close', function () {
+                resolve();
+            });
+        });
+
+        return output_stream;
+    }
+}
+
+module.exports = {
+    HLRead
+};

packages/backend/src/filesystem/hl_operations/hl_readdir.js

@@ -0,0 +1,65 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const APIError = require("../../api/APIError");
+const { chkperm } = require("../../helpers");
+const { TYPE_DIRECTORY } = require("../FSNodeContext");
+const { LLReadDir } = require("../ll_operations/ll_readdir");
+const { HLFilesystemOperation } = require("./definitions");
+
+class HLReadDir extends HLFilesystemOperation {
+    async _run () {
+        const { subject, user, no_thumbs, no_assocs } = this.values;
+
+        if ( ! await subject.exists() ) {
+            throw APIError.create('subject_does_not_exist');
+        }
+
+        if ( await subject.get('type') !== TYPE_DIRECTORY ) {
+            const { context } = this;
+            const svc_acl = context.get('services').get('acl');
+            if ( ! await svc_acl.check(actor, subject, 'see') ) {
+                throw await svc_acl.get_safe_acl_error(actor, subject, 'see');
+            }
+            return [subject];
+        }
+
+        const ll_readdir = new LLReadDir();
+        const children = await ll_readdir.run(this.values);
+
+        return Promise.all(children.map(async child => {
+            // await child.fetchAll(null, user);
+            if ( ! no_assocs ) {
+                await child.fetchSuggestedApps(user);
+                await child.fetchSubdomains(user);
+            }
+            const fs = require('fs');
+            fs.appendFileSync('/tmp/children.log',
+                JSON.stringify({
+                    no_thumbs,
+                    no_assocs,
+                    entry: child.entry,
+                }) + '\n');
+            return await child.getSafeEntry({ thumbnail: ! no_thumbs });
+        }));
+    }
+}
+
+module.exports = {
+    HLReadDir,
+};

packages/backend/src/filesystem/hl_operations/hl_remove.js

@@ -0,0 +1,57 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const APIError = require("../../api/APIError");
+const { chkperm } = require("../../helpers");
+const { TYPE_DIRECTORY } = require("../FSNodeContext");
+const { LLRmDir } = require("../ll_operations/ll_rmdir");
+const { LLRmNode } = require("../ll_operations/ll_rmnode");
+const { HLFilesystemOperation } = require("./definitions");
+
+class HLRemove extends HLFilesystemOperation {
+    static PARAMETERS = {
+        target: {},
+        user: {},
+        recursive: {},
+        descendants_only: {},
+    }
+
+    async _run () {
+        const { target, user } = this.values;
+
+        if ( ! await target.exists() ) {
+            throw APIError.create('subject_does_not_exist');
+        }
+
+        if ( ! chkperm(target.entry, user.id, 'rm') ) {
+            throw APIError.create('forbidden');
+        }
+
+        if ( await target.get('type') === TYPE_DIRECTORY ) {
+            const ll_rmdir = new LLRmDir();
+            return await ll_rmdir.run(this.values);
+        }
+
+        const ll_rmnode = new LLRmNode();
+        return await ll_rmnode.run(this.values);
+    }
+}
+
+module.exports = {
+    HLRemove,
+};

packages/backend/src/filesystem/hl_operations/hl_stat.js

@@ -0,0 +1,76 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { chkperm } = require("../../helpers");
+const { Context } = require("../../util/context");
+const { HLFilesystemOperation } = require("./definitions");
+
+class HLStat extends HLFilesystemOperation {
+    static MODULES = {
+        ['mime-types']: require('mime-types'),
+    }
+
+    async _run () {
+        const {
+            subject, user,
+            return_subdomains,
+            return_permissions,
+            return_versions,
+            return_size,
+        } = this.values;
+
+        await subject.fetchEntry();
+
+        // file not found
+        if( ! subject.found ) throw APIError.create('subject_does_not_exist');
+
+        await subject.fetchOwner();
+
+        const context = Context.get();
+        const svc_acl = context.get('services').get('acl');
+        const actor = context.get('actor');
+        if ( ! await svc_acl.check(actor, subject, 'read') ) {
+            throw await svc_acl.get_safe_acl_error(actor, subject.entry, 'read');
+        }
+
+        // check permission
+        // TODO: this check is redundant now that ACL is used;
+        //   we will need to remove it to implement user-user permissions
+        if(user && !await chkperm(subject.entry, user.id, 'stat')){
+            throw { code:`forbidden`, message: `permission denied.`};
+        }
+
+        // TODO: why is this specific to stat?
+        const mime = this.require('mime-types');
+        const contentType = mime.contentType(subject.entry.name)
+        subject.entry.type = contentType ? contentType : null;
+
+        if (return_size) await subject.fetchSize(user);
+        if (return_subdomains) await subject.fetchSubdomains(user)
+        if (return_permissions) await subject.fetchShares();
+        if (return_versions) await subject.fetchVersions();
+
+        await subject.fetchIsEmpty();
+
+        return await subject.getSafeEntry();
+    }
+}
+
+module.exports = {
+    HLStat
+};

packages/backend/src/filesystem/hl_operations/hl_write.js

@@ -0,0 +1,408 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const APIError = require("../../api/APIError");
+const FSNodeParam = require("../../api/filesystem/FSNodeParam");
+const FlagParam = require("../../api/filesystem/FlagParam");
+const StringParam = require("../../api/filesystem/StringParam");
+const UserParam = require("../../api/filesystem/UserParam");
+const config = require("../../config");
+const { chkperm, validate_fsentry_name } = require("../../helpers");
+const { TeePromise } = require("../../util/promise");
+const { pausing_tee, logging_stream, offset_write_stream } = require("../../util/streamutil");
+const { TYPE_DIRECTORY } = require("../FSNodeContext");
+const { LLRead } = require("../ll_operations/ll_read");
+const { RootNodeSelector, NodePathSelector } = require("../node/selectors");
+const { is_valid_node_name } = require("../validation");
+const { HLFilesystemOperation } = require("./definitions");
+const { MkTree } = require("./hl_mkdir");
+
+class WriteCommonTrait {
+    install_in_instance (instance) {
+        instance._verify_size = async function () {
+            if (
+                this.values.file &&
+                this.values.file.size > config.max_file_size
+            ) {
+                throw APIError.create('file_too_large', null, {
+                    max_size: config.max_file_size,
+                })
+            }
+
+            if (
+                this.values.thumbnail &&
+                this.values.thumbnail.size > config.max_thumbnail_size
+            ) {
+                throw APIError.create('thumbnail_too_large', null, {
+                    max_size: config.max_thumbnail_size,
+                })
+            }
+        }
+
+        instance._verify_room = async function () {
+            if ( ! this.values.file ) return;
+
+            const sizeService = this.context.get('services').get('sizeService');
+            const { file, user } = this.values;
+
+            const usage = await sizeService.get_usage(user.id);
+            let capacity = user.free_storage == undefined
+                ? config.storage_capacity : user.free_storage;
+            if( capacity - usage - file.size < 0 ) {
+                throw APIError.create('storage_limit_reached');
+            }
+        }
+    }
+}
+
+class HLWrite extends HLFilesystemOperation {
+    static DESCRIPTION = `
+        High-level write operation.
+
+        This operation is a wrapper around the low-level write operation.
+        It provides the following features:
+        - create missing parent directories
+        - overwrite existing files
+        - deduplicate files with the same name
+        // - create thumbnails; this will happen in low-level operation for now
+        - create shortcuts
+    `
+
+    static TRAITS = [
+        new WriteCommonTrait(),
+    ]
+
+    static PARAMETERS = {
+        // the parent directory, or a filepath that doesn't exist yet
+        destination_or_parent: new FSNodeParam('path'),
+
+        // if specified, destination_or_parent must be a directory
+        specified_name: new StringParam('specified_name', { optional: true }),
+
+        // used if specified_name is undefined and destination_or_parent is a directory
+        // NB: if destination_or_parent does not exist and create_missing_parents
+        //     is true then destination_or_parent will be a directory
+        fallback_name: new StringParam('fallback_name', { optional: true }),
+
+        overwrite: new FlagParam('overwrite', { optional: true }),
+        dedupe_name: new FlagParam('dedupe_name', { optional: true }),
+
+        // other options
+        shortcut_to: new FSNodeParam('shortcut_to', { optional: true }),
+        create_missing_parents: new FlagParam('create_missing_parents', { optional: true }),
+        user: new UserParam(),
+
+        // file: multer.File
+    };
+
+    static MODULES = {
+        _path: require('path'),
+        socketio: require('../../socketio.js'),
+        mime: require('mime-types'),
+    }
+
+    async _run () {
+        const { context, values } = this;
+        const { _path } = this.modules;
+
+        const fs = context.get('services').get('filesystem');
+
+        let parent = values.destination_or_parent;
+        let destination = null;
+
+        await this._verify_size();
+        await this._verify_room();
+
+        this.checkpoint('before parent exists check');
+
+        if ( ! await parent.exists() && values.create_missing_parents ) {
+            if ( ! (parent.selector instanceof NodePathSelector) ) {
+                throw APIError.create('dest_does_not_exist', null, {
+                    parent: parent.selector,
+                });
+            }
+            const path = parent.selector.value;
+            this.log.noticeme('EXPECTED PATH', { path });
+            const tree_op = new MkTree();
+            await tree_op.run({
+                parent: await fs.node(new RootNodeSelector()),
+                tree: [path],
+            });
+
+            parent = await fs.node(new NodePathSelector(path));
+            const parent_exists_now = await parent.exists();
+            if ( ! parent_exists_now ) {
+                this.log.error('FAILED TO CREATE DESTINATION');
+                throw APIError.create('dest_does_not_exist', null, {
+                    parent: parent.selector,
+                });
+            }
+        }
+
+        if ( parent.isRoot ) {
+            throw APIError.create('cannot_write_to_root');
+        }
+
+        let target_name = values.specified_name || values.fallback_name;
+
+        // If a name is specified then the destination must be a directory
+        if ( values.specified_name ) {
+            this.checkpoint('specified name condition');
+            if ( ! await parent.exists() ) {
+                throw APIError.create('dest_does_not_exist');
+            }
+            if ( await parent.get('type') !== TYPE_DIRECTORY ) {
+                throw APIError.create('dest_is_not_a_directory');
+            }
+            target_name = values.specified_name;
+        }
+
+        this.checkpoint('check parent DNE or is not a directory');
+        if (
+            ! await parent.exists() ||
+            await parent.get('type') !== TYPE_DIRECTORY
+        ) {
+            destination = parent;
+            parent = await destination.getParent();
+            target_name = destination.name;
+        }
+
+        if ( parent.isRoot ) {
+            throw APIError.create('cannot_write_to_root');
+        }
+
+        if ( values.user && ! await chkperm(await parent.get('entry'), values.user.id, 'write') ) {
+            throw APIError.create('forbidden');
+        }
+
+        try {
+            // old validator is kept here to avoid changing the
+            // error messages; eventually is_valid_node_name
+            // will support more detailed error reporting
+            validate_fsentry_name(target_name);
+            if ( ! is_valid_node_name(target_name) ) {
+                throw { message: 'invalid node name' };
+            }
+        } catch (e) {
+            throw APIError.create('invalid_file_name', null, {
+                name: target_name,
+                reason: e.message,
+            });
+        }
+
+        if ( ! destination ) {
+            destination = await parent.getChild(target_name);
+        }
+
+        let is_overwrite = false;
+
+        // TODO: Gotta come up with a reasonable guideline for if/when we put
+        //       object members in the scope; it feels too arbitrary right now.
+        const { overwrite, dedupe_name } = values;
+
+        this.checkpoint('before overwrite behaviours');
+
+        const dest_exists = await destination.exists();
+
+        if ( values.offset !== undefined && ! dest_exists ) {
+            throw APIError.create('offset_without_existing_file');
+        }
+
+        if ( dest_exists ) {
+            console.log('DESTINATION EXISTS', dedupe_name)
+            if ( ! overwrite && ! dedupe_name ) {
+                throw APIError.create('item_with_same_name_exists', null, {
+                    entry_name: target_name
+                });
+            }
+
+            if ( dedupe_name ) {
+                const fsEntryFetcher = context.get('services').get('fsEntryFetcher');
+                const target_ext = _path.extname(target_name);
+                const target_noext = _path.basename(target_name, target_ext);
+                for ( let i=1 ;; i++ ) {
+                    const try_new_name = `${target_noext} (${i})${target_ext}`;
+                    const exists = await fsEntryFetcher.nameExistsUnderParent(
+                        parent.uid, try_new_name
+                    );
+                    if ( ! exists ) {
+                        target_name = try_new_name;
+                        break;
+                    }
+                }
+
+                destination = await parent.getChild(target_name);
+            }
+
+            else if ( overwrite ) {
+                if ( await destination.get('immutable') ) {
+                    throw APIError.create('immutable');
+                }
+                if ( await destination.get('type') === TYPE_DIRECTORY ) {
+                    throw APIError.create('cannot_overwrite_a_directory');
+                }
+                is_overwrite = true;
+            }
+        }
+
+        if ( values.shortcut_to ) {
+            this.checkpoint('shortcut condition');
+            const shortcut_to = values.shortcut_to;
+            if ( ! await shortcut_to.exists() ) {
+                throw APIError.create('shortcut_to_does_not_exist');
+            }
+            if ( await shortcut_to.get('type') === TYPE_DIRECTORY ) {
+                throw APIError.create('shortcut_target_is_a_directory');
+            }
+            const has_perm = await chkperm(shortcut_to.entry, values.user.id, 'read');
+            if ( ! has_perm ) throw APIError.create('permission_denied');
+
+            this.created = await fs.mkshortcut({
+                parent,
+                name: target_name,
+                user: values.user,
+                target: shortcut_to,
+            });
+
+            await this.created.awaitStableEntry();
+            await this.created.fetchEntry({ thumbnail: true });
+            return await this.created.getSafeEntry();
+        }
+
+        this.checkpoint('before thumbnail');
+
+        let thumbnail_promise = new TeePromise();
+        let thumbnail; (async () => {
+            const reason = await (async () => {
+                const { mime } = this.modules;
+                const thumbnails = context.get('services').get('thumbnails');
+                if ( values.thumbnail ) return 'already thumbnail';
+
+                const content_type = mime.contentType(target_name);
+                console.log('CONTENT TYPE', content_type);
+                if ( ! content_type ) return 'no content type';
+                if ( ! thumbnails.is_supported_mimetype(content_type) ) return 'unsupported content type';
+                if ( ! thumbnails.is_supported_size(values.file.size) ) return 'too large';
+
+                // Create file object for thumbnail by either using an existing
+                // buffer (ex: /download endpoint) or by forking a stream
+                // (ex: /write and /batch endpoints).
+                const thumb_file = (() => {
+                    if ( values.file.buffer ) return values.file;
+
+                    const [replace_stream, thumbnail_stream] =
+                        pausing_tee(values.file.stream, 2);
+
+                    values.file.stream = replace_stream;
+                    return { ...values.file, stream: thumbnail_stream };
+                })();
+
+                thumbnail = await thumbnails.thumbify(thumb_file);
+                thumbnail_promise.resolve(thumbnail);
+            })();
+            if ( reason ) {
+                console.log('REASON', reason);
+                thumbnail_promise.resolve(undefined);
+
+                // values.file.stream = logging_stream(values.file.stream);
+            }
+        })();
+
+        this.checkpoint('before delegate');
+
+        if ( values.offset !== undefined ) {
+            if ( ! is_overwrite ) {
+                throw APIError.create('offset_requires_overwrite');
+            }
+
+            if ( ! values.file.stream ) {
+                throw APIError.create('offset_requires_stream');
+            }
+
+            const replace_length = values.file.size;
+            let dst_size = await destination.get('size');
+            if ( values.offset > dst_size ) {
+                values.offset = dst_size;
+            }
+
+            if ( values.offset + values.file.size > dst_size ) {
+                dst_size = values.offset + values.file.size;
+            }
+
+            const ll_read = new LLRead();
+            const read_stream = await ll_read.run({
+                fsNode: destination,
+            });
+
+            values.file.stream = offset_write_stream({
+                originalDataStream: read_stream,
+                newDataStream: values.file.stream,
+                offset: values.offset,
+                replace_length,
+            });
+            values.file.size = dst_size;
+        }
+
+        if ( is_overwrite ) {
+            this.written = await fs.owrite({
+                node: destination,
+                user: values.user,
+                file: values.file,
+                tmp: {
+                    socket_id: values.socket_id,
+                    operation_id: values.operation_id,
+                    item_upload_id: values.item_upload_id,
+                },
+                fsentry_tmp: {
+                    thumbnail_promise,
+                },
+                message: values.message,
+            });
+        } else {
+            this.written = await fs.cwrite({
+                parent,
+                name: target_name,
+                user: values.user,
+                file: values.file,
+                tmp: {
+                    socket_id: values.socket_id,
+                    operation_id: values.operation_id,
+                    item_upload_id: values.item_upload_id,
+                },
+                fsentry_tmp: {
+                    thumbnail_promise,
+                },
+                message: values.message,
+                app_id: values.app_id,
+            });
+        }
+
+        this.checkpoint('after delegate');
+
+        await this.written.awaitStableEntry();
+        this.checkpoint('after await stable entry');
+        const response = await this.written.getSafeEntry({ thumbnail: true });
+        this.checkpoint('after get safe entry');
+
+        return response;
+    }
+}
+
+module.exports = {
+    HLWrite,
+};

packages/backend/src/filesystem/lib/PuterPath.js

@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const _path = require('path');
+
+/**
+ * Puter paths look like any of the following:
+ * 
+ * Absolute path: /user/dir1/dir2/file
+ * From UID: AAAA-BBBB-CCCC-DDDD/../a/b/c
+ * 
+ * The difference between an absolute path and a UID-relative path
+ * is the leading forward-slash character.
+ */
+class PuterPath {
+    static NULL_UUID = '00000000-0000-0000-0000-000000000000';
+
+    static adapt (value) {
+        if ( value instanceof PuterPath ) return value;
+        return new PuterPath(value);
+    }
+
+    constructor (text) {
+        this.text = text;
+    }
+
+    set text (text) {
+        this.text_ = text.trim();
+        this.normUnix = _path.normalize(text);
+        this.normFlat =
+            (this.normUnix.endsWith('/') && this.normUnix.length > 1)
+            ? this.normUnix.slice(0, -1) : this.normUnix;
+    }
+    get text () { return this.text_; }
+
+    isRoot () {
+        if ( this.normFlat === '/' ) return true;
+        if ( this.normFlat === this.constructor.NULL_UUID ) {
+            return true;
+        }
+        return false;
+    }
+
+    isAbsolute () {
+        return this.text.startsWith('/');
+    }
+
+    isFromUID () {
+        return ! this.isAbsolute();
+    }
+
+    get hasRelativePortion () {
+
+    }
+
+    get reference () {
+        if ( this.isAbsolute ) return this.constructor.NULL_UUID;
+
+        return this.text.slice(0, this.text.indexOf('/'));
+    }
+
+    get relativePortion () {
+        if ( this.isAbsolute() ) {
+            return this.text.slice(1);
+        }
+
+        if ( ! this.text.includes('/') ) return '';
+        return this.text.slice(this.text.indexOf('/') + 1);
+    }
+}
+
+module.exports = { PuterPath };

packages/backend/src/filesystem/ll_operations/definitions.js

@@ -0,0 +1,25 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { BaseOperation } = require("../../services/OperationTraceService");
+
+class LLFilesystemOperation extends BaseOperation {}
+
+module.exports = {
+    LLFilesystemOperation
+};

packages/backend/src/filesystem/ll_operations/ll_copy.js

@@ -0,0 +1,257 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const config = require('../../config');
+const { Context } = require('../../util/context');
+const { ParallelTasks } = require('../../util/otelutil');
+const FSNodeContext = require('../FSNodeContext');
+const { NodeUIDSelector } = require('../node/selectors');
+const { RESOURCE_STATUS_PENDING_CREATE } = require('../storage/ResourceService');
+const { UploadProgressTracker } = require('../storage/UploadProgressTracker');
+const { LLFilesystemOperation } = require('./definitions');
+
+class LLCopy extends LLFilesystemOperation {
+    static MODULES = {
+        _path: require('path'),
+        uuidv4: require('uuid').v4,
+    }
+
+    async _run () {
+        const { _path, uuidv4 } = this.modules;
+        const { context } = this;
+        const { source, parent, user, actor, target_name } = this.values;
+        const svc = context.get('services');
+
+        const tracer = svc.get('traceService').tracer;
+        const fs = svc.get('filesystem');
+        const svc_event = svc.get('event');
+
+        const uuid = uuidv4();
+        const ts = Math.round(Date.now()/1000);
+
+        this.field('target-uid', uuid);
+        this.field('source', source.selector.describe());
+
+        this.checkpoint('before fetch parent entry');
+        await parent.fetchEntry();
+        this.checkpoint('before fetch source entry');
+        await source.fetchEntry({ thumbnail: true });
+        this.checkpoint('fetched source and parent entries');
+
+        console.log('PATH PARAMETERS', {
+            path: await parent.get('path'),
+            target_name,
+        })
+
+        // Access Control
+        {
+            const svc_acl = context.get('services').get('acl');
+            this.checkpoint('copy :: access control');
+
+            // Check read access to source
+            if ( ! await svc_acl.check(actor, source, 'read') ) {
+                throw await svc_acl.get_safe_acl_error(actor, source, 'read');
+            }
+
+            // Check write access to destination
+            if ( ! await svc_acl.check(actor, parent, 'write') ) {
+                throw await svc_acl.get_safe_acl_error(actor, source, 'write');
+            }
+        }
+
+        const raw_fsentry = {
+            uuid,
+            is_dir: source.entry.is_dir,
+            ...(source.entry.is_shortcut ? {
+                is_shortcut: source.entry.is_shortcut,
+                shortcut_to: source.entry.shortcut_to,
+            } :{}),
+            parent_uid: parent.uid,
+            name: target_name,
+            created: ts,
+            modified: ts,
+
+            path: _path.join(await parent.get('path'), target_name),
+
+            // if property exists but the value is undefined,
+            // it will still be included in the INSERT, causing
+            // an error
+            ...(source.entry.thumbnail ?
+                { thumbnail: source.entry.thumbnail } : {}),
+
+            user_id: user.id,
+        };
+
+        svc_event.emit('fs.pending.file', {
+            fsentry: FSNodeContext.sanitize_pending_entry_info(raw_fsentry),
+            context: this.context,
+        })
+
+        this.checkpoint('emitted fs.pending.file');
+
+        if ( await source.get('has-s3') ) {
+            Object.assign(raw_fsentry, {
+                size: source.entry.size,
+                associated_app_id: source.entry.associated_app_id,
+                bucket: source.entry.bucket,
+                bucket_region: source.entry.bucket_region,
+            });
+
+            await tracer.startActiveSpan(`fs:cp:storage-copy`, async span => {
+                let progress_tracker = new UploadProgressTracker();
+
+                svc_event.emit('fs.storage.progress.copy', {
+                    upload_tracker: progress_tracker,
+                    context: Context.get(),
+                    meta: {
+                        item_uid: uuid,
+                        item_path: raw_fsentry.path,
+                    }
+                });
+
+                this.checkpoint('emitted fs.storage.progress.copy');
+
+                // const storage = new PuterS3StorageStrategy({ services: svc });
+                const storage = Context.get('storage');
+                const state_copy = storage.create_copy();
+                await state_copy.run({
+                    src_node: source,
+                    dst_storage: {
+                        key: uuid,
+                        bucket: raw_fsentry.bucket,
+                        bucket_region: raw_fsentry.bucket_region,
+                    },
+                    storage_api: { progress_tracker },
+                });
+
+                this.checkpoint('finished storage copy');
+                span.end();
+            });
+        }
+
+        {
+            const svc_size = svc.get('sizeService');
+            await svc_size.add_node_size(undefined, source, user);
+
+            this.checkpoint('added source size');
+        }
+
+        const svc_resource = svc.get('resourceService');
+        svc_resource.register({
+            uid: uuid,
+            status: RESOURCE_STATUS_PENDING_CREATE,
+        });
+
+        const svc_fsentry = svc.get('systemFSEntryService');
+        this.log.info(`inserting entry: ` + uuid);
+        const entryOp = await svc_fsentry.insert(raw_fsentry);
+
+        let node;
+
+        this.checkpoint('before parallel tasks');
+        const tasks = new ParallelTasks({ tracer, max: 4 });
+        await tracer.startActiveSpan(`fs:cp:parallel-portion`, async span => {
+            this.checkpoint('starting parallel tasks');
+            // Add child copy tasks if this is a directory
+            if ( source.entry.is_dir ) {
+                const fsEntryService = svc.get('fsEntryService');
+                const children = await fsEntryService.fast_get_direct_descendants(
+                    source.uid
+                );
+                for ( const child_uuid of children ) {
+                    tasks.add(`fs:cp:copy-child`, async () => {
+                        const child_node = await fs.node(
+                            new NodeUIDSelector(child_uuid)
+                        );
+                        const child_name = await child_node.get('name');
+                        // TODO: this should be LLCopy instead
+                        const ll_copy = new LLCopy();
+                        console.log('LL Copy Start');
+                        await ll_copy.run({
+                            source: await fs.node(
+                                new NodeUIDSelector(child_uuid)
+                            ),
+                            parent: await fs.node(
+                                new NodeUIDSelector(uuid)
+                            ),
+                            user,
+                            target_name: child_name,
+                        });
+                        console.log('LL Copy End');
+                        // const hl_copy = new HLCopy();
+                        // await hl_copy.run({
+                        //     destination_or_parent: await fs.node(
+                        //         new NodeUIDSelector(uuid)
+                        //     ),
+                        //     source: await fs.node(
+                        //         new NodeUIDSelector(child_uuid)
+                        //     ),
+                        //     user
+                        // });
+                        // await fs.cp(fs, {
+                        //     source: await fs.node(
+                        //         new NodeUIDSelector(child_uuid)
+                        //     ),
+                        //     // TODO: don't do this when cp supports uuids
+                        //     destinationOrParent: await fs.node(
+                        //         new NodeUIDSelector(uuid)
+                        //     ),
+                        //     user,
+                        //     overwrite: false,
+                        //     create_missing_parents: false,
+                        //     ancestor_check_not_needed: true,
+                        // });
+                    });
+                }
+            }
+
+            // Add task to await entry
+            tasks.add(`fs:cp:entry-op`, async () => {
+                await entryOp.awaitDone();
+                svc_resource.free(uuid);
+                this.log.info(`done inserting entry: ` + uuid);
+                const copy_fsNode = await fs.node(new NodeUIDSelector(uuid));
+                copy_fsNode.entry = raw_fsentry;
+                copy_fsNode.found = true;
+                copy_fsNode.path = raw_fsentry.path;
+
+                node = copy_fsNode;
+
+                svc_event.emit('fs.create.file', {
+                    node,
+                    context: this.context,
+                })
+            }, { force: true });
+
+            this.checkpoint('waiting for parallel tasks');
+            await tasks.awaitAll();
+            this.checkpoint('finishing up');
+
+            span.end();
+        });
+
+        node = node || await fs.node(new NodeUIDSelector(uuid));
+
+        // TODO: What event do we emit? How do we know if we're overwriting?
+        return node;
+    }
+}
+
+module.exports = {
+    LLCopy,
+};

packages/backend/src/filesystem/ll_operations/ll_copy_idea.js

@@ -0,0 +1,139 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+/*
+
+    This file describes an idea to make fine-grained
+    steps of a filesystem operation more declarative.
+
+    This could have advantages like:
+    - easier tracking of side-effects
+    - steps automatically mark checkpoints
+    - steps automatically have tracing
+    - implications of re-ordering steps would
+      always be known
+    - easier to diagnose stuck operations
+
+*/
+
+const STEPS_COPY_CONTENTS = [
+    {
+        id: 'add storage info to fsentry',
+        behaviour: 'none',
+        fn: async ({ util, values }) => {
+            const { source } = values;
+            // "util.assign" makes it possible to
+            // track changes caused by this step
+            util.assign('raw_fsentry', {
+                size: source.entry.size,
+                // ...
+            })
+        }
+    },
+    {
+        id: 'create progress tracker',
+        behaviour: 'values',
+        fn: async () => {
+            const progress_tracker =
+                new UploadProgressTracker();
+            return {
+                progress_tracker
+            };
+        }
+    },
+    {
+        id: 'emit copy progress event',
+        behaviour: 'side-effect',
+        fn: async ({ services }) => {
+            services.event.emit(
+                /// ...
+            )
+        }
+    },
+    {
+        id: 'get storage backend',
+        behaviour: 'values',
+        fn: async ({ services }) => {
+            const storage = new
+                PuterS3StorageStrategy({
+                    services
+                })
+            return { storage };
+        }
+    },
+    // ...
+]
+
+const STEPS = [
+    {
+        id: 'generate uuid and ts',
+        behaviour: 'values',
+        fn: async ({ modules }) => {
+            return {
+                uuid: modules.uuidv4(),
+                ts: Math.round(Date.now()/1000)
+            };
+        }
+    },
+    {
+        id: 'redundancy fetch',
+        behaviour: 'side-effect',
+        fn: async ({ values }) => {
+            await values.source.fetchEntry({
+                thumbnail: true,
+            });
+            await values.parent.fetchEntry();
+        }
+    },
+    {
+        id: 'generate raw fsentry',
+        behaviour: 'values',
+        fn: async ({ values }) => {
+            const {
+                source,
+                parent, target_name,
+                uuid, ts,
+                user,
+            } = values;
+            const raw_fsentry = {
+                uuid,
+                is_dir: source.entry.is_dir,
+                // ...
+            };
+            return { raw_fsentry };
+        }
+    },
+    {
+        id: 'emit fs.pending.file',
+        fn: () => {
+            // ...
+        }
+    },
+    {
+        id: 'copy contents',
+        cond: async ({ values }) => {
+            return await values.source.get('has-s3');
+        },
+        steps: STEPS_COPY_CONTENTS,
+    },
+    // ...
+]
+
+class LLCopy extends LLFilesystemOperation {
+    static STEPS = STEPS
+}

packages/backend/src/filesystem/ll_operations/ll_mkdir.js

@@ -0,0 +1,139 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const APIError = require("../../api/APIError");
+const { MODE_WRITE } = require("../../services/fs/FSLockService");
+const { Context } = require("../../util/context");
+const { TYPE_DIRECTORY } = require("../FSNodeContext");
+const { NodeUIDSelector, NodeChildSelector } = require("../node/selectors");
+const { RESOURCE_STATUS_PENDING_CREATE } = require("../storage/ResourceService");
+const { LLFilesystemOperation } = require("./definitions");
+
+class LLMkdir extends LLFilesystemOperation {
+    static MODULES = {
+        _path: require('path'),
+        uuidv4: require('uuid').v4,
+    }
+
+    async _run () {
+        const { context } = this;
+        const { parent, name } = this.values;
+
+        this.checkpoint('lock requested');
+        this.log.noticeme('GET FSLOCK');
+        const svc_fslock = context.get('services').get('fslock');
+        this.log.noticeme('REQUESTING LOCK');
+        const lock_handle = await svc_fslock.lock_child(
+            await parent.get('path'),
+            name,
+            MODE_WRITE,
+        );
+        this.log.noticeme('GOT LOCK');
+        this.checkpoint('lock acquired');
+
+        try {
+            return await this._locked_run();
+        } finally {
+            await lock_handle.unlock();
+        }
+    }
+    async _locked_run () {
+        const { _path, uuidv4 } = this.modules;
+        const { context } = this;
+        const { parent, name, user, immutable, actor } = this.values;
+
+        const ts = Math.round(Date.now() / 1000);
+        const uid = uuidv4();
+        const resourceService = context.get('services').get('resourceService');
+        const systemFSEntryService = context.get('services').get('systemFSEntryService');
+        const svc_event = context.get('services').get('event');
+        const fs = context.get('services').get('filesystem');
+
+        this.field('fsentry-uid', uid);
+
+        const existing = await fs.node(
+            new NodeChildSelector(parent.selector, name)
+        );
+
+        if ( await existing.exists() ) {
+            throw APIError.create('item_with_same_name_exists', null, {
+                entry_name: name,
+            });
+        }
+
+        this.checkpoint('before acl');
+        const svc_acl = context.get('services').get('acl');
+        if ( ! await parent.exists() ) {
+            throw APIError.create('subject_does_not_exist');
+        }
+        if ( ! await svc_acl.check(actor, parent, 'write') ) {
+            throw await svc_acl.get_safe_acl_error(actor, parent, 'write');
+        }
+
+        resourceService.register({
+            uid,
+            status: RESOURCE_STATUS_PENDING_CREATE,
+        });
+
+        const raw_fsentry = {
+            is_dir: 1,
+            uuid: uid,
+            parent_uid: await parent.get('uid'),
+            path: _path.join(await parent.get('path'), name),
+            user_id: user.id,
+            name,
+            created: ts,
+            accessed: ts,
+            modified: ts,
+            immutable: immutable ?? false,
+            ...(this.values.thumbnail ? {
+                thumbnail: this.values.thumbnail,
+            } : {}),
+        };
+
+        this.log.debug('creating fsentry', { fsentry: raw_fsentry })
+
+        this.checkpoint('about to enqueue insert');
+        const entryOp = await systemFSEntryService.insert(raw_fsentry);
+
+        this.field('fsentry-created', false);
+
+        this.checkpoint('enqueued insert');
+        // Asynchronous behaviour temporarily disabled
+        // (async () => {
+            await entryOp.awaitDone();
+            this.log.debug('finished creating fsentry', { uid })
+            resourceService.free(uid);
+            this.field('fsentry-created', true);
+        // })();
+
+        const node = await fs.node(new NodeUIDSelector(uid));
+
+        svc_event.emit('fs.create.directory', {
+            node,
+            context: Context.get(),
+        });
+
+        this.checkpoint('returning node');
+        return node
+    }
+}
+
+module.exports = {
+    LLMkdir,
+};

packages/backend/src/filesystem/ll_operations/ll_move.js

@@ -0,0 +1,84 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { LLFilesystemOperation } = require("./definitions");
+
+class LLMove extends LLFilesystemOperation {
+    static MODULES = {
+        _path: require('path'),
+    }
+
+    async _run () {
+        const { _path } = this.modules;
+        const { context } = this;
+        const { source, parent, user, actor, target_name, metadata } = this.values;
+        const svc = context.get('services');
+
+        // Access Control
+        {
+            const svc_acl = context.get('services').get('acl');
+            this.checkpoint('move :: access control');
+
+            // Check write access to source
+            if ( ! await svc_acl.check(actor, source, 'write') ) {
+                throw await svc_acl.get_safe_acl_error(actor, source, 'write');
+            }
+
+            // Check write access to destination
+            if ( ! await svc_acl.check(actor, parent, 'write') ) {
+                throw await svc_acl.get_safe_acl_error(actor, parent, 'write');
+            }
+        }
+
+        const old_path = await source.get('path');
+
+        const svc_fsEntry = svc.get('fsEntryService');
+        const op_update = await svc_fsEntry.update(source.uid, {
+            ...(
+                await source.get('parent_uid') !== await parent.get('uid')
+                ? { parent_uid: await parent.get('uid') }
+                : {}
+            ),
+            path: _path.join(await parent.get('path'), target_name),
+            name: target_name,
+            ...(metadata ? { metadata } : {}),
+        });
+
+        source.entry.name = target_name;
+        source.entry.path = _path.join(await parent.get('path'), target_name);
+
+        await op_update.awaitDone();
+
+        const svc_fs = svc.get('filesystem');
+        await svc_fs.update_child_paths(old_path, source.entry.path, user.id);
+
+        const svc_event = svc.get('event');
+
+        await svc_event.emit('fs.move.file', {
+            context: this.context,
+            moved: source,
+            old_path,
+        });
+
+        return source;
+    }
+}
+
+module.exports = {
+    LLMove,
+};

packages/backend/src/filesystem/ll_operations/ll_read.js

@@ -0,0 +1,153 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const APIError = require("../../api/APIError");
+const { CodeModel } = require("../../codex/CodeModel");
+const { Sequence } = require("../../codex/Sequence");
+
+const { DB_WRITE } = require("../../services/database/consts");
+const { Context } = require("../../util/context");
+const { buffer_to_stream } = require("../../util/streamutil");
+const { TYPE_SYMLINK, TYPE_DIRECTORY } = require("../FSNodeContext");
+const { LLFilesystemOperation } = require("./definitions");
+
+class LLRead extends LLFilesystemOperation {
+    static METHODS = {
+        _run: new Sequence({
+            async before_each (a, step) {
+                const operation = a.iget();
+                operation.checkpoint('step:' + step.name);
+            }
+        }, [
+            async function check_that_node_exists (a) {
+                if ( ! await a.get('fsNode').exists() ) {
+                    throw APIError.create('subject_does_not_exist');
+                }
+            },
+            async function type_check_for_read (a) {
+                const fsNode = a.get('fsNode');
+                if ( await fsNode.get('type') === TYPE_DIRECTORY ) {
+                    throw APIError.create('cannot_read_a_directory');
+                }
+            },
+            async function resolve_symlink (a) {
+                let fsNode = a.get('fsNode');
+                let type = await fsNode.get('type');
+                while ( type === TYPE_SYMLINK ) {
+                    fsNode = await fsNode.getTarget();
+                    type = await fsNode.get('type');
+                }
+                a.set('fsNode', fsNode);
+            },
+            async function check_ACL_for_read (a) {
+                if ( a.get('no_acl') ) return;
+                const context = a.iget('context');
+                const svc_acl = context.get('services').get('acl');
+                const { fsNode, actor } = a.values();
+                if ( ! await svc_acl.check(actor, fsNode, 'read') ) {
+                    throw await svc_acl.get_safe_acl_error(actor, fsNode, 'read');
+                }
+            },
+            async function calculate_has_range (a) {
+                const { offset, length } = a.values();
+                const fsNode = a.get('fsNode');
+                const has_range = (
+                    offset !== undefined &&
+                    offset !== 0
+                ) || (
+                    length !== undefined &&
+                    length != await fsNode.get('size')
+                );
+                a.set('has_range', has_range);
+            },
+            async function update_accessed (a) {
+                const context = a.iget('context');
+                const db = context.get('services')
+                    .get('database').get(DB_WRITE, 'filesystem');
+
+                const fsNode = a.get('fsNode');
+
+                await db.write(
+                    'UPDATE `fsentries` SET `accessed` = ? WHERE `id` = ?',
+                    [Date.now()/1000, fsNode.mysql_id]
+                );
+            },
+            async function check_for_cached_copy (a) {
+                const context = a.iget('context');
+                const svc_fileCache = context.get('services').get('file-cache');
+
+                const { fsNode } = a.values();
+
+                const maybe_buffer = await svc_fileCache.try_get(fsNode, a.log);
+                if ( maybe_buffer ) {
+                    a.log.info('cache hit');
+                    const { has_range } = a.values();
+                    if ( has_range ) {
+                        return a.stop(
+                            buffer_to_stream(maybe_buffer.slice(offset, offset+length))
+                        );
+                    }
+                    return a.stop(
+                        buffer_to_stream(maybe_buffer)
+                    );
+                }
+
+                a.log.info('cache miss');
+            },
+            async function create_S3_read_stream (a) {
+                const context = a.iget('context');
+                const storage = context.get('storage');
+
+                const { fsNode, version_id, offset, length, has_range } = a.values();
+
+                const location = await fsNode.get('s3:location');
+
+                const stream = (await storage.create_read_stream(await fsNode.get('uid'), {
+                    // TODO: fs:decouple-s3
+                    bucket: location.bucket,
+                    bucket_region: location.bucket_region,
+                    version_id,
+                    key: location.key,
+                    ...(has_range ? {
+                        range: `bytes=${offset}-${offset+length-1}`
+                    } : {}),
+                }));
+
+                a.set('stream', stream);
+            },
+            async function store_in_cache (a) {
+                const context = a.iget('context');
+                const svc_fileCache = context.get('services').get('file-cache');
+
+                const { fsNode, stream, has_range } = a.values();
+
+                if ( ! has_range ) {
+                    const res = await svc_fileCache.maybe_store(fsNode, stream);
+                    if ( res.stream ) a.set('stream', res.stream);
+                }
+            },
+            async function return_stream (a) {
+                return a.get('stream');
+            },
+        ]),
+    };
+}
+
+module.exports = {
+    LLRead
+};

packages/backend/src/filesystem/ll_operations/ll_readdir.js

@@ -0,0 +1,83 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const APIError = require("../../api/APIError");
+const { chkperm, get_descendants } = require("../../helpers");
+const { TYPE_DIRECTORY } = require("../FSNodeContext");
+const { NodeUIDSelector, NodeRawEntrySelector } = require("../node/selectors");
+const { LLFilesystemOperation } = require("./definitions");
+
+class LLReadDir extends LLFilesystemOperation {
+    async _run () {
+        const { context } = this;
+        const { subject, user, actor } = this.values;
+
+        if ( ! await subject.exists() ) {
+            throw APIError.create('subject_does_not_exist');
+        }
+
+        const svc_acl = context.get('services').get('acl');
+        if ( ! await svc_acl.check(actor, subject, 'list') ) {
+            throw await svc_acl.get_safe_acl_error(actor, subject, 'list');
+        }
+
+        const subject_uuid = await subject.get('uid');
+
+        const svc = context.get('services');
+        const svc_fsentry = svc.get('fsEntryService');
+        const svc_fs = svc.get('filesystem');
+
+        if (
+            subject.isRoot ||
+            (await subject.isUserDirectory() && subject.name !== user.username)
+        ) {
+            this.checkpoint('before call to get_descendants')
+            const entries = await get_descendants(
+                await subject.get('path'),
+                user,
+                1, true,
+            )
+            this.checkpoint('after call to get_descendants')
+
+            const children = await Promise.all(entries.map(async entry => {
+                const node = await svc_fs.node(new NodeRawEntrySelector(entry));
+                node.found_thumbnail = false;
+                return node;
+            }))
+
+            this.checkpoint('after get children (2)');
+
+            return children;
+        }
+
+        this.checkpoint('before get direct descendants')
+        const child_uuids = await svc_fsentry
+            .fast_get_direct_descendants(subject_uuid);
+        this.checkpoint('after get direct descendants')
+        const children = await Promise.all(child_uuids.map(async uuid => {
+            return await svc_fs.node(new NodeUIDSelector(uuid));
+        }));
+        this.checkpoint('after get children');
+
+        return children;
+    }
+}
+
+module.exports = {
+    LLReadDir,
+};

packages/backend/src/filesystem/ll_operations/ll_rmdir.js

@@ -0,0 +1,117 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const APIError = require("../../api/APIError");
+const { ParallelTasks } = require("../../util/otelutil");
+const FSNodeContext = require("../FSNodeContext");
+const { NodeUIDSelector } = require("../node/selectors");
+const { LLFilesystemOperation } = require("./definitions");
+const { LLRmNode } = require('./ll_rmnode');
+
+class LLRmDir extends LLFilesystemOperation {
+    async _run () {
+        const {
+            target,
+            user,
+            actor,
+            descendants_only,
+            recursive,
+
+            max_tasks = 8,
+        } = this.values;
+
+        const { context } = this;
+
+        const svc = context.get('services');
+
+        // Access Control
+        {
+            const svc_acl = context.get('services').get('acl');
+            this.checkpoint('remove :: access control');
+
+            // Check write access to target
+            if ( ! await svc_acl.check(actor, target, 'write') ) {
+                throw await svc_acl.get_safe_acl_error(actor, target, 'write');
+            }
+        }
+
+        if ( await target.get('immutable') && ! descendants_only ) {
+            throw APIError.create('immutable');
+        }
+
+        const svc_fsEntry = svc.get('fsEntryService');
+        const fs = svc.get('filesystem');
+
+        const children = await svc_fsEntry.fast_get_direct_descendants(
+            await target.get('uid')
+        );
+
+        if ( children.length > 0 && ! recursive ) {
+            throw APIError.create('not_empty');
+        }
+
+        const tracer = svc.get('traceService').tracer;
+        const tasks = new ParallelTasks({ tracer, max: max_tasks });
+
+        for ( const child_uuid of children ) {
+            tasks.add(`fs:rm:rm-child`, async () => {
+                const child_node = await fs.node(
+                    new NodeUIDSelector(child_uuid)
+                );
+                const type = await child_node.get('type');
+                if ( type === FSNodeContext.TYPE_DIRECTORY ) {
+                    const ll_rm = new LLRmDir();
+                    await ll_rm.run({
+                        target: await fs.node(
+                            new NodeUIDSelector(child_uuid),
+                        ),
+                        user,
+                        recursive: true,
+                        descendants_only: false,
+
+                        max_tasks: (v => v > 1 ? v : 1)(Math.floor(max_tasks / 2)),
+                    });
+                } else {
+                    const ll_rm = new LLRmNode();
+                    await ll_rm.run({
+                        target: await fs.node(
+                            new NodeUIDSelector(child_uuid),
+                        ),
+                        user,
+                    });
+                }
+            });
+        }
+
+        if ( ! descendants_only ) {
+            tasks.add(`fs:rm:rm-self`, async () => {
+                const ll_rm = new LLRmNode();
+                await ll_rm.run({
+                    target,
+                    user,
+                });
+            });
+        }
+
+        await tasks.awaitAll();
+    }
+}
+
+module.exports = {
+    LLRmDir,
+};

packages/backend/src/filesystem/ll_operations/ll_rmnode.js

@@ -0,0 +1,78 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { Context } = require("../../util/context");
+const { ParallelTasks } = require("../../util/otelutil");
+const { LLFilesystemOperation } = require("./definitions");
+
+class LLRmNode extends LLFilesystemOperation {
+    async _run () {
+        const { target, actor } = this.values;
+
+        const { context } = this;
+
+        const svc = context.get('services');
+
+        // Access Control
+        {
+            const svc_acl = context.get('services').get('acl');
+            this.checkpoint('remove :: access control');
+
+            // Check write access to target
+            if ( ! await svc_acl.check(actor, target, 'write') ) {
+                throw await svc_acl.get_safe_acl_error(actor, target, 'write');
+            }
+        }
+
+        if ( await target.get('immutable') ) {
+            throw new APIError(403, 'File is immutable.');
+        }
+
+        const svc_size = svc.get('sizeService');
+        const svc_fsEntry = svc.get('fsEntryService');
+
+        svc_size.change_usage(
+            await target.get('user_id'),
+            -1 * await target.get('size')
+        );
+
+        const tracer = svc.get('traceService').tracer;
+        const tasks = new ParallelTasks({ tracer, max: 4 });
+
+        tasks.add(`remove-fsentry`, async () => {
+            await svc_fsEntry.delete(await target.get('uid'));
+        });
+
+        if ( await target.get('has-s3') ) {
+            tasks.add(`remove-from-s3`, async () => {
+                // const storage = new PuterS3StorageStrategy({ services: svc });
+                const storage = Context.get('storage');
+                const state_delete = storage.create_delete();
+                await state_delete.run({
+                    node: target,
+                });
+            });
+        }
+
+        await tasks.awaitAll();
+    }
+}
+
+module.exports = {
+    LLRmNode,
+};

packages/backend/src/filesystem/ll_operations/ll_write.js

@@ -0,0 +1,369 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { Context } = require("../../util/context");
+const { LLFilesystemOperation } = require("./definitions");
+const { RESOURCE_STATUS_PENDING_CREATE } = require("../storage/ResourceService");
+const { NodeUIDSelector } = require("../node/selectors");
+const { UploadProgressTracker } = require("../storage/UploadProgressTracker");
+const FSNodeContext = require("../FSNodeContext");
+const APIError = require("../../api/APIError");
+const { progress_stream, stuck_detector_stream } = require("../../util/streamutil");
+const { OperationFrame } = require("../../services/OperationTraceService");
+const { Actor } = require("../../services/auth/Actor");
+const { DB_WRITE } = require("../../services/database/consts");
+
+const STUCK_STATUS_TIMEOUT = 10 * 1000;
+const STUCK_ALARM_TIMEOUT = 20 * 1000;
+
+class LLWriteBase extends LLFilesystemOperation {
+    static MODULES = {
+        config: require('../../config.js'),
+        simple_retry: require('../../util/retryutil.js').simple_retry,
+    }
+
+    async _storage_upload ({
+        uuid,
+        bucket, bucket_region, file,
+        tmp,
+    }) {
+        const { config } = this.modules;
+
+        const svc = Context.get('services');
+        const log = svc.get('log-service').create('fs._storage_upload');
+        const errors = svc.get('error-service').create(log);
+        const svc_event = svc.get('event');
+
+        const storage = Context.get('storage');
+
+        bucket        ??= config.s3_bucket;
+        bucket_region ??= config.s3_region;
+
+        let upload_tracker = new UploadProgressTracker();
+
+        svc_event.emit('fs.storage.upload-progress', {
+            upload_tracker,
+            context: Context.get(),
+            meta: {
+                item_uid: uuid,
+                item_path: tmp.path,
+            }
+        })
+
+        if ( ! file.buffer ) {
+            let stream = file.stream;
+            let alarm_timeout = null;
+            stream = stuck_detector_stream(stream, {
+                timeout: STUCK_STATUS_TIMEOUT,
+                on_stuck: () => {
+                    this.frame.status = OperationFrame.FRAME_STATUS_STUCK;
+                    log.warn('Upload stream stuck might be stuck', {
+                        bucket_region,
+                        bucket,
+                        uuid,
+                    });
+                    alarm_timeout = setTimeout(() => {
+                        errors.report('fs.write.s3-upload', {
+                            message: 'Upload stream stuck for too long',
+                            alarm: true,
+                            extra: {
+                                bucket_region,
+                                bucket,
+                                uuid,
+                            },
+                        });
+                    }, STUCK_ALARM_TIMEOUT);
+                },
+                on_unstuck: () => {
+                    clearTimeout(alarm_timeout);
+                    this.frame.status = OperationFrame.FRAME_STATUS_WORKING;
+                }
+            });
+            file = { ...file, stream, };
+        }
+
+        const state_upload = storage.create_upload();
+
+        try {
+            await state_upload.run({
+                uid: uuid,
+                file,
+                storage_meta: { bucket, bucket_region },
+                storage_api: { progress_tracker: upload_tracker },
+            });
+        } catch (e) {
+            errors.report('fs.write.storage-upload', {
+                source: e || new Error('unknown'),
+                trace: true,
+                alarm: true,
+                extra: {
+                    bucket_region,
+                    bucket,
+                    uuid,
+                },
+            });
+            throw APIError.create('upload_failed');
+        }
+
+        return state_upload;
+    }
+}
+
+class LLOWrite extends LLWriteBase {
+    async _run () {
+        const {
+            node, user, immutable,
+            file, tmp, fsentry_tmp,
+            message,
+        } = this.values;
+
+        let { actor } = this.values;
+
+        const svc = Context.get('services');
+        const sizeService = svc.get('sizeService');
+        const resourceService = svc.get('resourceService');
+        const systemFSEntryService = svc.get('systemFSEntryService');
+        const svc_event = svc.get('event');
+
+        // TODO: fs:decouple-versions
+        //       add version hook externally so LLCWrite doesn't
+        //       need direct database access
+        const db = svc.get('database').get(DB_WRITE, 'filesystem');
+
+        // TODO: Add symlink write
+        if ( ! await node.exists() ) {
+            // TODO: different class of errors for low-level operations
+            throw APIError.create('subject_does_not_exist');
+        }
+
+        actor = actor ?? Actor.adapt(user);
+
+        const svc_acl = this.context.get('services').get('acl');
+        if ( ! await svc_acl.check(actor, node, 'write') ) {
+            throw await svc_acl.get_safe_acl_error(actor, node, 'write');
+        }
+
+        const uid = await node.get('uid');
+
+        const bucket_region = node.entry.bucket_region;
+        const bucket = node.entry.bucket;
+
+        const state_upload = await this._storage_upload({
+            uuid: node.entry.uuid,
+            bucket, bucket_region, file,
+            tmp: {
+                ...tmp,
+                path: await node.get('path'),
+            }
+        });
+
+        fsentry_tmp.thumbnail = await fsentry_tmp.thumbnail_promise;
+        delete fsentry_tmp.thumbnail_promise;
+
+        const ts = Math.round(Date.now() / 1000);
+        const raw_fsentry_delta = {
+            modified: ts,
+            accessed: ts,
+            size: file.size,
+            ...fsentry_tmp,
+        };
+
+        resourceService.register({
+            uid,
+            status: RESOURCE_STATUS_PENDING_CREATE,
+        });
+
+        const filesize = file.size;
+        sizeService.change_usage(user.id, filesize);
+
+        const entryOp = await systemFSEntryService.update(uid, raw_fsentry_delta);
+
+        // depends on fsentry, does not depend on S3
+        (async () => {
+            await entryOp.awaitDone();
+            this.log.debug('[owrite] finished creating fsentry', { uid })
+            resourceService.free(uid);
+        })();
+
+        state_upload.post_insert({
+            db, user, node, uid, message, ts,
+        });
+
+        const svc_fileCache = this.context.get('services').get('file-cache');
+        await svc_fileCache.invalidate(node);
+
+        svc_event.emit('fs.write.file', {
+            node,
+            context: this.context,
+        });
+
+        return node;
+    }
+}
+
+class LLCWrite extends LLWriteBase {
+    static MODULES = {
+        _path: require('path'),
+        uuidv4: require('uuid').v4,
+        config: require('../../config.js'),
+    }
+
+    async _run () {
+        const { _path, uuidv4, config } = this.modules;
+        const {
+            parent, name, user, immutable,
+            file, tmp, fsentry_tmp,
+            message,
+
+            actor,
+            app_id,
+        } = this.values;
+
+        const svc = Context.get('services');
+        const sizeService = svc.get('sizeService');
+        const resourceService = svc.get('resourceService');
+        const systemFSEntryService = svc.get('systemFSEntryService');
+        const svc_event = svc.get('event');
+        const fs = svc.get('filesystem');
+
+        // TODO: fs:decouple-versions
+        //       add version hook externally so LLCWrite doesn't
+        //       need direct database access
+        const db = svc.get('database').get(DB_WRITE, 'filesystem');
+
+        const uid = uuidv4();
+        this.field('fsentry-uid', uid);
+
+        // determine bucket region
+        let bucket_region = config.s3_region;
+        let bucket = config.s3_bucket;
+
+        this.checkpoint('before acl');
+        if ( ! await parent.exists() ) {
+            throw APIError.create('subject_does_not_exist');
+        }
+
+        const svc_acl = this.context.get('services').get('acl');
+        if ( ! await svc_acl.check(actor, parent, 'write') ) {
+            throw await svc_acl.get_safe_acl_error(actor, parent, 'write');
+        }
+
+        this.checkpoint('before storage upload');
+
+        const storage_resp = await this._storage_upload({
+            uuid: uid,
+            bucket, bucket_region, file,
+            tmp: {
+                ...tmp,
+                path: _path.join(await parent.get('path'), name),
+            }
+        });
+
+        this.checkpoint('after storage upload');
+
+        fsentry_tmp.thumbnail = await fsentry_tmp.thumbnail_promise;
+        delete fsentry_tmp.thumbnail_promise;
+
+        this.checkpoint('after thumbnail promise');
+
+        const ts = Math.round(Date.now() / 1000);
+        const raw_fsentry = {
+            uuid: uid,
+            is_dir: 0,
+            user_id: user.id,
+            created: ts,
+            accessed: ts,
+            modified: ts,
+            parent_uid: await parent.get('uid'),
+            name,
+            size: file.size,
+            path: _path.join(await parent.get('path'), name),
+            ...fsentry_tmp,
+
+            bucket_region,
+            bucket,
+
+            associated_app_id: app_id ?? null,
+        };
+
+        svc_event.emit('fs.pending.file', {
+            fsentry: FSNodeContext.sanitize_pending_entry_info(raw_fsentry),
+            context: this.context,
+        })
+
+        this.checkpoint('after emit pending file');
+
+        resourceService.register({
+            uid,
+            status: RESOURCE_STATUS_PENDING_CREATE,
+        });
+
+        const filesize = file.size;
+        sizeService.change_usage(user.id, filesize);
+
+        this.checkpoint('after change_usage');
+
+        const entryOp = await systemFSEntryService.insert(raw_fsentry);
+
+        this.checkpoint('after fsentry insert enqueue');
+
+        (async () => {
+            await entryOp.awaitDone();
+            this.log.debug('finished creating fsentry', { uid })
+            resourceService.free(uid);
+
+            const new_item_node = await fs.node(new NodeUIDSelector(uid));
+            const new_item = await new_item_node.get('entry');
+            const store_version_id = storage_resp.VersionId;
+            if( store_version_id ){
+                // insert version into db
+                db.write(
+                    "INSERT INTO `fsentry_versions` (`user_id`, `fsentry_id`, `fsentry_uuid`, `version_id`, `message`, `ts_epoch`) VALUES (?, ?, ?, ?, ?, ?)",
+                    [
+                        user.id,
+                        new_item.id,
+                        new_item.uuid,
+                        store_version_id,
+                        message ?? null,
+                        ts,
+                    ]
+                );
+        }
+        })();
+
+        this.checkpoint('after version IIAFE');
+
+        const node = await fs.node(new NodeUIDSelector(uid));
+
+        this.checkpoint('after create FSNodeContext');
+
+        svc_event.emit('fs.create.file', {
+            node,
+            context: this.context,
+        });
+
+        this.checkpoint('return result');
+
+        return node;
+    }
+}
+
+module.exports = {
+    LLCWrite,
+    LLOWrite,
+};

packages/backend/src/filesystem/node/selectors.js

@@ -0,0 +1,172 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const _path = require('path');
+const { PuterPath } = require('../lib/PuterPath');
+
+class NodePathSelector {
+    constructor (path) {
+        this.value = path;
+    }
+
+    setPropertiesKnownBySelector (node) {
+        node.path = this.value;
+        node.name = _path.basename(this.value);
+    }
+
+    describe () {
+        return this.value;
+    }
+}
+
+class NodeUIDSelector {
+    constructor (uid) {
+        this.value = uid;
+    }
+
+    setPropertiesKnownBySelector (node) {
+        node.uid = this.value;
+    }
+
+    // Note: the selector could've been added by FSNodeContext
+    // during fetch, but this was more efficient because the
+    // object is created lazily, and it's somtimes not needed.
+    static implyFromFetchedData (node) {
+        if ( node.uid ) {
+            return new NodeUIDSelector(node.uid);
+        }
+        return null;
+    }
+
+    describe () {
+        return `[uid:${this.value}]`;
+    }
+}
+
+class NodeInternalIDSelector {
+    constructor (service, id, debugInfo) {
+        this.service = service;
+        this.id = id;
+        this.debugInfo = debugInfo;
+    }
+
+    setPropertiesKnownBySelector (node) {
+        if ( this.service === 'mysql' ) {
+            node.mysql_id = this.id;
+        }
+    }
+
+    describe (showDebug) {
+        if ( showDebug ) {
+            return `[db:${this.id}] (${
+                JSON.stringify(this.debugInfo, null, 2)
+            })`
+        }
+        return `[db:${this.id}]`
+    }
+}
+
+class NodeChildSelector {
+    constructor (parent, name) {
+        this.parent = parent;
+        this.name = name;
+    }
+
+    setPropertiesKnownBySelector (node) {
+        node.name = this.name;
+        // no properties known
+    }
+
+    describe () {
+        return this.parent.describe() + '/' + this.name;
+    }
+}
+
+class RootNodeSelector {
+    static entry = {
+        is_dir: true,
+        is_root: true,
+        uuid: PuterPath.NULL_UUID,
+        name: '/',
+    };
+    setPropertiesKnownBySelector (node) {
+        node.path = '/';
+        node.root = true;
+        node.uid = PuterPath.NULL_UUID;
+    }
+    constructor () {
+        this.entry = this.constructor.entry;
+    }
+
+    describe () {
+        return '[root]';
+    }
+}
+
+class NodeRawEntrySelector {
+    constructor (entry) {
+        // Fix entries from get_descendants
+        if ( ! entry.uuid && entry.uid ) {
+            entry.uuid = entry.uid;
+            if ( entry._id ) {
+                entry.id = entry._id;
+                delete entry._id;
+            }
+        }
+
+        this.entry = entry;
+    }
+
+    setPropertiesKnownBySelector (node) {
+        node.found = true;
+        node.entry = this.entry;
+        node.uid = this.entry.uid ?? this.entry.uuid;
+        node.name = this.entry.name;
+        if ( this.entry.path ) node.path = this.entry.path;
+    }
+
+    describe () {
+        return '[raw entry]';
+    }
+}
+
+const relativeSelector = (parent, path) => {
+    if ( path === '.' ) return parent;
+    if ( path.startsWith('..') ) {
+        throw new Error('currently unsupported');
+    }
+
+    let selector = parent;
+
+    const parts = path.split('/').filter(Boolean);
+    for ( const part of parts ) {
+        selector = new NodeChildSelector(selector, part);
+    }
+
+    return selector;
+}
+
+module.exports = {
+    NodePathSelector,
+    NodeUIDSelector,
+    NodeInternalIDSelector,
+    NodeChildSelector,
+    RootNodeSelector,
+    NodeRawEntrySelector,
+    relativeSelector,
+};

packages/backend/src/filesystem/node/states.js

@@ -0,0 +1,23 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+class NodeFoundState {}
+
+class NodeDoesNotExistState {}
+
+class NodeInitialState {}

packages/backend/src/filesystem/storage/DatabaseFSEntryFetcher.js

@@ -0,0 +1,225 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { DB_READ } = require("../../services/database/consts");
+const { abtest } = require("../../util/otelutil");
+const { NodePathSelector, NodeUIDSelector, NodeInternalIDSelector, NodeChildSelector, RootNodeSelector } = require("../node/selectors");
+
+module.exports = class DatabaseFSEntryFetcher {
+    constructor ({ services }) {
+        this.services = services;
+        this.log = services.get('log-service').create('DatabaseFSEntryFetcher');
+
+        this.db = services.get('database').get(DB_READ, 'filesystem');
+
+        this.defaultProperties = [
+            'id',
+            'associated_app_id',
+            'uuid',
+            'public_token',
+            'bucket',
+            'bucket_region',
+            'file_request_token',
+            'user_id',
+            'parent_uid',
+            'is_dir',
+            'is_public',
+            'is_shortcut',
+            'is_symlink',
+            'symlink_path',
+            'shortcut_to',
+            'sort_by',
+            'sort_order',
+            'immutable',
+            'name',
+            'metadata',
+            'modified',
+            'created',
+            'accessed',
+            'size',
+            'layout',
+            'path',
+        ]
+    }
+
+    async find (selector, fetch_entry_options) {
+        if ( selector instanceof RootNodeSelector ) {
+            return selector.entry;
+        }
+        if ( selector instanceof NodePathSelector ) {
+            return await this.findByPath(
+                selector.value, fetch_entry_options);
+        }
+        if ( selector instanceof NodeUIDSelector ) {
+            return await this.findByUID(
+                selector.value, fetch_entry_options);
+        }
+        if ( selector instanceof NodeInternalIDSelector ) {
+            return await this.findByID(
+                selector.id, fetch_entry_options);
+        }
+        if ( selector instanceof NodeChildSelector ) {
+            let id;
+
+            if ( selector.parent instanceof RootNodeSelector ) {
+                id = await this.findNameInRoot(selector.name);
+            } else {
+                const parentEntry = await this.find(selector.parent);
+                if ( ! parentEntry ) return null;
+                id = await this.findNameInParent(
+                    parentEntry.uuid, selector.name
+                );
+            }
+
+            if ( id === undefined ) return null;
+            if ( typeof id !== 'number' ) {
+                throw new Error(
+                    'unexpected type for id value',
+                    typeof id,
+                    id
+                );
+            }
+            return this.find(new NodeInternalIDSelector('mysql', id));
+        }
+    }
+
+    async findByUID(uuid, fetch_entry_options = {}) {
+        const { thumbnail } = fetch_entry_options;
+
+        let fsentry = await this.db.requireRead(
+            `SELECT ` +
+                this.defaultProperties.join(', ') +
+                (thumbnail ? `, thumbnail` : '') +
+                ` FROM fsentries WHERE uuid = ? LIMIT 1`,
+            [uuid]
+        );
+
+        return fsentry[0];
+    }
+
+    async findByID(id, fetch_entry_options = {}) {
+        const { thumbnail } = fetch_entry_options;
+
+        let fsentry = await this.db.requireRead(
+            `SELECT ` +
+                this.defaultProperties.join(', ') +
+                (thumbnail ? `, thumbnail` : '') +
+                ` FROM fsentries WHERE id = ? LIMIT 1`,
+            [id]
+        );
+
+        return fsentry[0];
+    }
+
+    async findByPath(path, fetch_entry_options = {}) {
+        const { thumbnail } = fetch_entry_options;
+
+        if ( path === '/' ) {
+            return this.find(new RootNodeSelector());
+        }
+
+        const parts = path.split('/').filter(path => path !== '');
+        if ( parts.length === 0 ) {
+            // TODO: invalid path; this should be an error
+            return false;
+        }
+
+
+        // TODO: use a closure table for more efficient path resolving
+        let parent_uid = null;
+        let result;
+
+        const resultColsSql = this.defaultProperties.join(', ') +
+            (thumbnail ? `, thumbnail` : '');
+
+        result = await this.db.read(
+            `SELECT ` + resultColsSql +
+            ` FROM fsentries WHERE path=? LIMIT 1`,
+            [path]
+        );
+
+        // using knex instead
+
+        if ( result[0] ) return result[0];
+
+        this.log.info(`findByPath (not cached): ${path}`)
+
+        const loop = async () => {
+            for ( let i=0 ; i < parts.length ; i++ ) {
+                const part = parts[i];
+                const isLast = i == parts.length - 1;
+                const colsSql = isLast ? resultColsSql : 'uuid';
+                if ( parent_uid === null ) {
+                    result = await this.db.read(
+                        `SELECT ` + colsSql +
+                            ` FROM fsentries WHERE parent_uid IS NULL AND name=? LIMIT 1`,
+                        [part]
+                    );
+                } else {
+                    result = await this.db.read(
+                        `SELECT ` + colsSql +
+                            ` FROM fsentries WHERE parent_uid=? AND name=? LIMIT 1`,
+                        [parent_uid, part]
+                    );
+                }
+
+                if ( ! result[0] ) return false;
+                parent_uid = result[0].uuid;
+            }
+        }
+
+        if ( fetch_entry_options.tracer ) {
+            const tracer = fetch_entry_options.tracer;
+            const options = fetch_entry_options.trace_options;
+            await tracer.startActiveSpan(`fs:sql:findByPath`,
+                ...(options ? [options] : []),
+                async span => {
+                    await loop();
+                    span.end();
+                });
+        } else {
+            await loop();
+        }
+
+        return result[0];
+    }
+
+    async findNameInRoot (name) {
+        let child_id = await this.db.read(
+            "SELECT `id` FROM `fsentries` WHERE `parent_uid` IS NULL AND name = ? LIMIT 1",
+            [name]
+        );
+        return child_id[0]?.id;
+    }
+
+    async findNameInParent (parent_uid, name) {
+        let child_id = await this.db.read(
+            "SELECT `id` FROM `fsentries` WHERE `parent_uid` = ? AND name = ? LIMIT 1",
+            [parent_uid, name]
+        );
+        return child_id[0]?.id;
+    }
+
+    async nameExistsUnderParent (parent_uid, name) {
+        let check_dupe = await this.db.read(
+            "SELECT `id` FROM `fsentries` WHERE `parent_uid` = ? AND name = ? LIMIT 1",
+            [parent_uid, name]
+        );
+        return !! check_dupe[0];
+    }
+}

packages/backend/src/filesystem/storage/DatabaseFSEntryService.js

@@ -0,0 +1,545 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { AdvancedBase } = require("puter-js-common");
+const { id2path } = require("../../helpers");
+
+const { PuterPath } = require("../lib/PuterPath");
+const { NodeUIDSelector } = require("../node/selectors");
+const { OtelTrait } = require("../../traits/OtelTrait");
+const { Context } = require("../../util/context");
+const { DB_WRITE } = require("../../services/database/consts");
+
+class AbstractDatabaseFSEntryOperation {
+    static STATUS_PENDING = {};
+    static STATUS_RUNNING = {};
+    static STATUS_DONE = {};
+    constructor () {
+        this.status_ = this.constructor.STATUS_PENDING;
+        this.donePromise = new Promise((resolve, reject) => {
+            this.doneResolve = resolve;
+            this.doneReject = reject;
+        });
+    }
+    get status () {
+        return this.status_;
+    }
+    set status (status) {
+        this.status_ = status;
+        if ( status === this.constructor.STATUS_DONE ) {
+            this.doneResolve();
+        }
+    }
+    awaitDone () {
+        return this.donePromise;
+    }
+    onComplete(fn) {
+        this.donePromise.then(fn);
+    }
+}
+
+class DatabaseFSEntryInsert extends AbstractDatabaseFSEntryOperation {
+    static requiredForCreate = [
+        'uuid',
+        'parent_uid',
+    ];
+
+    static allowedForCreate = [
+        ...this.requiredForCreate,
+        'name',
+        'user_id',
+        'is_dir',
+        'created',
+        'modified',
+        'immutable',
+        'shortcut_to',
+        'is_shortcut',
+        'metadata',
+        'bucket',
+        'bucket_region',
+        'thumbnail',
+        'accessed',
+        'size',
+        'symlink_path',
+        'is_symlink',
+        'associated_app_id',
+        'path',
+    ];
+
+    constructor (entry) {
+        super();
+        const requiredForCreate = this.constructor.requiredForCreate;
+        const allowedForCreate = this.constructor.allowedForCreate;
+
+        {
+            const sanitized_entry = {};
+            for ( const k of allowedForCreate ) {
+                if ( entry.hasOwnProperty(k) ) {
+                    sanitized_entry[k] = entry[k];
+                }
+            }
+            entry = sanitized_entry;
+        }
+
+        for ( const k of requiredForCreate ) {
+            if ( ! entry.hasOwnProperty(k) ) {
+                throw new Error(`Missing required property: ${k}`);
+            }
+        }
+
+        this.entry = entry;
+    }
+
+    getStatement () {
+        const fields = Object.keys(this.entry);
+        const statement = `INSERT INTO fsentries ` +
+            `(${fields.join(', ')}) ` +
+            `VALUES (${fields.map(() => '?').join(', ')})`;
+        const values = fields.map(k => this.entry[k]);
+        return { statement, values };
+    }
+
+    apply (answer) {
+        answer.entry = { ...this.entry };
+    }
+
+    get uuid () {
+        return this.entry.uuid;
+    }
+}
+
+class DatabaseFSEntryUpdate extends AbstractDatabaseFSEntryOperation {
+    static allowedForUpdate = [
+        'name',
+        'parent_uid',
+        'user_id',
+        'modified',
+        'shortcut_to',
+        'metadata',
+        'thumbnail',
+        'size',
+        'path',
+    ];
+
+    constructor (uuid, entry) {
+        super();
+        const allowedForUpdate = this.constructor.allowedForUpdate;
+
+        {
+            const sanitized_entry = {};
+            for ( const k of allowedForUpdate ) {
+                if ( entry.hasOwnProperty(k) ) {
+                    sanitized_entry[k] = entry[k];
+                }
+            }
+            entry = sanitized_entry;
+        }
+
+        this.uuid = uuid;
+        this.entry = entry;
+    }
+
+    getStatement () {
+        const fields = Object.keys(this.entry);
+        const statement = `UPDATE fsentries SET ` +
+            `${fields.map(k => `${k} = ?`).join(', ')} ` +
+            `WHERE uuid = ? LIMIT 1`;
+        const values = fields.map(k => this.entry[k]);
+        values.push(this.uuid);
+        return { statement, values };
+    }
+
+    apply (answer) {
+        if ( ! answer.entry ) {
+            answer.is_diff = true;
+            answer.entry = {};
+        }
+        Object.assign(answer.entry, this.entry);
+    }
+}
+
+class DatabaseFSEntryDelete extends AbstractDatabaseFSEntryOperation {
+    constructor (uuid) {
+        super();
+        this.uuid = uuid;
+    }
+
+    getStatement () {
+        const statement = `DELETE FROM fsentries WHERE uuid = ? LIMIT 1`;
+        const values = [this.uuid];
+        return { statement, values };
+    }
+
+    apply (answer) {
+        answer.entry = null;
+    }
+}
+
+
+class DatabaseFSEntryService extends AdvancedBase {
+    static STATUS_READY = {};
+    static STATUS_RUNNING_JOB = {};
+
+    static TRAITS = [
+        new OtelTrait([
+            'insert',
+            'update',
+            'delete',
+            'fast_get_descendants',
+            'fast_get_direct_descendants',
+            'get',
+            'get_descendants',
+            'get_recursive_size',
+            'enqueue_',
+            'checkShouldExec_',
+            'exec_',
+        ]),
+    ]
+
+    constructor ({ services, label }) {
+        super();
+        this.db = services.get('database').get(DB_WRITE, 'filesystem');
+
+        this.log = services.get('log-service').create('fsentry-service');
+
+        this.label = label || 'DatabaseFSEntryService';
+
+        const params = services.get('params');
+        params.createParameters('fsentry-service', [
+            {
+                id: 'max_queue',
+                description: 'Maximum queue size',
+                default: 50,
+            },
+        ], this);
+
+        this.status = this.constructor.STATUS_READY;
+
+        this.currentState = {
+            queue: [],
+            updating_uuids: {},
+        };
+        this.deferredState = {
+            queue: [],
+            updating_uuids: {},
+        };
+
+        this.entryListeners_ = {};
+
+        this.mkPromiseForQueueSize_();
+
+        // Register information providers
+        const info = services.get('information');
+
+        // uuid -> path via mysql
+        info.given('fs.fsentry:uuid').provide('fs.fsentry:path')
+            .addStrategy('mysql', async uuid => {
+                // TODO: move id2path here
+                return await id2path(uuid);
+            });
+
+        (async () => {
+            await services.ready;
+            if ( services.has('commands') ) {
+                this._registerCommands(services.get('commands'));
+            }
+        })();
+    }
+
+    mkPromiseForQueueSize_ () {
+        this.queueSizePromise = new Promise((resolve, reject) => {
+            this.queueSizeResolve = resolve;
+        });
+    }
+
+    async insert (entry) {
+        const op = new DatabaseFSEntryInsert(entry);
+        await this.enqueue_(op);
+        return op;
+    }
+
+    async update (uuid, entry) {
+        const op = new DatabaseFSEntryUpdate(uuid, entry);
+        await this.enqueue_(op);
+        return op;
+    }
+
+    async delete (uuid) {
+        const op = new DatabaseFSEntryDelete(uuid);
+        await this.enqueue_(op);
+        return op;
+    }
+
+    async fast_get_descendants (uuid) {
+        return (await this.db.read(`
+            WITH RECURSIVE descendant_cte AS (
+                SELECT uuid, parent_uid
+                FROM fsentries
+                WHERE parent_uid = ?
+
+                UNION ALL
+
+                SELECT f.uuid, f.parent_uid
+                FROM fsentries f
+                INNER JOIN descendant_cte d ON f.parent_uid = d.uuid
+            )
+            SELECT uuid FROM descendant_cte
+        `, [uuid])).map(x => x.uuid);
+    }
+
+    async fast_get_direct_descendants (uuid) {
+        return (uuid === PuterPath.NULL_UUID
+            ? await this.db.read(
+                `SELECT uuid FROM fsentries WHERE parent_uid IS NULL`)
+            : await this.db.read(
+                `SELECT uuid FROM fsentries WHERE parent_uid = ?`,
+                [uuid])).map(x => x.uuid);
+    }
+
+    waitForEntry (node, callback) {
+        // *** uncomment to debug slow waits ***
+        // console.log('ATTEMPT TO WAIT FOR', selector.describe())
+        let selector = node.get_selector_of_type(NodeUIDSelector);
+        if ( selector === null ) {
+            this.log.debug('cannot wait for this selector');
+            // console.log(new Error('========'));
+            return;
+        }
+
+        const entry_already_enqueued =
+            this.currentState.updating_uuids.hasOwnProperty(selector.value) ||
+            this.deferredState.updating_uuids.hasOwnProperty(selector.value) ;
+
+        if ( entry_already_enqueued ) {
+            callback();
+            return;
+        }
+
+        const k = `uid:${selector.value}`;
+        if ( ! this.entryListeners_.hasOwnProperty(k) ) {
+            this.entryListeners_[k] = [];
+        }
+
+        const det = {
+            detach: () => {
+                const i = this.entryListeners_[k].indexOf(callback);
+                if ( i === -1 ) return;
+                this.entryListeners_[k].splice(i, 1);
+                if ( this.entryListeners_[k].length === 0 ) {
+                    delete this.entryListeners_[k];
+                }
+            }
+        };
+
+        this.entryListeners_[k].push(callback);
+
+        return det;
+    }
+
+    async get (uuid, fetch_entry_options) {
+        this.log.debug('--- finding ops for', { uuid })
+        const answer = {};
+        for ( const op of this.currentState.queue ) {
+            if ( op.uuid != uuid ) continue;
+            this.log.debug('=== found op!', { op });
+            op.apply(answer);
+        }
+        for ( const op of this.deferredState.queue ) {
+            if ( op.uuid != uuid ) continue;
+            this.log.debug('=== found op**!', { op });
+            op.apply(answer);
+            op.apply(answer);
+        }
+        if ( answer.is_diff ) {
+            const fsEntryFetcher = Context.get('services').get('fsEntryFetcher');
+            const base_entry = await fsEntryFetcher.find(
+                new NodeUIDSelector(uuid),
+                fetch_entry_options,
+            );
+            answer.entry = { ...base_entry, ...answer.entry };
+        }
+        return answer.entry;
+    }
+
+    async get_descendants (uuid) {
+        return uuid === PuterPath.NULL_UUID
+            ? await this.db.read(
+                `SELECT uuid FROM fsentries WHERE parent_uid IS NULL`,
+                [uuid],
+            )
+            : await this.db.read(
+                `SELECT uuid FROM fsentries WHERE parent_uid = ?`,
+                [uuid],
+            )
+            ;
+    }
+
+    async get_recursive_size (uuid) {
+        const cte_query = `
+            WITH RECURSIVE descendant_cte AS (
+                SELECT uuid, parent_uid, size
+                FROM fsentries
+                WHERE parent_uid = ?
+
+                UNION ALL
+
+                SELECT f.uuid, f.parent_uid, f.size
+                FROM fsentries f
+                INNER JOIN descendant_cte d
+                ON f.parent_uid = d.uuid
+            )
+            SELECT SUM(size) AS total_size FROM descendant_cte
+        `;
+        const rows = await this.db.read(cte_query, [uuid]);
+        return rows[0].total_size;
+    }
+
+    async enqueue_ (op) {
+        while (
+            this.currentState.queue.length > this.max_queue ||
+            this.deferredState.queue.length > this.max_queue
+        ) {
+            await this.queueSizePromise;
+        }
+
+        if ( ! (op instanceof AbstractDatabaseFSEntryOperation) ) {
+            throw new Error('Invalid operation');
+        }
+
+        const state = this.status === this.constructor.STATUS_READY ?
+            this.currentState : this.deferredState;
+
+        if ( ! state.updating_uuids.hasOwnProperty(op.uuid) ) {
+            state.updating_uuids[op.uuid] = [];
+        }
+        state.updating_uuids[op.uuid].push(state.queue.length);
+
+        state.queue.push(op);
+
+        // DRY: same pattern as FSOperationContext:provideValue
+        // DRY: same pattern as FSOperationContext:rejectValue
+        if ( this.entryListeners_.hasOwnProperty(op.uuid) ) {
+            const listeners = this.entryListeners_[op.uuid];
+
+            delete this.entryListeners_[op.uuid];
+
+            for ( const lis of listeners ) lis();
+        }
+
+        this.checkShouldExec_();
+    }
+
+    checkShouldExec_ () {
+        if ( this.status !== this.constructor.STATUS_READY ) return;
+        if ( this.currentState.queue.length === 0 ) return;
+        this.exec_();
+    }
+
+    async exec_ () {
+        if ( this.status !== this.constructor.STATUS_READY ) {
+            throw new Error('Duplicate exec_ call');
+        }
+
+        const queue = this.currentState.queue;
+
+        this.log.info(
+            `\x1B[36;1m[${this.label}]\x1B[0m ` +
+            `Executing ${queue.length} operations...`
+        );
+
+        this.status = this.constructor.STATUS_RUNNING_JOB;
+
+        // const conn = await this.db_primary.promise().getConnection();
+        // await conn.beginTransaction();
+
+        for ( const op of queue ) {
+            op.status = op.constructor.STATUS_RUNNING;
+            // await conn.execute(stmt, values);
+        }
+
+        // await conn.commit();
+        // conn.release();
+
+        // const stmtAndVals = queue.map(op => op.getStatementAndValues());
+        // const stmts = stmtAndVals.map(x => x.stmt).join('; ');
+        // const vals = stmtAndVals.reduce((acc, x) => acc.concat(x.values), []);
+
+        // *** uncomment to debug batch queries ***
+        // this.log.debug({ stmts, vals });
+        // console.log('<<========================');
+        // console.log({ stmts, vals });
+        // console.log('>>========================');
+
+        // this.log.debug('array?', Array.isArray(vals))
+
+        await this.db.batch_write(queue.map(op => op.getStatement()));
+
+
+        for ( const op of queue ) {
+            op.status = op.constructor.STATUS_DONE;
+        }
+
+        this.flipState_();
+        this.status = this.constructor.STATUS_READY;
+
+        this.log.info(
+            `\x1B[36;1m[${this.label}]\x1B[0m ` +
+            `Finished ${queue.length} operations.`
+        )
+
+        for ( const op of queue ) {
+            op.status = op.constructor.STATUS_DONE;
+        }
+
+        this.checkShouldExec_();
+    }
+
+    flipState_ () {
+        this.currentState = this.deferredState;
+        this.deferredState = {
+            queue: [],
+            updating_uuids: {},
+        };
+        const queueSizeResolve = this.queueSizeResolve;
+        this.mkPromiseForQueueSize_();
+        queueSizeResolve();
+    }
+
+    _registerCommands (commands) {
+        commands.registerCommands('mysql-fsentry-service', [
+            {
+                id: 'get-queue-size-current',
+                description: 'Get the current queue size',
+                handler: async (args, log) => {
+                    log.log(this.currentState.queue.length);
+                }
+            },
+            {
+                id: 'get-queue-size-deferred',
+                description: 'Get the deferred queue size',
+                handler: async (args, log) => {
+                    log.log(this.deferredState.queue.length);
+                }
+            }
+        ])
+    }
+}
+
+module.exports = {
+    DatabaseFSEntryService
+};

packages/backend/src/filesystem/storage/ResourceService.js

@@ -0,0 +1,134 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+let waiti = 0;
+const {
+    NodePathSelector,
+    NodeUIDSelector,
+    NodeInternalIDSelector,
+    NodeChildSelector,
+} = require("../node/selectors");
+
+const RESOURCE_STATUS_PENDING_CREATE = {};
+const RESOURCE_STATUS_PENDING_UPDATE = {};
+const RS_DIRECTORY_PENDING_CHILD_INSERT = {};
+
+/**
+ * ResourceService is a very simple locking mechanism meant
+ * only to ensure consistency between requests being sent
+ * to the same server.
+ *
+ * For example, if you send an HTTP request to `/write`, and
+ * then a subsequent HTTP request to `/read`, you would expect
+ * the newly written file to be available. Therefore, the call
+ * to `/read` should wait until the write is complete.
+ *
+ * At least for now; I'm sure we'll think of a smarter way to
+ * handle this in the future.
+ */
+class ResourceService {
+    constructor ({ services }) {
+        this.uidToEntry = {};
+        this.uidToPath = {};
+        this.pathToEntry = {};
+
+        this.log = services.get('log-service').create('resource-service');
+    }
+
+    register (entry) {
+        entry = { ...entry };
+
+        if ( ! entry.uid ) {
+            // TODO: resource service needs logger access
+            return;
+        }
+
+        entry.freePromise = new Promise((resolve, reject) => {
+            entry.free = () => {
+                resolve();
+            };
+        });
+        entry.onFree = entry.freePromise.then.bind(entry.freePromise);
+        this.log.info(`registering resource`, { uid: entry.uid });
+        this.uidToEntry[entry.uid] = entry;
+        if ( entry.path ) {
+            this.uidToPath[entry.uid] = entry.path;
+            this.pathToEntry[entry.path] = entry;
+        }
+        return entry;
+    }
+
+    free (uid) {
+        this.log.info(`freeing`, uid);
+        const entry = this.uidToEntry[uid];
+        if ( ! entry ) return;
+        delete this.uidToEntry[uid];
+        if ( this.uidToPath.hasOwnProperty(uid) ) {
+            const path = this.uidToPath[uid];
+            delete this.pathToEntry[path];
+            delete this.uidToPath[uid];
+        }
+        entry.free();
+    }
+
+    async waitForResourceByPath (path) {
+        const entry = this.pathToEntry[path];
+        if (!entry) {
+            return;
+        }
+        await entry.freePromise;
+    }
+
+    async waitForResourceByUID (uid) {
+        const entry = this.uidToEntry[uid];
+        if (!entry) {
+            return;
+        }
+        await entry.freePromise;
+    }
+
+    async waitForResource (selector) {
+        const i = waiti++;
+        if ( selector instanceof NodePathSelector ) {
+            await this.waitForResourceByPath(selector.value);
+        }
+        else
+        if ( selector instanceof NodeUIDSelector ) {
+            await this.waitForResourceByUID(selector.value);
+        }
+        else
+        if ( selector instanceof NodeInternalIDSelector ) {
+            // Can't wait intelligently for this
+        }
+        if ( selector instanceof NodeChildSelector ) {
+            await this.waitForResource(selector.parent);
+        }
+    }
+
+    getResourceInfo (uid) {
+        if ( ! uid ) return;
+        return this.uidToEntry[uid];
+    }
+}
+
+module.exports = {
+    ResourceService,
+    RESOURCE_STATUS_PENDING_CREATE,
+    RESOURCE_STATUS_PENDING_UPDATE,
+    RS_DIRECTORY_PENDING_CHILD_INSERT,
+};

packages/backend/src/filesystem/storage/SizeService.js

@@ -0,0 +1,193 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { get_dir_size, id2path, get_user, invalidate_cached_user_by_id } = require("../../helpers");
+
+const { DB_WRITE } = require("../../services/database/consts");
+const { Context } = require("../../util/context");
+
+// TODO: expose to a utility library
+class UserParameter {
+    static async adapt (value) {
+        if ( typeof value == 'object' ) return value;
+        const query_object = typeof value === 'number'
+            ? { id: value }
+            : { username: value };
+        return await get_user(query_object);
+    }
+}
+
+class SizeService {
+    constructor ({ services }) {
+        this.db = services.get('database').get(DB_WRITE, 'filesystem');
+        this.log = services.get('log-service').create('size-service');
+        this.errors = services.get('error-service').create(this.log);
+
+        this.usages = {};
+
+        const svc_commands = services.get('commands');
+        svc_commands.registerCommands('size', [
+            {
+                id: 'get-usage',
+                description: 'get usage for a user',
+                handler: async (args, log) => {
+                    const user = await UserParameter.adapt(args[0]);
+                    const usage = await this.get_usage(user.id);
+                    log.log(`usage: ${usage} bytes`);
+                }
+            },
+            {
+                id: 'get-capacity',
+                description: 'get storage capacity for a user',
+                handler: async (args, log) => {
+                    const user = await UserParameter.adapt(args[0]);
+                    const capacity = await this.get_storage_capacity(user);
+                    log.log(`capacity: ${capacity} bytes`);
+                }
+            },
+            {
+                id: 'get-cache-size',
+                description: 'get the number of cached users',
+                handler: async (args, log) => {
+                    const size = Object.keys(this.usages).length;
+                    log.log(`cache size: ${size}`);
+                }
+            },
+        ])
+    }
+
+    async get_usage (user_id) {
+        // if ( this.usages.hasOwnProperty(user_id) ) {
+        //     return this.usages[user_id];
+        // }
+
+        const fsentry = await this.db.read(
+            "SELECT SUM(size) AS total FROM `fsentries` WHERE `user_id` = ? LIMIT 1",
+            [user_id]
+        );
+        if(!fsentry[0] || !fsentry[0].total) {
+            this.usages[user_id] = 0;
+        } else {
+            this.usages[user_id] = parseInt(fsentry[0].total);
+        }
+
+        return this.usages[user_id];
+    }
+
+    async change_usage (user_id, delta) {
+        const usage = await this.get_usage(user_id);
+        this.usages[user_id] = usage + delta;
+    }
+
+    // TODO: remove fs arg and update all calls
+    async add_node_size (fs, node, user, factor = 1) {
+        const {
+            fsEntryService
+        } = Context.get('services').values;
+
+        let sz;
+        if ( node.entry.is_dir ) {
+            if ( node.entry.uuid ) {
+                sz = await fsEntryService.get_recursive_size(node.entry.uuid);
+            } else {
+                // very unlikely, but a warning is better than a throw right now
+                // TODO: remove this once we're sure this is never hit
+                this.log.warn('add_node_size: node has no uuid :(', node)
+                sz = await get_dir_size(await id2path(node.mysql_id), user);
+            }
+        } else {
+            sz = node.entry.size;
+        }
+        await this.change_usage(user.id, sz * factor);
+    }
+
+    async get_storage_capacity (user_or_id) {
+        const user = await UserParameter.adapt(user_or_id);
+        return user.free_storage;
+    }
+
+    /**
+     * Attempt to add storage for a user.
+
+     * In the case of an error, this method will fail silently to the caller and
+     * produce an alarm for further investigation.
+     *
+     * @param {*} user_or_id - user id, username, or user object
+     * @param {*} amount_in_bytes - amount of bytes to add
+     * @param {*} reason - please specify a reason for the storage increase
+     * @param {*} param3 - optional fields to add to the audit log
+     */
+    async add_storage (user_or_id, amount_in_bytes, reason, { field_a, field_b } = {}) {
+        const user = await UserParameter.adapt(user_or_id);
+        const capacity = await this.get_storage_capacity(user);
+
+        // Audit log
+        {
+            const entry = {
+                user_id: user.id,
+                user_id_keep: user.id,
+                amount: amount_in_bytes,
+                reason,
+                ...(field_a ? { field_a } : {}),
+                ...(field_b ? { field_b } : {}),
+            };
+
+            const fields_ = Object.keys(entry);
+            const fields = fields_.join(', ');
+            const placeholders = fields_.map(f => '?').join(', ');
+            const values = fields_.map(f => entry[f]);
+
+            try {
+                await this.db.write(
+                    `INSERT INTO storage_audit (${fields}) VALUES (${placeholders})`,
+                    values,
+                );
+            } catch (e) {
+                this.errors.report('size-service.audit-add-storage', {
+                    source: e,
+                    trace: true,
+                    alarm: true,
+                })
+            }
+        }
+
+        // Storage increase
+        {
+            try {
+                const res = await this.db.write(
+                    "UPDATE `user` SET `free_storage` = ? WHERE `id` = ? LIMIT 1",
+                    [capacity + amount_in_bytes, user.id]
+                );
+                if ( ! res.anyRowsAffected ) {
+                    throw new Error(`add_storage: failed to update user ${user.id}`);
+                }
+            } catch (e) {
+                this.errors.report('size-service.add-storage', {
+                    source: e,
+                    trace: true,
+                    alarm: true,
+                })
+            }
+            invalidate_cached_user_by_id(user.id);
+        }
+    }
+}
+
+module.exports = {
+    SizeService,
+};

packages/backend/src/filesystem/storage/SystemFSEntryService.js

@@ -0,0 +1,164 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+const { PuterPath } = require("../lib/PuterPath");
+
+// Redis keys:
+// <env>:<service>:<class>:<type>:<property>:<id>
+//
+// note: <environment> is added by redisService automatically.
+//
+// If `<type>` is `multi`, then the format differs slightly:
+// <env>:<service>:<class>:multi:<type>:<property>:<id-property>:<id>
+// where `<id-property>` specifies the propery being used for the id
+
+class SystemFSEntryService {
+    constructor ({ services }) {
+        this.redis = { enabled: false };
+        this.DatabaseFSEntryService = services.get('fsEntryService');
+
+        this.log = services.get('log-service').create('system-fsentry-service');
+
+        // Register information providers
+        const info = services.get('information');
+        this.info = info;
+        
+        if ( ! this.redis.enabled ) return;
+        
+        // path -> uuid via redis
+        info.given('fs.fsentry:path').provide('fs.fsentry:uuid')
+            .addStrategy('redis', async path => {
+                return await this.get_uuid_from_path(path);
+            });
+        // uuid -> path via redis
+        info.given('fs.fsentry:uuid').provide('fs.fsentry:path')
+            .addStrategy('redis', async uuid => {
+                this.log.debug('getting path for: ' + uuid);
+                if ( uuid === PuterPath.NULL_UUID ) return '/';
+                const res =  ( await this.redis.get(`fs:fsentry:path:path:${uuid}`) ) ?? undefined;
+                this.log.debug('got path: ' + res);
+                return res;
+            });
+        // uuid -> parent_uuid via redis
+        info.given('fs.fsentry:uuid').provide('fs.fsentry:children(fs.fsentry:uuid)')
+            .addStrategy('redis', async uuid => {
+                return await this.get_child_uuids(uuid);
+            });
+    }
+
+    async insert (entry) {
+        if ( this.redis.enabled ) {
+            await this._link(entry.uuid, entry.parent_uid, entry.name);
+        }
+        return await this.DatabaseFSEntryService.insert(entry);
+    }
+
+    async update (uuid, entry) {
+        // If parent_uid is set during an update, we assume that it
+        // has been changed. If it hasn't, no problem: just an extra
+        // cache invalidation; but the code that set it should know
+        // better because it probably has the fsentry data already.
+        if ( entry.hasOwnProperty('parent_uid') ) {
+            await this._relocate(uuid, entry.parent_uid)
+        }
+        return await this.DatabaseFSEntryService.update(uuid, entry);
+    }
+
+    async delete (uuid) {
+        //
+    }
+
+    async get_child_uuids (uuid) {
+        let members;
+        members = await this.redis.smembers(`fs:fsentry:set:childs:${uuid}`);
+        if ( members ) return members;
+        members = await this.DatabaseFSEntryService.get_descendants(uuid);
+        return members ?? [];
+    }
+
+    async get_uuid_from_path (path) {
+        path = PuterPath.adapt(path);
+
+        let current = path.reference;
+        let pathOfReference = path.reference === PuterPath.NULL_UUID
+            ? '/' : this.get_path_from_uuid(path.reference);
+        
+        const fullPath = _path.join(pathOfReference, path.relativePortion);
+        let uuid = await this.redis.get(`fs:fsentry:multi:uuid:uuid:path:${fullPath}`);
+        return uuid;
+    }
+
+    // Cache related functions
+    async _link (subject_uuid, parent_uuid, subject_name) {
+        this.log.info(`linking ${subject_uuid} to ${parent_uuid}`);
+        // We need the parent's path to update everything
+        
+        let pathOfParent = await this.info.with('fs.fsentry:uuid')
+            .obtain('fs.fsentry:path').exec(parent_uuid);
+
+        this.log.debug(`path of parent: ${pathOfParent}`);
+
+        if ( ! subject_name ) {
+            subject_name = await this.redis.get(`fs:fsentry:str:name:${subject_uuid}`);
+        }
+
+        // Register properties
+        await this.redis.set(`fs:fsentry:uuid:parent:${subject_uuid}`, parent_uuid);
+        await this.redis.set(`fs:fsentry:str:name:${subject_uuid}`, subject_name);
+        
+        // Add as child of parent
+        await this.redis.sadd(`fs:fsentry:set:childs:${parent_uuid}`, subject_uuid);
+        
+        // Register path
+        const subject_path = `${pathOfParent}/${subject_name}`;
+        this.log.debug(`registering path: ${subject_path} for ${subject_uuid}`);
+        await this.redis.set(`fs:fsentry:path:path:${subject_uuid}`, subject_path);
+        await this.redis.set(`fs:fsentry:multi:uuid:uuid:path:${subject_path}`, subject_uuid);
+    }
+
+    async _unlink (subject_uuid) {
+        let parent_uuid = await this.redis.get(`fs:fsentry:uuid:parent:${subject_uuid}`);
+        // TODO: try getting from database
+
+        // Remove from parent
+        await this.redis.srem(`fs:fsentry:set:childs:${parent_uuid}`, subject_uuid);
+    }
+
+    async _purge (subject_uuid) {
+        await this._unlink(subject_uuid);
+    
+        // Remove properties
+        await this.redis.del(`fs:fsentry:uuid:parent:${subject_uuid}`);
+        await this.redis.del(`fs:fsentry:str:name:${subject_uuid}`);
+
+        // Remove path
+        const subject_path =
+            await this.redis.get(`fs:fsentry:path:path:${subject_uuid}`);
+        await this.redis.del(`fs:fsentry:path:path:${subject_uuid}`);
+        if ( subject_path ) {
+            await this.redis.del(`fs:fsentry:multi:uuid:path:${subject_path}`);
+        }
+    }
+
+    async _relocate (subject_uuid, new_parent_uuid) {
+        await this._unlink(subject_uuid);
+        await this._link(subject_uuid, new_parent_uuid);
+    }
+}
+
+module.exports = SystemFSEntryService;

packages/backend/src/filesystem/storage/UploadProgressTracker.js

@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+class UploadProgressTracker {
+    constructor () {
+        this.progress_ = 0;
+        this.total_ = 0;
+        this.done_ = false;
+
+        this.listeners_ = [];
+    }
+
+    set_total (v) {
+        this.total_ = v;
+    }
+
+    set (value) {
+        if ( value < this.progress_ ) {
+            // TODO: provide a logger for a warning
+            return;
+        }
+        const delta = value - this.progress_;
+        this.add(delta);
+    }
+
+    add (amount) {
+        if ( this.done_ ) {
+            return; // TODO: warn
+        }
+
+        this.progress_ += amount;
+
+        for ( const lis of this.listeners_ ) {
+            lis(amount);
+        }
+
+        this.check_if_done_();
+    }
+
+    sub (callback) {
+        if ( this.done_ ) {
+            return;
+        }
+
+        const listeners = this.listeners_;
+
+        listeners.push(callback);
+
+        const det = {
+            detach: () => {
+                const idx = listeners.indexOf(callback);
+                if ( idx !== -1 ) {
+                    listeners.splice(idx, 1);
+                }
+            }
+        }
+
+        return det;
+    }
+
+    check_if_done_ () {
+        if ( this.progress_ === this.total_ ) {
+            this.done_ = true;
+            // clear listeners so they get GC'd
+            this.listeners_ = [];
+        }
+    }
+}
+
+module.exports = {
+    UploadProgressTracker,
+};

packages/backend/src/filesystem/strategies/README.md

@@ -0,0 +1,12 @@
+## Puter Filesystem Strategies
+
+Each subdirectory is named in the format `<concern>_<class>`,
+where `<concern>` specifies broadly what that strategies contained within
+the directory are concerned with (storage, fsentry, etc), and `<class>`
+is a letter from A-Z indicating the layer/level of concern.
+
+The class **A** indicates that this is the highest level of swappable
+behaviour, which generally means there will be two strategies:
+- one which supports legacy behaviour that is coupled with multiple concerns
+- one which adapts more cohesive strategies to an interface which
+  supports the case above.