test: add database migration tests

package-lock.json

@@ -9798,6 +9798,10 @@
         "node": ">=8.6"
       }
     },
+    "node_modules/migrations-test": {
+      "resolved": "tools/migrations-test",
+      "link": true
+    },
     "node_modules/mime": {
       "version": "1.6.0",
       "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
@@ -13728,7 +13732,6 @@
       "version": "17.7.2",
       "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz",
       "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==",
-      "license": "MIT",
       "dependencies": {
         "cliui": "^8.0.1",
         "escalade": "^3.1.1",
@@ -14102,7 +14105,8 @@
         "uuid": "^9.0.0",
         "validator": "^13.9.0",
         "winston": "^3.9.0",
-        "winston-daily-rotate-file": "^4.7.1"
+        "winston-daily-rotate-file": "^4.7.1",
+        "yargs": "^17.7.2"
       },
       "devDependencies": {
         "@types/node": "^20.5.3",
@@ -14399,6 +14403,21 @@
         "js-levenshtein": "^1.1.6",
         "yaml": "^2.4.5"
       }
+    },
+    "tools/migrations-test": {
+      "version": "1.0.0",
+      "license": "AGPL-3.0-only",
+      "dependencies": {
+        "commander": "^12.1.0"
+      }
+    },
+    "tools/migrations-test/node_modules/commander": {
+      "version": "12.1.0",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz",
+      "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==",
+      "engines": {
+        "node": ">=18"
+      }
     }
   }
 }

src/backend/package.json

@@ -71,7 +71,8 @@
     "uuid": "^9.0.0",
     "validator": "^13.9.0",
     "winston": "^3.9.0",
-    "winston-daily-rotate-file": "^4.7.1"
+    "winston-daily-rotate-file": "^4.7.1",
+    "yargs": "^17.7.2"
   },
   "devDependencies": {
     "@types/node": "^20.5.3",

src/backend/src/CoreModule.js

@@ -311,6 +311,12 @@ const install = async ({ services, app, useapi }) => {
     
     const { SUService } = require('./services/SUService');
     services.registerService('su', SUService);
+
+    const { ShutdownService } = require('./services/ShutdownService');
+    services.registerService('shutdown', ShutdownService);
+
+    const { BootScriptService } = require('./services/BootScriptService');
+    services.registerService('boot-script', BootScriptService);
 }
 
 const install_legacy = async ({ services }) => {

src/backend/src/Kernel.js

@@ -20,6 +20,9 @@ const { AdvancedBase } = require("@heyputer/puter-js-common");
 const { Context } = require('./util/context');
 const BaseService = require("./services/BaseService");
 const useapi = require('useapi');
+const yargs = require('yargs/yargs')
+const { hideBin } = require('yargs/helpers')
+
 
 class Kernel extends AdvancedBase {
     constructor ({ entry_path } = {}) {
@@ -67,7 +70,9 @@ class Kernel extends AdvancedBase {
     }
 
     boot () {
-        this._runtime_init();
+        const args = yargs(hideBin(process.argv)).argv
+
+        this._runtime_init({ args });
 
         // const express = require('express')
         // const app = express();
@@ -111,6 +116,7 @@ class Kernel extends AdvancedBase {
             services,
             config,
             logger: this.bootLogger,
+            args,
         }, 'app');
         globalThis.root_context = root_context;
 

src/backend/src/services/BootScriptService.js

@@ -0,0 +1,42 @@
+const { Context } = require("../util/context");
+const BaseService = require("./BaseService");
+
+class BootScriptService extends BaseService {
+    static MODULES = {
+        fs: require('fs'),
+    }
+    async ['__on_boot.ready'] () {
+        const args = Context.get('args');
+        if ( ! args['boot-script'] ) return;
+        const script_name = args['boot-script'];
+
+        const require = this.require;
+        const fs = require('fs');
+        const boot_json_raw = fs.readFileSync(script_name, 'utf8');
+        const boot_json = JSON.parse(boot_json_raw);
+        await this.run_script(boot_json);
+    }
+
+    async run_script (boot_json) {
+        const scope = {
+            runner: 'boot-script',
+            ['end-puter-process']: ({ args }) => {
+                const svc_shutdown = this.services.get('shutdown');
+                svc_shutdown.shutdown(args[0]);
+            }
+        };
+
+        for ( let i=0 ; i < boot_json.length ; i++ ) {
+            const statement = boot_json[i];
+            const [cmd, ...args] = statement;
+            if ( ! scope[cmd] ) {
+                throw new Error(`Unknown command: ${cmd}`);
+            }
+            await scope[cmd]({ scope, args });
+        }
+    }
+}
+
+module.exports = {
+    BootScriptService
+};

src/backend/src/services/ShutdownService.js

@@ -0,0 +1,11 @@
+const BaseService = require("./BaseService");
+
+class ShutdownService extends BaseService {
+    shutdown ({ reason, code } = {}) {
+        this.log.info(`Puter is shutting down: ${reason ?? 'no reason provided'}`);
+        process.stdout.write('\x1B[0m\r\n');
+        process.exit(code ?? 0);
+    }
+}
+
+module.exports = { ShutdownService };

src/backend/src/services/database/SqliteDatabaseAccessService.js

@@ -18,6 +18,7 @@
  */
 const { es_import_promise } = require("../../fun/dev-console-ui-utils");
 const { surrounding_box } = require("../../fun/dev-console-ui-utils");
+const { Context } = require("../../util/context");
 const { CompositeError } = require("../../util/errorutil");
 const structutil = require("../../util/structutil");
 const { BaseDatabaseAccessService } = require("./BaseDatabaseAccessService");
@@ -44,7 +45,14 @@ class SqliteDatabaseAccessService extends BaseDatabaseAccessService {
         this.db = new Database(this.config.path);
 
         // Database upgrade logic
-        const TARGET_VERSION = 24;
+        const HIGHEST_VERSION = 24;
+        const TARGET_VERSION = (() => {
+            const args = Context.get('args');
+            if ( args['database-target-version'] ) {
+                return parseInt(args['database-target-version']);
+            }
+            return HIGHEST_VERSION;
+        })();
 
         const [{ user_version }] = do_setup
             ? [{ user_version: -1 }]
@@ -53,105 +61,93 @@ class SqliteDatabaseAccessService extends BaseDatabaseAccessService {
 
         const upgrade_files = [];
 
-        if ( user_version === -1 ) {
-            upgrade_files.push('0001_create-tables.sql');
-            upgrade_files.push('0002_add-default-apps.sql');
-        }
-
-        if ( user_version <= 0 ) {
-            upgrade_files.push('0003_user-permissions.sql');
-        }
-
-        if ( user_version <= 1 ) {
-            upgrade_files.push('0004_sessions.sql');
-        }
-
-        if ( user_version <= 2 ) {
-            upgrade_files.push('0005_background-apps.sql');
-        }
-
-        if ( user_version <= 3 ) {
-            upgrade_files.push('0006_update-apps.sql');
-        }
-
-        if ( user_version <= 4 ) {
-            upgrade_files.push('0007_sessions.sql');
-        }
-
-        if ( user_version <= 5 ) {
-            upgrade_files.push('0008_otp.sql');
-        }
-
-        if ( user_version <= 6 ) {
-            upgrade_files.push('0009_app-prefix-fix.sql');
-        }
-
-        if ( user_version <= 7 ) {
-            upgrade_files.push('0010_add-git-app.sql');
-        }
-
-        if ( user_version <= 8 ) {
-            upgrade_files.push('0011_notification.sql');
-        }
-
-        if ( user_version <= 9 ) {
-            upgrade_files.push('0012_appmetadata.sql');
-        }
-
-        if ( user_version <= 10 ) {
-            upgrade_files.push('0013_protected-apps.sql');
-        }
-
-        if ( user_version <= 11 ) {
-            upgrade_files.push('0014_share.sql');
-        }
-
-        if ( user_version <= 12 ) {
-            upgrade_files.push('0015_group.sql');
-        }
-
-        if ( user_version <= 13 ) {
-            upgrade_files.push('0016_group-permissions.sql');
-        }
-
-        if ( user_version <= 14 ) {
-            upgrade_files.push('0017_publicdirs.sql');
-        }
-
-        if ( user_version <= 15 ) {
-            upgrade_files.push('0018_fix-0003.sql');
-        }
-
-        if ( user_version <= 16 ) {
-            upgrade_files.push('0019_fix-0016.sql');
-        }
-
-        if ( user_version <= 17 ) {
-            upgrade_files.push('0020_dev-center.sql');
-        }
-
-        if ( user_version <= 18 ) {
-            upgrade_files.push('0021_app-owner-id.sql');
-        }
-
-        if ( user_version <= 19 ) {
-            upgrade_files.push('0022_dev-center-max.sql');
-        }
-
-        if ( user_version <= 20 ) {
-            upgrade_files.push('0023_fix-kv.sql');
-        }
-
-        if ( user_version <= 21 ) {
-            upgrade_files.push('0024_default-groups.sql');
-        }
-
-        if ( user_version <= 22 ) {
-            upgrade_files.push('0025_system-user.dbmig.js');
-        }
-
-        if ( user_version <= 23 ) {
-            upgrade_files.push('0026_user-groups.dbmig.js');
+        const available_migrations = [
+            [-1, [
+                '0001_create-tables.sql',
+                '0002_add-default-apps.sql',
+            ]],
+            [0, [
+                '0003_user-permissions.sql',
+            ]],
+            [1, [
+                '0004_sessions.sql',
+            ]],
+            [2, [
+                '0005_background-apps.sql',
+            ]],
+            [3, [
+                '0006_update-apps.sql',
+            ]],
+            [4, [
+                '0007_sessions.sql',
+            ]],
+            [5, [
+                '0008_otp.sql',
+            ]],
+            [6, [
+                '0009_app-prefix-fix.sql',
+            ]],
+            [7, [
+                '0010_add-git-app.sql',
+            ]],
+            [8, [
+                '0011_notification.sql',
+            ]],
+            [9, [
+                '0012_appmetadata.sql',
+            ]],
+            [10, [
+                '0013_protected-apps.sql',
+            ]],
+            [11, [
+                '0014_share.sql',
+            ]],
+            [12, [
+                '0015_group.sql',
+            ]],
+            [13, [
+                '0016_group-permissions.sql',
+            ]],
+            [14, [
+                '0017_publicdirs.sql',
+            ]],
+            [15, [
+                '0018_fix-0003.sql',
+            ]],
+            [16, [
+                '0019_fix-0016.sql',
+            ]],
+            [17, [
+                '0020_dev-center.sql',
+            ]],
+            [18, [
+                '0021_app-owner-id.sql',
+            ]],
+            [19, [
+                '0022_dev-center-max.sql',
+            ]],
+            [20, [
+                '0023_fix-kv.sql',
+            ]],
+            [21, [
+                '0024_default-groups.sql',
+            ]],
+            [22, [
+                '0025_system-user.dbmig.js',
+            ]],
+            [23, [
+                '0026_user-groups.dbmig.js',
+            ]],
+        ];
+
+        for ( const [v_lt_or_eq, files] of available_migrations ) {
+            if ( v_lt_or_eq + 1 >= TARGET_VERSION && TARGET_VERSION !== HIGHEST_VERSION ) {
+                this.log.noticeme(`Early exit: target version set to ${TARGET_VERSION}`);
+                break;
+            }
+            if ( user_version <= v_lt_or_eq ) {
+                upgrade_files.push(...files);
+            }
         }
 
         if ( upgrade_files.length > 0 ) {

src/backend/src/services/runtime-analysis/AlarmService.js

@@ -28,6 +28,7 @@ const { generate_identifier } = require('../../util/identifier.js');
 const { stringify_log_entry } = require('./LogService.js');
 const BaseService = require('../BaseService.js');
 const { split_lines } = require('../../util/stdioutil.js');
+const { Context } = require('../../util/context.js');
 
 class AlarmService extends BaseService {
     async _construct () {
@@ -251,6 +252,15 @@ class AlarmService extends BaseService {
             svc_devConsole.add_widget(this.alarm_widget);
         }
 
+        const args = Context.get('args');
+        if ( args['quit-on-alarm'] ) {
+            const svc_shutdown = this.services.get('shutdown');
+            svc_shutdown.shutdown({
+                reason: '--quit-on-alarm is set',
+                code: 1,
+            });
+        }
+
         if ( alarm.no_alert ) return;
 
         const severity = alarm.severity ?? 'critical';

tools/migrations-test/main.js

@@ -0,0 +1,139 @@
+const path_ = require('node:path');
+const fs = require('node:fs');
+const { spawnSync } = require('node:child_process');
+const prompt = require('prompt-sync')({sigint: true}); 
+
+const ind_str = () => Array(ind).fill(' --').join('');
+
+let ind = 0;
+
+const log = {
+    // log with unicode warning symbols in yellow
+    warn: (msg) => {
+        console.log(`\x1b[33;1m[!]${ind_str()} ${msg}\x1b[0m`);
+    },
+    crit: (msg) => {
+        console.log(`\x1b[31;1m[!]${ind_str()} ${msg}\x1b[0m`);
+    },
+    info: (msg) => {
+        console.log(`\x1B[36;1m[i]\x1B[0m${ind_str()} ${msg}`);
+    },
+    named: (name, value) => {
+        console.log(`\x1B[36;1m[i]${ind_str()} ${name}\x1B[0m ${value}`);
+    },
+    error: e => {
+        if ( e instanceof UserError ) {
+            log.crit(e.message);
+        } else {
+            console.error(e);
+        }
+    },
+    indent () { ind++; },
+    dedent () { ind--; },
+    heading (title) {
+        const circle = '🔵';
+        console.log(`\n\x1b[36;1m${circle} ${title} ${circle}\x1b[0m`);
+    }
+};
+
+const areyousure = (message, options = {}) => {
+    const { crit } = options;
+    const logfn = crit ? log.crit : log.warn;
+    
+    logfn(message);
+    const answer = prompt(`\x1B[35;1m[?]\x1B[0m ${ options?.prompt ?? 'Are you sure?' } (y/n): `);
+    if ( answer !== 'y' ) {
+
+        if ( options.fail_hint ) {
+            log.info(options.fail_hint);
+        }
+
+        console.log(`\x1B[31;21;1mAborted.\x1B[0m`);
+        process.exit(1);
+    }
+}
+
+if ( ! fs.existsSync('.is_puter_repository') ) {
+    throw new Error('This script must be run from the root of a puter repository');
+}
+
+areyousure(
+    'This script will delete all data in the database. Are you sure you want to proceed?',
+    { crit: true }
+)
+
+let backup_created = false;
+
+const DBPATH = 'volatile/runtime/puter-database.sqlite';
+const delete_db = () => {
+    if ( ! fs.existsSync(DBPATH) ) {
+        log.info('No database file to remove');
+        // no need to create a backup if the database doesn't exist
+        backup_created = true;
+        return;
+    }
+    if ( ! backup_created ) {
+        log.info(`Creating a backup of the database...`);
+        const RANDOM = Math.floor(Math.random() * 1000000);
+        const DATE = new Date().toISOString().replace(/:/g, '-');
+        fs.renameSync(DBPATH, `${DBPATH}_${DATE}_${RANDOM}.bak`);
+        backup_created = true;
+        return;
+    }
+    log.info('Removing database file');
+    fs.unlinkSync(DBPATH);
+}
+
+const pwd = process.cwd();
+const boot_script_path = path_.join(pwd, 'tools/migrations-test/noop.puter.json');
+
+const launch_puter = (args) => {
+    const ret = spawnSync(
+        'node',
+        ['tools/run-selfhosted.js', ...args],
+        {
+            stdio: 'inherit',
+            env: {
+                ...process.env,
+                NO_VAR_RUNTIME: '1',
+            },
+        }
+    );
+    ret.ok = ret.status === 0;
+    return ret;
+};
+
+{
+    delete_db();
+    log.info(`Test case: fresh install`);
+    if ( ! launch_puter([
+        '--quit-on-alarm',
+        `--boot-script=${boot_script_path}`,
+    ]).ok ) {
+        log.crit('Migration to v21 raised alarm');
+        process.exit(1);
+    }
+}
+{
+    delete_db();
+    log.info(`Test case: migrate to 21, then migrate to 24`);
+    if ( ! launch_puter([
+        `--database-target-version=21`,
+        '--quit-on-alarm',
+        `--boot-script=${boot_script_path}`,
+    ]).ok ) {
+        log.crit('Migration to v21 raised alarm');
+        process.exit(1);
+    }
+    if ( ! launch_puter([
+        `--database-target-version=24`,
+        '--quit-on-alarm',
+        `--boot-script=${boot_script_path}`,
+    ]).ok ) {
+        log.crit('Migration to v24 raised alarm');
+        process.exit(1);
+    }
+}
+
+log.info('No migration scripts produced any obvious errors.');
+log.warn('This is not a substitute for release candidate migration testing!');

tools/migrations-test/noop.puter.json

@@ -0,0 +1,3 @@
+[
+    ["end-puter-process", { "reason": "migrations test" }]
+]

tools/migrations-test/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "migrations-test",
+  "version": "1.0.0",
+  "main": "main.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "AGPL-3.0-only",
+  "description": "",
+  "dependencies": {
+    "commander": "^12.1.0"
+  }
+}