dev: de-couple legacy mkdir and remove it

A legacy mkdir function was still being used by the
generate_system_fsentries function. However, it wasn't necessary because
we now associate fsentries by uuid, eliminating the requirement to
create the home directory separately from its subdirectories.

The old mkdir function was removed, and generate_system_fsentries was
moved to a new service called UserService, which is intended to
eventually hold all methods related creating, removing, and updating
users.

mods/mods_available/kdmod/ShareTestService.js

@@ -21,7 +21,6 @@
 //       we have these things registered in "useapi".
 const {
     get_user,
-    generate_system_fsentries,
     invalidate_cached_user,
     deleteUser,
 } = require('../../../src/backend/src/helpers.js');
@@ -146,7 +145,8 @@ class ShareTestService extends use.Service {
             ],
         );
         const user = await get_user({ username });
-        await generate_system_fsentries(user);
+        const svc_user = this.services.get('user');
+        await svc_user.generate_default_fsentries({ user });
         invalidate_cached_user(user);
         return user;
     }

src/backend/src/CoreModule.js

@@ -351,6 +351,9 @@ const install = async ({ services, app, useapi, modapi }) => {
 
     const { ReferralCodeService } = require('./services/ReferralCodeService');
     services.registerService('referral-code', ReferralCodeService);
+    
+    const { UserService } = require('./services/UserService');
+    services.registerService('user', UserService);
 }
 
 const install_legacy = async ({ services }) => {

src/backend/src/helpers.js

@@ -1155,201 +1155,6 @@ async function jwt_auth(req){
     return ancestors;
 }
 
-// THIS LEGACY FUNCTION IS STILL IN USE
-// by: generate_system_fsentries
-// TODO: migrate generate_system_fsentries to use QuickMkdir
-async function mkdir(options){
-    const fs = systemfs;
-
-    debugger;
-
-    const resolved_path = PathBuilder.resolve(options.path, { puterfs: true });
-
-    const dirpath           = _path.dirname(resolved_path);
-    let target_name         = _path.basename(resolved_path);
-    const overwrite         = options.overwrite ?? false;
-    const dedupe_name       = options.dedupe_name ?? false;
-    const immutable         = options.immutable ?? false;
-    const return_id         = options.return_id ?? false;
-    const no_perm_check     = options.no_perm_check ?? false;
-
-    // make parent directories as needed
-    const create_missing_parents           = options.create_missing_parents ?? false;
-
-    // hold a list of all parent directories created in the process
-    let parent_dirs_created                = [];
-    let overwritten_uid;
-
-    // target_name validation
-    try{
-        validate_fsentry_name(target_name)
-    }catch(e){
-        throw e.message;
-    }
-
-    // resolve dirpath to its fsentry
-    let parent = await convert_path_to_fsentry(dirpath);
-
-    // dirpath not found
-    if(parent === false && !create_missing_parents)
-        throw new Error("Target path not found");
-    // create missing parent directories
-    else if(parent === false && create_missing_parents){
-        const dirs = _path.resolve('/', dirpath).split('/');
-        let cur_path = '';
-        for(let j=0; j < dirs.length; j++){
-            if(dirs[j] === '')
-                continue;
-
-            cur_path += '/'+dirs[j];
-            // skip creating '/[username]'
-            if(j === 1)
-                continue;
-            try{
-                let d = await mkdir(fs, {path: cur_path, user: options.user});
-                d.path = cur_path;
-                parent_dirs_created.push(d);
-            }catch(e){
-                console.log(`Skipped mkdir ${cur_path}`);
-            }
-        }
-        // try setting parent again
-        parent = await convert_path_to_fsentry(dirpath);
-        if(parent === false)
-            throw new Error("Target path not found");
-    }
-
-    // check permission
-    if(!no_perm_check && !await chkperm(parent, options.user.id, 'write'))
-        throw { code:`forbidden`, message: `permission denied.`};
-
-    // check if a fsentry with the same name exists under this path
-    const existing_fsentry = await convert_path_to_fsentry(_path.resolve('/', dirpath + '/' + target_name ));
-
-    /** @type BaseDatabaseAccessService */
-    const db = services.get('database').get(DB_WRITE, 'filesystem');
-
-    // if trying to create a directory with an existing path and overwrite==false, throw an error
-    if(!overwrite && !dedupe_name && existing_fsentry !== false){
-        throw {
-            code: 'path_exists',
-            message:"A file/directory with the same path already exists.",
-            entry_name: existing_fsentry.name,
-            existing_fsentry: {
-                name: existing_fsentry.name,
-                uid: existing_fsentry.uuid,
-            }
-        };
-    }
-    else if(overwrite && existing_fsentry){
-        overwritten_uid = existing_fsentry.uuid;
-        // check permission
-        if(!await chkperm(existing_fsentry, options.user.id, 'write'))
-            throw {code:`forbidden`, message: `permission denied.`};
-        // delete existing dir
-        await db.write(
-            `DELETE FROM fsentries WHERE id = ? AND user_id = ?`,
-            [
-                //parent_uid
-                existing_fsentry.uuid,
-                //user_id
-                options.user.id,
-            ]);
-    }
-    // dedupe name, generate a new name until its unique
-    else if(dedupe_name && existing_fsentry !== false){
-        for( let i = 1; ; i++){
-            let try_new_name = existing_fsentry.name + ' (' + i + ')';
-            let check_dupe = await db.read(
-                "SELECT `id` FROM `fsentries` WHERE `parent_uid` = ? AND name = ? LIMIT 1",
-                [existing_fsentry.parent_uid, try_new_name]
-            );
-            if(check_dupe[0] === undefined){
-                target_name = try_new_name;
-                break;
-            }
-        }
-    }
-
-    // shrotcut?
-    let shortcut_fsentry;
-    if(options.shortcut_to){
-        shortcut_fsentry = await uuid2fsentry(options.shortcut_to);
-        if(shortcut_fsentry === false){
-            throw ({ code:`not_found`, message: `shortcut_to not found.`})
-        }else if(!parent.is_dir){
-            throw ({ code:`not_dir`, message: `parent of shortcut_to must be a directory`})
-        }else if(!await chkperm(shortcut_fsentry, options.user.id, 'read')){
-            throw ({ code:`forbidden`, message: `shortcut_to permission denied.`})
-        }
-    }
-
-    // current epoch
-    const ts = Math.round(Date.now() / 1000)
-    const uid = uuidv4();
-
-    // record in db
-    let user_id = (parent === null ? options.user.id : parent.user_id);
-    const { insertId: mkdir_db_id } = await db.write(
-        `INSERT INTO fsentries
-        (uuid, parent_uid, user_id,  name,    is_dir, created, modified, immutable,  shortcut_to, is_shortcut) VALUES
-        (   ?,          ?,       ?,     ?,      true,       ?,        ?,         ?,            ?,           ?)`,
-        [
-            //uuid
-            uid,
-            //parent_uid
-            (parent === null) ? null : parent.uuid,
-            //user_id
-            user_id,
-            //name
-            target_name,
-            //created
-            ts,
-            //modified
-            ts,
-            //immutable
-            immutable,
-            //shortcut_to,
-            shortcut_fsentry ? shortcut_fsentry.id : null,
-            //is_shortcut,
-            shortcut_fsentry ? 1 : 0,
-        ]
-    );
-
-    const ret_obj = {
-        uid : uid,
-        name: target_name,
-        immutable: immutable,
-        is_dir: true,
-        path: options.path ?? false,
-        dirpath: dirpath,
-        is_shared: await is_shared_with_anyone(mkdir_db_id),
-        overwritten_uid: overwritten_uid,
-        shortcut_to: shortcut_fsentry ? shortcut_fsentry.uuid : null,
-        shortcut_to_path: shortcut_fsentry ? await id2path(shortcut_fsentry.id) : null,
-        parent_dirs_created: parent_dirs_created,
-        original_client_socket_id: options.original_client_socket_id,
-    };
-    // add existing_fsentry if exists
-    if(existing_fsentry){
-        ret_obj.existing_fsentry ={
-            name: existing_fsentry.name,
-            uid: existing_fsentry.uuid,
-        }
-    }
-
-    if(return_id)
-        ret_obj.id = mkdir_db_id;
-
-    // send realtime success msg to client
-    let socketio = require('./socketio.js').getio();
-    if(socketio){
-        socketio.to(user_id).emit('item.added', ret_obj)
-    }
-
-    return ret_obj;
-}
-
 function is_valid_uuid ( uuid ) {
     let s = "" + uuid;
     s = s.match(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-5][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}$/i);
@@ -1412,100 +1217,6 @@ async function app_name_exists(name){
         return true;
 }
 
-
-// generates all the default files and directories a user needs,
-// generally used for a brand new account
-async function generate_system_fsentries(user){
-    /** @type BaseDatabaseAccessService */
-    const db = services.get('database').get(DB_WRITE, 'filesystem');
-
-    //-------------------------------------------------------------
-    // create root `/[username]/`
-    //-------------------------------------------------------------
-    const root_dir = await mkdir({
-        path: '/' + user.username,
-        user: user,
-        immutable: true,
-        no_perm_check: true,
-        return_id: true,
-    });
-
-    // Normally, it is recommended to use mkdir() to create new folders,
-    // but during signup this could result in multiple queries to the DB server
-    // and for servers in remote regions such as Asia this could result in a
-    // very long time for /signup to finish, sometimes up to 30-40 seconds!
-    // by combining as many queries as we can into one and avoiding multiple back-and-forth
-    // with the DB server, we can speed this process up significantly.
-    const ts = Date.now()/1000;
-
-    // Generate UUIDs for all the default folders and files
-    let trash_uuid = uuidv4();
-    let appdata_uuid = uuidv4();
-    let desktop_uuid = uuidv4();
-    let documents_uuid = uuidv4();
-    let pictures_uuid = uuidv4();
-    let videos_uuid = uuidv4();
-    let public_uuid = uuidv4();
-
-    const insert_res = await db.write(
-        `INSERT INTO fsentries
-        (uuid, parent_uid, user_id, name, path, is_dir, created, modified, immutable) VALUES
-        (   ?,          ?,       ?,    ?,    ?,   true,       ?,        ?,      true),
-        (   ?,          ?,       ?,    ?,    ?,   true,       ?,        ?,      true),
-        (   ?,          ?,       ?,    ?,    ?,   true,       ?,        ?,      true),
-        (   ?,          ?,       ?,    ?,    ?,   true,       ?,        ?,      true),
-        (   ?,          ?,       ?,    ?,    ?,   true,       ?,        ?,      true),
-        (   ?,          ?,       ?,    ?,    ?,   true,       ?,        ?,      true),
-        (   ?,          ?,       ?,    ?,    ?,   true,       ?,        ?,      true)
-        `,
-        [
-            // Trash
-            trash_uuid, root_dir.uid, user.id, 'Trash', `/${user.username}/Trash`, ts, ts,
-            // AppData
-            appdata_uuid, root_dir.uid, user.id, 'AppData', `/${user.username}/AppData`, ts, ts,
-            // Desktop
-            desktop_uuid, root_dir.uid, user.id, 'Desktop', `/${user.username}/Desktop`, ts, ts,
-            // Documents
-            documents_uuid, root_dir.uid, user.id, 'Documents', `/${user.username}/Documents`, ts, ts,
-            // Pictures
-            pictures_uuid, root_dir.uid, user.id, 'Pictures', `/${user.username}/Pictures`, ts, ts,
-            // Videos
-            videos_uuid, root_dir.uid, user.id, 'Videos', `/${user.username}/Videos`, ts, ts,
-            // Public
-            public_uuid, root_dir.uid, user.id, 'Public', `/${user.username}/Public`, ts, ts,
-        ]
-    );
-
-    // https://stackoverflow.com/a/50103616
-    let trash_id = insert_res.insertId;
-    let appdata_id = insert_res.insertId + 1;
-    let desktop_id = insert_res.insertId + 2;
-    let documents_id = insert_res.insertId + 3;
-    let pictures_id = insert_res.insertId + 4;
-    let videos_id = insert_res.insertId + 5;
-    let public_id = insert_res.insertId + 6;
-
-    // Asynchronously set the user's system folders uuids in database
-    // This is for caching purposes, so we don't have to query the DB every time we need to access these folders
-    // This is also possible because we know the user's system folders uuids will never change
-
-    // TODO: pass to IIAFE manager to avoid unhandled promise rejection
-    // (IIAFE manager doesn't exist yet, hence this is a TODO)
-    db.write(
-        `UPDATE user SET
-        trash_uuid=?, appdata_uuid=?, desktop_uuid=?, documents_uuid=?, pictures_uuid=?, videos_uuid=?, public_uuid=?,
-        trash_id=?, appdata_id=?, desktop_id=?, documents_id=?, pictures_id=?, videos_id=?, public_id=?
-
-        WHERE id=?`,
-        [
-            trash_uuid, appdata_uuid, desktop_uuid, documents_uuid, pictures_uuid, videos_uuid, public_uuid,
-            trash_id, appdata_id, desktop_id, documents_id, pictures_id, videos_id, public_id,
-            user.id
-        ]
-    );
-    invalidate_cached_user(user);
-}
-
 function send_email_verification_code(email_confirm_code, email){
     const svc_email = Context.get('services').get('email');
     svc_email.send_email({ email }, 'email_verification_code', {
@@ -1905,7 +1616,6 @@ module.exports = {
     gen_public_token,
     get_taskbar_items,
     get_url_from_req,
-    generate_system_fsentries,
     generate_random_str,
     generate_random_username,
     get_app,
@@ -1926,7 +1636,6 @@ module.exports = {
     is_specifically_uuidv4,
     is_valid_url,
     jwt_auth,
-    mkdir,
     mv,
     number_format,
     refresh_apps_cache,

src/backend/src/modules/selfhosted/DefaultUserService.js

@@ -20,7 +20,7 @@ const { QuickMkdir } = require("../../filesystem/hl_operations/hl_mkdir");
 const { HLWrite } = require("../../filesystem/hl_operations/hl_write");
 const { NodePathSelector } = require("../../filesystem/node/selectors");
 const { surrounding_box } = require("../../fun/dev-console-ui-utils");
-const { get_user, generate_system_fsentries, invalidate_cached_user } = require("../../helpers");
+const { get_user, invalidate_cached_user } = require("../../helpers");
 const { Context } = require("../../util/context");
 const { asyncSafeSetInterval } = require("../../util/promise");
 const { buffer_to_stream } = require("../../util/streamutil");
@@ -165,7 +165,8 @@ class DefaultUserService extends BaseService {
             ],
         );
         user.password = password_hashed;
-        await generate_system_fsentries(user);
+        const svc_user = this.services.get('user');
+        await svc_user.generate_default_fsentries({ user });
         // generate default files for admin user
         const svc_fs = this.services.get('filesystem');
         const make_tree_ = async ({ components, tree }) => {

src/backend/src/routers/signup.js

@@ -345,7 +345,8 @@ module.exports = eggspress(['/signup'], {
         }
     }
 
-    await generate_system_fsentries(user);
+    const svc_user = Context.get('services').get('user');
+    await svc_user.generate_default_fsentries({ user });
 
     //set cookie
     res.cookie(config.cookie_name, token, {

src/backend/src/services/UserService.js

@@ -0,0 +1,111 @@
+const { LLMkdir } = require("../filesystem/ll_operations/ll_mkdir");
+const { RootNodeSelector } = require("../filesystem/node/selectors");
+const { invalidate_cached_user } = require("../helpers");
+const BaseService = require("./BaseService");
+const { DB_WRITE } = require("./database/consts");
+
+class UserService extends BaseService {
+    static MODULES = {
+        uuidv4: require('uuid').v4,
+    };
+
+    async _init () {
+        this.db = this.services.get('database').get(DB_WRITE, 'user-service');
+    }
+
+    // used to be called: generate_system_fsentries
+    async generate_default_fsentries ({ user }) {
+        
+        this.log.noticeme('YES THIS WAS USED');
+        
+        // Note: The comment below is outdated as we now do parallel writes for
+        //       all filesystem operations. However, there may still be some
+        //       performance hit so this requires further investigation.
+
+        // Normally, it is recommended to use mkdir() to create new folders,
+        // but during signup this could result in multiple queries to the DB server
+        // and for servers in remote regions such as Asia this could result in a
+        // very long time for /signup to finish, sometimes up to 30-40 seconds!
+        // by combining as many queries as we can into one and avoiding multiple back-and-forth
+        // with the DB server, we can speed this process up significantly.
+
+        const ts = Date.now()/1000;
+
+        // Generate UUIDs for all the default folders and files
+        const uuidv4 = this.modules.uuidv4;
+
+        let home_uuid = uuidv4();
+        let trash_uuid = uuidv4();
+        let appdata_uuid = uuidv4();
+        let desktop_uuid = uuidv4();
+        let documents_uuid = uuidv4();
+        let pictures_uuid = uuidv4();
+        let videos_uuid = uuidv4();
+        let public_uuid = uuidv4();
+
+        const insert_res = await this.db.write(
+            `INSERT INTO fsentries
+            (uuid, parent_uid, user_id, name, path, is_dir, created, modified, immutable) VALUES
+            (   ?,          ?,       ?,    ?,    ?,   true,       ?,        ?,      true),
+            (   ?,          ?,       ?,    ?,    ?,   true,       ?,        ?,      true),
+            (   ?,          ?,       ?,    ?,    ?,   true,       ?,        ?,      true),
+            (   ?,          ?,       ?,    ?,    ?,   true,       ?,        ?,      true),
+            (   ?,          ?,       ?,    ?,    ?,   true,       ?,        ?,      true),
+            (   ?,          ?,       ?,    ?,    ?,   true,       ?,        ?,      true),
+            (   ?,          ?,       ?,    ?,    ?,   true,       ?,        ?,      true),
+            (   ?,          ?,       ?,    ?,    ?,   true,       ?,        ?,      true)
+            `,
+            [
+                // Home
+                home_uuid, null, user.id, user.username, `/${user.username}`, ts, ts,
+                // Trash
+                trash_uuid, home_uuid, user.id, 'Trash', `/${user.username}/Trash`, ts, ts,
+                // AppData
+                appdata_uuid, home_uuid, user.id, 'AppData', `/${user.username}/AppData`, ts, ts,
+                // Desktop
+                desktop_uuid, home_uuid, user.id, 'Desktop', `/${user.username}/Desktop`, ts, ts,
+                // Documents
+                documents_uuid, home_uuid, user.id, 'Documents', `/${user.username}/Documents`, ts, ts,
+                // Pictures
+                pictures_uuid, home_uuid, user.id, 'Pictures', `/${user.username}/Pictures`, ts, ts,
+                // Videos
+                videos_uuid, home_uuid, user.id, 'Videos', `/${user.username}/Videos`, ts, ts,
+                // Public
+                public_uuid, home_uuid, user.id, 'Public', `/${user.username}/Public`, ts, ts,
+            ]
+        );
+
+        // https://stackoverflow.com/a/50103616
+        let trash_id = insert_res.insertId;
+        let appdata_id = insert_res.insertId + 1;
+        let desktop_id = insert_res.insertId + 2;
+        let documents_id = insert_res.insertId + 3;
+        let pictures_id = insert_res.insertId + 4;
+        let videos_id = insert_res.insertId + 5;
+        let public_id = insert_res.insertId + 6;
+
+        // Asynchronously set the user's system folders uuids in database
+        // This is for caching purposes, so we don't have to query the DB every time we need to access these folders
+        // This is also possible because we know the user's system folders uuids will never change
+
+        // TODO: pass to IIAFE manager to avoid unhandled promise rejection
+        // (IIAFE manager doesn't exist yet, hence this is a TODO)
+        this.db.write(
+            `UPDATE user SET
+            trash_uuid=?, appdata_uuid=?, desktop_uuid=?, documents_uuid=?, pictures_uuid=?, videos_uuid=?, public_uuid=?,
+            trash_id=?, appdata_id=?, desktop_id=?, documents_id=?, pictures_id=?, videos_id=?, public_id=?
+
+            WHERE id=?`,
+            [
+                trash_uuid, appdata_uuid, desktop_uuid, documents_uuid, pictures_uuid, videos_uuid, public_uuid,
+                trash_id, appdata_id, desktop_id, documents_id, pictures_id, videos_id, public_id,
+                user.id
+            ]
+        );
+        invalidate_cached_user(user);
+    }
+}
+
+module.exports = {
+    UserService,
+};