Merge pull request #2233 from zauberzeug/utf-8

Migrate storage from system's default encoding to UTF-8

nicegui/storage.py

@@ -42,10 +42,11 @@ class ReadOnlyDict(MutableMapping):
 
 class PersistentDict(observables.ObservableDict):
 
-    def __init__(self, filepath: Path) -> None:
+    def __init__(self, filepath: Path, encoding: Optional[str] = None) -> None:
         self.filepath = filepath
+        self.encoding = encoding
         try:
-            data = json.loads(filepath.read_text()) if filepath.exists() else {}
+            data = json.loads(filepath.read_text(encoding)) if filepath.exists() else {}
         except Exception:
             log.warning(f'Could not load storage file {filepath}')
             data = {}
@@ -59,7 +60,7 @@ class PersistentDict(observables.ObservableDict):
             self.filepath.parent.mkdir(exist_ok=True)
 
         async def backup() -> None:
-            async with aiofiles.open(self.filepath, 'w') as f:
+            async with aiofiles.open(self.filepath, 'w', encoding=self.encoding) as f:
                 await f.write(json.dumps(self))
         if core.loop:
             background_tasks.create_lazy(backup(), name=self.filepath.stem)
@@ -93,7 +94,8 @@ class Storage:
 
     def __init__(self) -> None:
         self.path = Path(os.environ.get('NICEGUI_STORAGE_PATH', '.nicegui')).resolve()
-        self._general = PersistentDict(self.path / 'storage_general.json')
+        self.migrate_to_utf8()
+        self._general = PersistentDict(self.path / 'storage-general.json', encoding='utf-8')
         self._users: Dict[str, PersistentDict] = {}
 
     @property
@@ -132,7 +134,7 @@ class Storage:
             raise RuntimeError('app.storage.user needs a storage_secret passed in ui.run()')
         session_id = request.session['id']
         if session_id not in self._users:
-            self._users[session_id] = PersistentDict(self.path / f'storage_user_{session_id}.json')
+            self._users[session_id] = PersistentDict(self.path / f'storage-user-{session_id}.json', encoding='utf-8')
         return self._users[session_id]
 
     @property
@@ -144,5 +146,16 @@ class Storage:
         """Clears all storage."""
         self._general.clear()
         self._users.clear()
-        for filepath in self.path.glob('storage_*.json'):
+        for filepath in self.path.glob('storage-*.json'):
             filepath.unlink()
+
+    def migrate_to_utf8(self) -> None:
+        """Migrates storage files from system's default encoding to UTF-8.
+
+        To distinguish between the old and new encoding, the new files are named with dashes instead of underscores.
+        """
+        for filepath in self.path.glob('storage_*.json'):
+            new_filepath = filepath.with_stem(filepath.stem.replace('_', '-'))
+            data = json.loads(filepath.read_text())
+            filepath.rename(new_filepath)
+            new_filepath.write_text(json.dumps(data), encoding='utf-8')

tests/test_storage.py

@@ -91,7 +91,7 @@ async def test_access_user_storage_from_fastapi(screen: Screen):
         assert response.status_code == 200
         assert response.text == '"OK"'
         await asyncio.sleep(0.5)  # wait for storage to be written
-        assert next(Path('.nicegui').glob('storage_user_*.json')).read_text() == '{"msg":"yes"}'
+        assert next(Path('.nicegui').glob('storage-user-*.json')).read_text('utf-8') == '{"msg":"yes"}'
 
 
 def test_access_user_storage_on_interaction(screen: Screen):
@@ -105,7 +105,7 @@ def test_access_user_storage_on_interaction(screen: Screen):
     screen.open('/')
     screen.click('switch')
     screen.wait(0.5)
-    assert next(Path('.nicegui').glob('storage_user_*.json')).read_text() == '{"test_switch":true}'
+    assert next(Path('.nicegui').glob('storage-user-*.json')).read_text('utf-8') == '{"test_switch":true}'
 
 
 def test_access_user_storage_from_button_click_handler(screen: Screen):
@@ -117,7 +117,7 @@ def test_access_user_storage_from_button_click_handler(screen: Screen):
     screen.open('/')
     screen.click('test')
     screen.wait(1)
-    assert next(Path('.nicegui').glob('storage_user_*.json')).read_text() == '{"inner_function":"works"}'
+    assert next(Path('.nicegui').glob('storage-user-*.json')).read_text('utf-8') == '{"inner_function":"works"}'
 
 
 async def test_access_user_storage_from_background_task(screen: Screen):
@@ -130,7 +130,7 @@ async def test_access_user_storage_from_background_task(screen: Screen):
 
     screen.ui_run_kwargs['storage_secret'] = 'just a test'
     screen.open('/')
-    assert next(Path('.nicegui').glob('storage_user_*.json')).read_text() == '{"subtask":"works"}'
+    assert next(Path('.nicegui').glob('storage-user-*.json')).read_text('utf-8') == '{"subtask":"works"}'
 
 
 def test_user_and_general_storage_is_persisted(screen: Screen):
@@ -166,4 +166,4 @@ def test_rapid_storage(screen: Screen):
     screen.open('/')
     screen.click('test')
     screen.wait(0.5)
-    assert Path('.nicegui', 'storage_general.json').read_text() == '{"one":1,"two":2,"three":3}'
+    assert Path('.nicegui', 'storage-general.json').read_text('utf-8') == '{"one":1,"two":2,"three":3}'