Refactor/exp data

.eslintrc-auto-import.json

@@ -69,4 +69,4 @@
     "watchPostEffect": true,
     "watchSyncEffect": true
   }
-}
+}

swanlab/data/run/exp.py

@@ -211,19 +211,19 @@ def add(self, data: DataType, step: int = None):
             self.__data = self.__new_tags()
         # 添加数据
         data = data if not is_nan else "NaN"
-        self.__data["data"].append(self.__new_tag(step, data, more=more))
+        new_data = self.__new_tag(step, data, more=more)
+        self.__data["data"].append(new_data)
         # 优化文件分片，每__slice_size个tag数据保存为一个文件，通过sum来判断
         sum = len(self.__steps)
         mu = math.ceil(sum / self.__slice_size)
         # 存储路径
-        file_path = os.path.join(self.save_path, str(mu * self.__slice_size) + ".json")
-        # 方便一些，直接使用w+模式覆盖写入
-        with get_a_lock(file_path, mode="w+") as f:
-            ujson.dump(self.__data, f, ensure_ascii=False)
+        file_path = os.path.join(self.save_path, str(mu * self.__slice_size) + ".log")
         # 更新实验信息总结
         with get_a_lock(os.path.join(self.save_path, "_summary.json"), "w+") as f:
             ujson.dump(self._summary, f, ensure_ascii=False)
-
+        # 保存数据
+        with open(file_path, "a") as f:
+            f.write(ujson.dumps(new_data, ensure_ascii=False) + "\n")
         return step
 
     @property

swanlab/server/controller/experiment.py

@@ -27,16 +27,15 @@
 from ...utils.time import create_time
 import yaml
 from ...log import swanlog
-from typing import List, Dict
-from .db import connect, NotExistedError
 from .db import (
     Project,
     Experiment,
     Chart,
-    Namespace,
     Tag,
+    connect,
+    NotExistedError,
 )
-from .utils import get_exp_charts, clear_field
+from .utils import get_exp_charts, clear_field, read_tag_data, get_tag_files, LOGS_CONFIGS
 
 __to_list = Experiment.search2list
 
@@ -47,13 +46,6 @@
 DEFAULT_PROJECT_ID = Project.DEFAULT_PROJECT_ID
 # 实验运行状态
 RUNNING_STATUS = Experiment.RUNNING_STATUS
-
-# tag 总结文件名
-TAG_SUMMARY_FILE = "_summary.json"
-# logs 目录下的配置文件
-LOGS_CONFIGS = [TAG_SUMMARY_FILE]
-
-
 # ---------------------------------- 工具函数 ----------------------------------
 
 
@@ -180,46 +172,15 @@ def get_tag_data(experiment_id: int, tag: str) -> dict:
     if not os.path.exists(tag_path):
         return NOT_FOUND_404("tag not found")
     # 获取目录下存储的所有数据
-    # 降序排列，最新的数据在最前面
-    files: list = os.listdir(tag_path)
-    # files中去除_summary.json文件
-    files = [f for f in files if not f.endswith("_summary.json")]
-    if len(files) == 0:
-        return []
-    files.sort()
     tag_data: list = []
-    # 最后一个文件代表当前数据量
-    last_file = files[-1]
-    tag_json = None
-    # ---------------------------------- 开始读取最后一个文件 ----------------------------------
-    # 如果最后一个文件内容为空
-    if os.path.getsize(os.path.join(tag_path, last_file)) == 0:
-        tag_json = {"data": []}
-        count = 0
-    else:
-        # 锁住此文件，不再允许其他进程访问，换句话说，实验日志在log的时候被阻塞
-        with get_a_lock(os.path.join(tag_path, last_file), mode="r") as f:
-            # 读取数据
-            tag_json = ujson.load(f)
-            # 倒数第二个文件+当前文件的数据量等于总数据量
-            # 倒数第二个文件可能不存在
-            count = files[-2].split(".")[0] if len(files) > 1 else 0
-            count = int(count) + len(tag_json["data"])
-    # 读取完毕，文件解锁
+    # ---------------------------------- 读取文件数据 ----------------------------------
+    current_logs = get_tag_files(tag_path, LOGS_CONFIGS)
+    for file in current_logs:
+        tag_data = tag_data + read_tag_data(os.path.join(tag_path, file))
     # ---------------------------------- 返回所有数据 ----------------------------------
-    # FIXME: 性能问题
-    # 读取所有数据
-    # tag_json是最后一个文件的数据
-    # 按顺序读取其他文件的数据
-    tag_data_list: List[List[Dict]] = []
-    for path in files[:-1]:
-        # 读取tag数据，由于目前在设计上这些文件不会再被修改，所以不需要加锁
-        with open(os.path.join(tag_path, path), "r") as f:
-            tag_data_list.append(ujson.load(f)["data"])
-    # 将数据合并
-    for data in tag_data_list:
-        tag_data.extend(data)
-    tag_data.extend(tag_json["data"])
+    # 如果数据为空，返回空列表
+    if len(tag_data) == 0:
+        return SUCCESS_200(data={"sum": 0, "list": [], "experiment_id": experiment_id})
     # COMPAT 如果第一个数据没有index，就循环每个数据，加上index
     if tag_data[0].get("index") is None:
         for index, data in enumerate(tag_data):
@@ -318,22 +279,23 @@ def get_experiment_summary(experiment_id: int) -> dict:
             summaries.append({"key": tag, "value": "TypeError"})
             continue
         tag_path = os.path.join(experiment_path, quote(tag, safe=""))
-        logs = sorted([item for item in os.listdir(tag_path) if not item in LOGS_CONFIGS])
+        # 获取 tag 目录下的所有存储的日志文件
+        logs = get_tag_files(tag_path, LOGS_CONFIGS)
         # 打开 tag 目录下最后一个存储文件，获取最后一条数据
-        with get_a_lock(os.path.join(tag_path, logs[-1]), mode="r") as f:
-            data = ujson.load(f)
-            # str 转化的目的是为了防止有些不合规范的数据导致返回体对象化失败
-            data = str(data["data"][-1]["data"])
-            summaries.append({"key": tag, "value": data})
+        with open(os.path.join(tag_path, logs[-1])) as f:
+            lines = f.readlines()
+            # 最后一行数据，如果为空，取倒数第二行
+            data = lines[-1] if len(lines[-1]) else lines[-2]
+            last_data = ujson.loads(data)
+            summaries.append({"key": tag, "value": last_data["data"]})
     # 获取数据库记录时在实验下的排序
     sorts = {item["name"]: item["sort"] for item in __to_list(experiment.tags)}
-    # 如果 sorts 中的值不都为 0，说明是新版添加排序后的 tag，这里进行排序 (如果是旧版没有排序的tag，直接按照数据库顺序即可)
+    # COMPAT 如果 sorts 中的值不都为 0，说明是新版添加排序后的 tag，这里进行排序 (如果是旧版没有排序的tag，直接按照数据库顺序即可)
     if not all(value == 0 for value in sorts.values()):
         temp = [0] * len(summaries)
         for item in summaries:
             temp[sorts[item["key"]]] = item
         summaries = temp
-
     return SUCCESS_200({"summaries": summaries})
 
 

swanlab/server/controller/project.py

@@ -21,6 +21,7 @@
 from ...db.utils.chart import transform_to_multi_exp_charts
 from .db import NotExistedError
 from .utils import get_proj_charts
+from ...log import swanlog
 
 # 自定义响应
 from ..module.resp import (
@@ -45,6 +46,7 @@
     Chart,
     Namespace,
 )
+from .utils import get_tag_files, LOGS_CONFIGS
 
 # 将查询结果对象转为列表
 __to_list = Project.search2list
@@ -160,14 +162,16 @@ def get_project_summary(project_id: int = DEFAULT_PROJECT_ID) -> dict:
             if not os.path.exists(tag_path):
                 experiment_summaries[tag] = "TypeError"
                 continue
-            logs = sorted([item for item in os.listdir(tag_path) if item != "_summary.json"])
+            # 获取所有 tag 文件
+            logs = get_tag_files(tag_path, exclude=LOGS_CONFIGS)
+            # 打开最后一个文件，获取最后一行数据
             with open(os.path.join(tag_path, logs[-1]), mode="r") as f:
                 try:
-                    tag_data = ujson.load(f)
-                    # str 转化的目的是为了防止有些不合规范的数据导致返回体对象化失败
-                    experiment_summaries[tag] = str(tag_data["data"][-1]["data"])
+                    lines = f.readlines()
+                    tag_data = ujson.loads(lines[-1])
+                    experiment_summaries[tag] = tag_data["data"]
                 except Exception as e:
-                    print(f"[expr: {expr['name']} - {tag}] --- {e}")
+                    swanlog.error(f"[expr: {expr['name']} - {tag}] --- {e}")
                     continue
 
         data[expr["name"]] = experiment_summaries

swanlab/server/controller/utils/__init__.py

@@ -8,6 +8,7 @@
     共享工具函数
 """
 from .charts import get_exp_charts, get_proj_charts
+from .tag import read_tag_data, get_tag_files, LOGS_CONFIGS
 from typing import List
 
 

swanlab/server/controller/utils/tag.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+r"""
+@DATE: 2024-03-17 15:13:31
+@File: swanlab/server/controller/utils/tag.py
+@IDE: vscode
+@Description:
+    tag相关处理函数
+"""
+from typing import List
+import json
+import ujson
+import os
+
+# tag 总结文件名
+TAG_SUMMARY_FILE = "_summary.json"
+# logs 目录下的配置文件
+LOGS_CONFIGS = [TAG_SUMMARY_FILE]
+
+
+def read_tag_data(file_path: str) -> List[dict]:
+    """读取某一个tag文件数据，数据依据其中的index字段排序
+
+    Parameters
+    ----------
+    file_path : str
+        tag文件路径，绝对路径
+    """
+    # 如果文件内容为空，返回空列表
+    if os.path.getsize(file_path) == 0:
+        return []
+    # 读取文件内容，文件内部本质上是字符串一堆json格式的数据，每一行是一个json并且有换行符分隔，需要拿到然后解析为list<dict>
+    data = []
+    with open(file_path, "r") as f:
+        lines = f.readlines()
+        # 解析为list<dict>
+        for i in range(len(lines)):
+            if len(lines[i]):
+                data.append(json.loads(lines[i]))
+        return data
+
+
+def get_tag_files(tag_path: str, exclude: List[str] = []) -> List[str]:
+    """
+    获取实验数据，并且做向下兼容，在v0.2.4版本以前的实验日志数据将转换为新的格式
+
+    Parameters
+    ----------
+    file_path : str
+        tag的路径
+    exclude : List[str]
+        需要排除的文件列表
+
+    Returns
+    -------
+    List[str]
+        tag文件列表
+    """
+    # 降序排列，最新的数据在最前面
+    files: list = os.listdir(tag_path)
+    previous_logs = [f for f in files if f.endswith(".json") and f not in exclude]
+    current_logs = [f for f in files if f.endswith(".log")]
+    current_logs.sort()
+    # COMPAT 如果目标文件夹不存在*.log文件但存在*.json文件，说明是之前的日志格式，需要转换
+    if len(current_logs) == 0 and len(previous_logs) > 0:
+        for file in previous_logs:
+            with open(os.path.join(tag_path, file), "r") as f:
+                data = ujson.load(f)
+                with open(os.path.join(tag_path, file.replace(".json", ".log")), "a") as f:
+                    for d in data["data"]:
+                        f.write(ujson.dumps(d) + "\n")
+        current_logs = [f for f in os.listdir(tag_path) if f.endswith(".log")]
+        current_logs.sort()
+    return current_logs

vue/src/charts/modules/ImageModule.vue

@@ -0,0 +1,55 @@
+<template>
+  <div class="image-detail">
+    <div class="text-xs flex items-center justify-center pb-1 w-full" :title="name" v-if="name">
+      <div class="h-2 w-2 rounded-full shrink-0" :style="{ backgroundColor: color }"></div>
+      <p class="pl-1 truncate max-w-full">{{ name }}</p>
+    </div>
+    <div class="image-container" @click="$emit('zoom', filename, index)">
+      <img :src="imagesData" />
+      <DownloadButton class="image-download-button" @click.stop="$emit('download', filename)" />
+    </div>
+    <p class="text-xs w-full text-center truncate" v-if="caption">{{ caption }}</p>
+  </div>
+</template>
+
+<script setup>
+/**
+ * @description: 图像模块，负责单个图像显示和暴露下载事件
+ * @file: ImageModule.vue
+ * @since: 2024-03-17 22:11:10
+ **/
+import DownloadButton from '../components/DownloadButton.vue'
+defineProps({
+  filename: String,
+  imagesData: String,
+  // 图像描述
+  caption: String,
+  index: Number,
+  // 实验名称
+  name: String,
+  color: String,
+  // 是否为多实验
+  multi: Boolean
+})
+
+defineEmits(['zoom', 'download'])
+</script>
+
+<style lang="scss" scoped>
+.image-detail {
+  @apply relative h-full w-full;
+  max-height: 13rem;
+
+  .image-container {
+    @apply relative w-full grow inline-flex justify-center cursor-pointer;
+    height: calc(100% - 2.5rem);
+    img {
+      @apply my-auto object-contain;
+      height: 100%;
+    }
+    &:hover .image-download-button {
+      @apply block;
+    }
+  }
+}
+</style>