Feat/toolkit

requirements-dashboard.txt

@@ -1 +1 @@
-swanboard==0.1.8b1
+swanboard==0.1.9b1

requirements.txt

@@ -1,4 +1,3 @@
-swankit==0.2.4
 urllib3>=1.26.0
 requests>=2.28.0
 setuptools

swanlab/cli/commands/dashboard/watch.py

@@ -13,7 +13,8 @@
 
 import click
 
-from swanlab.env import get_swanlog_dir, SwanLabEnv
+from swanlab.env import SwanLabEnv
+from swanlab.env import get_swanlog_dir
 from swanlab.log import swanlog
 
 

swanlab/core_python/uploader/model.py

@@ -10,8 +10,7 @@
 from enum import Enum
 from typing import List, Optional, TypedDict, Literal
 
-from swanlab.data.modules import MediaBuffer
-from swanlab.toolkit import ColumnClass, ColumnConfig, LogContent
+from swanlab.toolkit import ColumnClass, ColumnConfig, LogContent, MediaBuffer
 
 
 class ColumnModel:

swanlab/data/callbacker/local.py

@@ -21,13 +21,13 @@
 try:
     # noinspection PyPackageRequirements
     import swanboard
-except ImportError:
+except ImportError as e:
     raise ImportError("Please install swanboard to use 'local' mode: pip install 'swanlab[dashboard]'")
 
 from importlib.metadata import version
 
 package_version = version("swanboard")
-if package_version != "0.1.8b1":
+if package_version != "0.1.9b1":
     raise ImportError(
         "Your swanboard version does not match, please use this command to install the matching version: pip install 'swanlab[dashboard]'"
     )

swanlab/data/porter/__init__.py

@@ -21,13 +21,15 @@
 from swanlab.core_python.uploader import ColumnModel, ScalarModel, MediaModel, LogModel
 from swanlab.core_python.uploader.thread import ThreadPool, UploadType
 from swanlab.data.store import RunStore, get_run_store, reset_run_store
+from swanlab.env import create_time
 from swanlab.error import ValidationError
 from swanlab.log.type import LogData
 from swanlab.proto.v0 import Log, Header, Project, Experiment, Column, Metric, BaseModel, Runtime, Footer, Media, Scalar
-from swanlab.toolkit import MetricInfo, ColumnInfo, RuntimeInfo, create_time, LogContent
+from swanlab.toolkit import MetricInfo, ColumnInfo, RuntimeInfo
 from .datastore import DataStore
 from .mounter import Mounter
 from .utils import filter_metric, filter_epoch, filter_column
+from ...toolkit.models.log import LogContent
 
 __all__ = ['DataPorter', 'Mounter']
 

swanlab/data/run/key.py

@@ -10,6 +10,7 @@
 from typing import Optional, Tuple
 
 from swanlab.data.modules import DataWrapper, Line
+from swanlab.env import create_time
 from swanlab.log import swanlog
 from swanlab.toolkit import (
     MetricInfo,
@@ -18,7 +19,6 @@
     ColumnClass,
     SectionType,
     ColumnConfig,
-    create_time,
     ParseErrorInfo,
     ChartType,
 )

swanlab/data/run/metadata/cooperation/__init__.py

@@ -8,9 +8,8 @@
 from typing import TypedDict, Optional
 
 from swanlab.core_python import get_client
-from swanlab.env import get_mode
+from swanlab.env import get_mode, get_swanlog_dir
 from swanlab.package import get_package_version
-from swanlab.toolkit import get_swanlog_dir
 from .qing_cloud import get_qing_cloud_info
 
 

swanlab/data/run/metadata/hardware/cpu.py

@@ -12,7 +12,7 @@
 import psutil
 
 from swanlab.data.run.metadata.hardware.type import HardwareFuncResult, HardwareCollector, HardwareInfoList
-from swanlab.toolkit import is_macos
+from swanlab.env import is_macos
 from .utils import CpuBaseCollector as C
 
 

swanlab/data/run/metadata/hardware/memory.py

@@ -9,7 +9,7 @@
 
 import psutil
 
-from swanlab.toolkit import is_macos
+from swanlab.env import is_macos
 from .type import HardwareFuncResult, HardwareCollector, HardwareInfo
 from .utils import MemoryBaseCollector as M
 

swanlab/data/run/metadata/hardware/soc/apple.py

@@ -11,7 +11,7 @@
 
 import psutil
 
-from swanlab.toolkit import is_macos
+from swanlab.env import is_macos
 from ..type import HardwareFuncResult, HardwareInfoList, HardwareCollector as H
 from ..utils import CpuBaseCollector as C, MemoryBaseCollector as M
 

swanlab/data/utils.py

@@ -13,12 +13,12 @@
 from swanlab.core_python import auth
 from swanlab.data.run import SwanLabRun
 from swanlab.data.run.helper import SwanLabRunOperator
-from swanlab.env import is_interactive, SwanLabEnv
+from swanlab.env import is_interactive, SwanLabEnv, SwanLabMode
 from swanlab.error import KeyFileError
 from swanlab.formatter import check_load_json_yaml
 from swanlab.log import swanlog
 from swanlab.package import get_key, get_host_web
-from swanlab.toolkit import SwanKitCallback, SwanLabMode
+from swanlab.toolkit import SwanKitCallback
 
 
 def should_call_before_init(text):

swanlab/env.py

@@ -7,20 +7,37 @@
     swanlab全局共用环境变量(运行时环境变量)
     除了utils和error模块，其他模块都可以使用这个模块
 """
+import datetime
 import enum
 import io
 import netrc
 import os
 import re
 import sys
+from enum import Enum
 from pathlib import Path
+from platform import platform
 from typing import List, Union
 from urllib.parse import urlparse
 
-import swanlab.toolkit as E
 
+class SwanLabMode(Enum):
+    """
+    swanlab的解析模式，枚举类
+    """
 
-# ---------------------------------- 环境变量枚举类 ----------------------------------
+    DISABLED = "disabled"
+    CLOUD = "cloud"
+    OFFLINE = "offline"
+    LOCAL = "local"
+
+    @classmethod
+    def list(cls) -> List[str]:
+        """
+        获取所有的枚举值
+        :return: 所有的枚举值
+        """
+        return [item.value for item in cls]
 
 
 class SwanLabEnv(enum.Enum):
@@ -175,15 +192,115 @@ def list(cls) -> List[str]:
         return [item.value for item in cls]
 
 
-# ---------------------------------- API ----------------------------------
+def create_time() -> str:
+    """获取当前时间(UTC时区)"""
+    return datetime.datetime.now(datetime.timezone.utc).isoformat()
+
+
+def is_windows() -> bool:
+    """判断当前操作系统是否是windows还是类unix系统，主要是路径分隔上的差别
+    此外的系统会报错为 UnKnownSystemError
+    :raise OSError: 未知系统错误，此时swanlab运行在未知系统上，这个系统不是windows或者类unix系统
+    :return: True表示是windows系统，False表示是类unix系统
+    """
+    if sys.platform.startswith("win"):
+        return True
+    elif sys.platform.startswith("linux") or sys.platform.startswith("darwin"):
+        return False
+    raise OSError("Unknown system, not windows or unix-like system")
+
+
+def is_macos() -> bool:
+    """判断当前操作系统是否是macos
+    :return: True表示是macos系统，False表示不是macos系统
+    """
+    return "mac" in platform().lower()
+
+
+def get_mode() -> str:
+    """
+    获取当前的swanlab解析模式，如果没有设置，默认为cloud
+    :raise ValueError: 未知的swanlab模式
+    :return: swanlab的解析模式
+    """
+    mode = os.getenv(SwanLabEnv.MODE.value)
+    if mode is None:
+        mode = SwanLabMode.CLOUD.value
+    mode = mode.lower()
+    if mode not in SwanLabMode.list():
+        raise ValueError(f"Unknown swanlab mode: {mode}")
+    return mode
+
 
-is_windows = E.is_windows
+def get_save_dir() -> str:
+    """
+    获取存放swanlab全局文件的文件夹路径，如果不存在就创建
+    此函数对应为SWANLAB_SAVE_FOLDER全局变量，如果没有设置，默认为用户主目录下的.swanlab文件夹
+    执行此函数时，如果文件夹不存在，自动创建，但是出于安全考虑，不会自动创建父文件夹
+    :raises
+        :raise FileNotFoundError: folder的父目录不存在
+        :raise NotADirectoryError: folder不是一个文件夹
+    :return: swanlab全局文件夹保存的路径，返回处理后的绝对路径
+    """
+    folder = os.getenv(SwanLabEnv.SWANLAB_FOLDER.value)
+    if folder is None:
+        folder = os.path.join(os.path.expanduser("~"), ".swanlab")
+    folder = os.path.abspath(folder)
+    if not os.path.exists(os.path.dirname(folder)):
+        raise FileNotFoundError(f"{os.path.dirname(folder)} not found")
+    if not os.path.exists(folder):
+        # 只创建当前文件夹，不创建父文件夹，之所以还要捕捉 FileExistsError，是因为在多线程或多进程环境下，可能会有多个线程或进程同时创建同一个文件夹
+        # 比如：https://github.com/SwanHubX/SwanLab/issues/1033
+        try:
+            os.mkdir(folder)
+        except FileExistsError:
+            pass
+    if not os.path.isdir(folder):
+        raise NotADirectoryError(f"{folder} is not a directory")
+    return folder
 
-get_mode = E.get_mode
 
-get_swanlog_dir = E.get_swanlog_dir
+def get_swanlog_dir() -> str:
+    """
+    获取存放swanlog日志文件的文件夹路径
+    此函数对应为SWANLAB_LOG_FOLDER全局变量，如果没有设置，默认为当前运行目录下的swanlog文件夹
+    需要注意，此函数并不会保证文件夹的存在，但是会检查父文件夹是否存在以及folder是否是一个文件夹
+    :raises
+        :raise FileNotFoundError: folder的父目录不存在
+        :raise NotADirectoryError: folder不是一个文件夹
+    :return: swanlog日志文件保存的路径，返回处理后的绝对路径
+    """
+    folder = os.getenv(SwanLabEnv.SWANLOG_FOLDER.value)
+    if folder is None:
+        folder = os.path.join(os.getcwd(), "swanlog")
+    folder = os.path.abspath(folder)
+    if not os.path.exists(os.path.dirname(folder)):
+        raise FileNotFoundError(f"{os.path.dirname(folder)} not found")
+    if not os.path.exists(folder):
+        return folder
+    if not os.path.isdir(folder):
+        raise NotADirectoryError(f"{folder} is not a directory")
+    return folder
 
-get_save_dir = E.get_save_dir
+
+def create_swanlog_dir(logdir: Union[Path, str] = None):
+    """
+    获取swanlog文件夹，如果文件夹不存在则创建
+    :param logdir: swanlog文件夹路径，默认为当前工作目录下的swanlog文件夹
+    """
+    if logdir is None:
+        logdir = get_swanlog_dir()
+    try:
+        os.makedirs(logdir, exist_ok=True)
+        if not os.access(logdir, os.W_OK):
+            raise IOError(f"no write permission for path: {logdir}")
+    except Exception as error:
+        raise IOError(f"Failed to create or access logdir: {logdir}, error: {error}")
+    # 如果logdir是空的，创建.gitignore文件，写入*
+    if not os.listdir(logdir):
+        with open(os.path.join(logdir, ".gitignore"), "w", encoding="utf-8") as f:
+            f.write("*")
+    return logdir
 
 
 def in_jupyter() -> bool:
@@ -227,23 +344,3 @@ def remove_host_suffix(host: str, suffix: str) -> str:
     if host.endswith(suffix):
         return host[: -len(suffix)]
     return host
-
-
-def create_swanlog_dir(logdir: Union[Path, str] = None):
-    """
-    获取swanlog文件夹，如果文件夹不存在则创建
-    :param logdir: swanlog文件夹路径，默认为当前工作目录下的swanlog文件夹
-    """
-    if logdir is None:
-        logdir = get_swanlog_dir()
-    try:
-        os.makedirs(logdir, exist_ok=True)
-        if not os.access(logdir, os.W_OK):
-            raise IOError(f"no write permission for path: {logdir}")
-    except Exception as error:
-        raise IOError(f"Failed to create or access logdir: {logdir}, error: {error}")
-    # 如果logdir是空的，创建.gitignore文件，写入*
-    if not os.listdir(logdir):
-        with open(os.path.join(logdir, ".gitignore"), "w", encoding="utf-8") as f:
-            f.write("*")
-    return logdir

swanlab/log/log.py

@@ -9,9 +9,11 @@
 import sys
 from typing import List, Tuple, Callable
 
-from swanlab.toolkit import SwanKitLogger, create_time, LogContent
+from swanlab.env import create_time
+from swanlab.toolkit import SwanKitLogger
 from .counter import AtomicCounter
 from .type import LogHandler, LogType, WriteHandler, LogData, ProxyType
+from ..toolkit.models.log import LogContent
 
 
 class SwanLog(SwanKitLogger):

swanlab/log/type.py

@@ -7,7 +7,7 @@
 
 from typing import TypedDict, Literal, Callable, List, Any
 
-from swanlab.toolkit import LogContent
+from swanlab.toolkit.models.log import LogContent
 
 # 支持的代理类型
 ProxyType = Literal['all', 'stdout', 'stderr', 'none']

swanlab/proto/v0.py

@@ -4,6 +4,7 @@
 @time: 2025/6/20 13:30
 @description: 历史版本的备份、上传协议采用 JSON 序列化实现
 为了保证向下兼容性，在此保留相关模型定义
+本质上是增加了一层中间层，此中间层的作用是实现 swanlab.toolkit 中模型与 swanlab.core_python.uploader 中模型的相互转换
 """
 
 import json
@@ -25,8 +26,8 @@
     YRange,
     ChartReference,
     MediaBuffer,
-    LogContent,
 )
+from swanlab.toolkit.models.log import LogContent
 
 
 class BaseModel(PydanticBaseModel):
@@ -168,7 +169,6 @@ def to_file_model(self, file_dir) -> FileModel:
         return FileModel(conda=conda, requirements=requirements, metadata=metadata, config=config)
 
 
-
 class Column(BaseModel):
 
     key: str  # 列的唯一标识符

swanlab/proto/v1.py

@@ -3,4 +3,6 @@
 @file: v1.py
 @time: 2025/6/20 15:16
 @description: swanlab data transfer protocol version 1
+基于protobuf的数据模型将自动实现数据从python到go的转换，由于我们的数据在最开始基于swanlab.toolkit中的模型定义，因此我们需要在这里实现从toolkit模型到protobuf模型的转换
+仅需几个函数实现字段映射即可
 """

swanlab/swanlab_settings.py

@@ -11,7 +11,7 @@
 import os
 from pathlib import Path
 
-from swanlab.toolkit import is_windows, SwanLabMode
+from swanlab.env import is_windows, SwanLabMode
 
 try:
     from typing import Annotated, Literal, Optional, Any, Union  # Python 3.9+