feat: support signoz (labring#5218)

* feat: support signoz

* chore: restructure otel

Author: FinleyGe

packages/service/common/otel/const.ts

@@ -0,0 +1,2 @@
+export const SignozBaseURL = process.env.SIGNOZ_BASE_URL;
+export const SignozServiceName = process.env.SIGNOZ_SERVICE_NAME || 'fastgpt';

packages/service/common/otel/log/index.ts

@@ -0,0 +1,35 @@
+import { LoggerProvider, SimpleLogRecordProcessor } from '@opentelemetry/sdk-logs';
+import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http';
+import { SignozBaseURL, SignozServiceName } from '../const';
+import { resourceFromAttributes } from '@opentelemetry/resources';
+import type { Logger } from '@opentelemetry/api-logs';
+
+export const getLogger = () => {
+  if (!global.logger) {
+    if (!SignozBaseURL) {
+      return null;
+    }
+    const otlpExporter = new OTLPLogExporter({
+      url: `${SignozBaseURL}/v1/logs`,
+      headers: {
+        'Content-Type': 'application/json'
+      }
+    });
+
+    const loggerProvider = new LoggerProvider({
+      processors: [new SimpleLogRecordProcessor(otlpExporter)],
+      resource: resourceFromAttributes({
+        'service.name': SignozServiceName
+      })
+    });
+
+    // logsAPI.logs.setGlobalLoggerProvider(loggerProvider);
+    global.logger = loggerProvider.getLogger('default');
+  }
+
+  return global.logger;
+};
+
+declare global {
+  var logger: Logger;
+}

packages/service/common/otel/trace/register.ts

@@ -0,0 +1,20 @@
+import { registerOTel, OTLPHttpJsonTraceExporter } from '@vercel/otel';
+// Add otel logging
+// import { diag, DiagConsoleLogger, DiagLogLevel } from '@opentelemetry/api';
+import { SignozBaseURL, SignozServiceName } from '../const';
+import { addLog } from '../../system/log';
+// diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.INFO);
+
+export function connectSignoz() {
+  if (!SignozBaseURL) {
+    addLog.warn('Signoz is not configured');
+    return;
+  }
+  addLog.info(`Connecting signoz, ${SignozBaseURL}, ${SignozServiceName}`);
+  return registerOTel({
+    serviceName: SignozServiceName,
+    traceExporter: new OTLPHttpJsonTraceExporter({
+      url: `${SignozBaseURL}/v1/traces`
+    })
+  });
+}

packages/service/common/system/log.ts

@@ -3,13 +3,16 @@ import chalk from 'chalk';
 import { LogLevelEnum } from './log/constant';
 import { connectionMongo } from '../mongo/index';
 import { getMongoLog } from './log/schema';
+import { getLogger } from '../otel/log';
 
 export enum EventTypeEnum {
   outLinkBot = '[Outlink bot]',
   feishuBot = '[Feishu bot]',
   wxOffiaccount = '[Offiaccount bot]'
 }
 
+const logger = getLogger();
+
 const logMap = {
   [LogLevelEnum.debug]: {
     levelLog: chalk.green('[Debug]')
@@ -57,6 +60,17 @@ export const addLog = {
 
     level === LogLevelEnum.error && console.error(obj);
 
+    if (logger) {
+      logger.emit({
+        severityNumber: level.valueOf(),
+        severityText: ['debug', 'info', 'warn', 'error'][level],
+        body: {
+          msg,
+          obj
+        }
+      });
+    }
+
     // store log
     if (level >= STORE_LOG_LEVEL && connectionMongo.connection.readyState === 1) {
       (async () => {

packages/service/package.json

@@ -7,6 +7,13 @@
     "@fastgpt/global": "workspace:*",
     "@modelcontextprotocol/sdk": "^1.12.1",
     "@node-rs/jieba": "2.0.1",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/api-logs": "^0.203.0",
+    "@opentelemetry/exporter-logs-otlp-http": "^0.203.0",
+    "@opentelemetry/resources": "^2.0.1",
+    "@opentelemetry/sdk-logs": "^0.203.0",
+    "@opentelemetry/winston-transport": "^0.14.0",
+    "@vercel/otel": "^1.13.0",
     "@xmldom/xmldom": "^0.8.10",
     "@zilliz/milvus2-sdk-node": "2.4.2",
     "axios": "^1.8.2",
@@ -39,10 +46,13 @@
     "papaparse": "5.4.1",
     "pdfjs-dist": "4.10.38",
     "pg": "^8.10.0",
+    "pino": "^9.7.0",
+    "pino-opentelemetry-transport": "^1.0.1",
     "request-ip": "^3.3.0",
     "tiktoken": "1.0.17",
     "tunnel": "^0.0.6",
-    "turndown": "^7.1.2"
+    "turndown": "^7.1.2",
+    "winston": "^3.17.0"
   },
   "devDependencies": {
     "@types/cookie": "^0.5.2",

pnpm-lock.yaml

@@ -88,3 +88,6 @@ CONFIG_JSON_PATH=
 # # 日志来源ID前缀
 # CHAT_LOG_SOURCE_ID_PREFIX=fastgpt-
 
+# Signoz
+SIGNOZ_BASE_URL=
+SIGNOZ_SERVICE_NAME=

projects/app/.env.template

@@ -18,7 +18,8 @@ export async function register() {
         { startCron },
         { startTrainingQueue },
         { preLoadWorker },
-        { loadSystemModels }
+        { loadSystemModels },
+        { connectSignoz }
       ] = await Promise.all([
         import('@fastgpt/service/common/mongo/init'),
         import('@fastgpt/service/common/mongo/index'),
@@ -30,9 +31,13 @@ export async function register() {
         import('@/service/common/system/cron'),
         import('@/service/core/dataset/training/utils'),
         import('@fastgpt/service/worker/preload'),
-        import('@fastgpt/service/core/ai/config/utils')
+        import('@fastgpt/service/core/ai/config/utils'),
+        import('@fastgpt/service/common/otel/trace/register')
       ]);
 
+      // connect to signoz
+      connectSignoz();
+
       // 执行初始化流程
       systemStartCb();
       initGlobalVariables();