feat(backend): clone-mysql-priv-in-actor #12601

dbm-services/go.work.sum

@@ -3536,6 +3536,7 @@ golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p
 golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
 golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8=
 golang.org/x/exp v0.0.0-20231108232855-2478ac86f678/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
+golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
 golang.org/x/image v0.0.0-20210216034530-4410531fe030 h1:lP9pYkih3DUSC641giIXa2XqfTIbbbRr0w2EOTA7wHA=
 golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
 golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=

dbm-services/mysql/db-tools/dbactuator/go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/BurntSushi/toml v1.3.2
 	github.com/IBM/sarama v1.42.1
 	github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2
+	github.com/antlr4-go/antlr/v4 v4.13.1
 	github.com/cloudfoundry/gosigar v1.3.59
 	github.com/dustin/go-humanize v1.0.1
 	github.com/github/gh-ost v1.1.6

dbm-services/mysql/db-tools/dbactuator/go.sum

@@ -15,6 +15,8 @@ github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 h1:ZBbLwSJqkH
 github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2/go.mod h1:VSw57q4QFiWDbRnjdX8Cb3Ow0SFncRw+bA/ofY6Q83w=
 github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo=
 github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
+github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
+github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw=
 github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
 github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=

dbm-services/mysql/db-tools/dbactuator/internal/subcmd/mysqlcmd/import_grants_file.go

@@ -0,0 +1,82 @@
+package mysqlcmd
+
+import (
+	"dbm-services/common/go-pubpkg/logger"
+	"dbm-services/mysql/db-tools/dbactuator/internal/subcmd"
+
+	"dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/grant/import_grants_file"
+	"dbm-services/mysql/db-tools/dbactuator/pkg/util"
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+const ImportGrantsFileCmd = "import-grants-file"
+
+type ImportGrantsFileAct struct {
+	*subcmd.BaseOptions
+	Service import_grants_file.ImportGrantsFile
+}
+
+func NewImportGrantsFileCommand() *cobra.Command {
+	act := ImportGrantsFileAct{
+		BaseOptions: subcmd.GBaseOptions,
+	}
+	cmd := &cobra.Command{
+		Use:   ImportGrantsFileCmd,
+		Short: "clone db grants",
+		Example: fmt.Sprintf(
+			`dbactuator mysql %s %s %s`,
+			ImportGrantsFileCmd, subcmd.CmdBaseExampleStr, subcmd.ToPrettyJson(act.Service.Example()),
+		),
+		Run: func(cmd *cobra.Command, args []string) {
+			util.CheckErr(act.Validate())
+			util.CheckErr(act.Init())
+			util.CheckErr(act.Run())
+		},
+	}
+	return cmd
+}
+
+func (c *ImportGrantsFileAct) Validate() error {
+	return c.BaseOptions.Validate()
+}
+
+func (c *ImportGrantsFileAct) Init() error {
+	if err := c.Deserialize(&c.Service.Params); err != nil {
+		logger.Error("DeserializeAndValidate err %s", err.Error())
+		return err
+	}
+	c.Service.GeneralParam = subcmd.GeneralRuntimeParam
+	logger.Info("extend params: %s", c.Service.Params)
+	return nil
+}
+
+func (c *ImportGrantsFileAct) Run() (err error) {
+	defer util.LoggerErrorStack(logger.Error, err)
+	steps := subcmd.Steps{
+		{
+			FunName: "初始化",
+			Func:    c.Service.Init,
+		},
+		{
+			FunName: "修改原始权限",
+			Func:    c.Service.ModifyPrivs,
+		},
+		{
+			FunName: "写入最终权限文件",
+			Func:    c.Service.GenerateFinalFile,
+		},
+		{
+			FunName: "执行导入",
+			Func:    c.Service.Import,
+		},
+	}
+
+	if err = steps.Run(); err != nil {
+		return err
+	}
+
+	logger.Info("import grants file completed")
+	return nil
+}

dbm-services/mysql/db-tools/dbactuator/internal/subcmd/mysqlcmd/mysqlcmd.go

@@ -76,6 +76,7 @@ func NewMysqlCommand() *cobra.Command {
 				v2.NewGenPeripheralToolsConfigCommand(),
 				v2.NewReloadPeripheralToolsConfigCommand(),
 				v2.NewInitCommonConfigCommand(),
+				NewImportGrantsFileCommand(),
 			},
 		},
 		{

dbm-services/mysql/db-tools/dbactuator/internal/subcmd/proxycmd/clone_proxy_user.go

@@ -6,27 +6,16 @@
 package proxycmd
 
 import (
-	"fmt"
-
 	"dbm-services/common/go-pubpkg/logger"
 	"dbm-services/mysql/db-tools/dbactuator/internal/subcmd"
 	"dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql_proxy"
 	"dbm-services/mysql/db-tools/dbactuator/pkg/util"
+	"fmt"
 
 	"github.com/spf13/cobra"
 )
 
 // CloneProxyUserAct TODO
-// extend payload
-/*
- {
-		 "source_proxy_host": "1.1.1.1",
-		 "source_proxy_port"  10000,
-		 "target_proxy_host"  "2.2.2.2",
-		 "target_proxy_port"  10000
-
- }
-*/
 type CloneProxyUserAct struct {
 	*subcmd.BaseOptions
 	Service mysql_proxy.CloneProxyUserComp
@@ -74,10 +63,6 @@ func (c *CloneProxyUserAct) Validate() (err error) {
 // Run TODO
 func (c *CloneProxyUserAct) Run() (err error) {
 	steps := subcmd.Steps{
-		{
-			FunName: "初始化",
-			Func:    c.Service.Init,
-		},
 		{
 			FunName: "Clone proxy user",
 			Func:    c.Service.CloneProxyUser,

dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/backupdemand/backup_demand.go

@@ -265,7 +265,7 @@ func (c *Component) DoBackup() error {
 func (c *Component) generateReport() (report *Report, indexFile string, err error) {
 	report = &Report{}
 
-	indexFileSearch := filepath.Join(c.backupDir, "*.index")
+	indexFileSearch := filepath.Join(c.backupDir, fmt.Sprintf("*%d*.index", c.Params.Port))
 	if files, err := filepath.Glob(indexFileSearch); err != nil {
 		return nil, indexFile, err
 	} else {
@@ -281,7 +281,7 @@ func (c *Component) generateReport() (report *Report, indexFile string, err erro
 				continue
 				//return nil, err
 			}
-			if result.BillId == c.Params.BillId && c.Params.BillId != "" {
+			if result.BillId == c.Params.BillId && c.Params.BillId != "" && result.BackupId == c.Params.BackupId {
 				report.Result = &result
 				indexFile = f
 				break

dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/grant/import_grants_file/example.go

@@ -0,0 +1,24 @@
+package import_grants_file
+
+import (
+	"dbm-services/mysql/db-tools/dbactuator/pkg/components"
+	"dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/common"
+)
+
+func (c *ImportGrantsFile) Example() interface{} {
+	return ImportGrantsFile{
+		GeneralParam: &components.GeneralParam{
+			RuntimeAccountParam: components.RuntimeAccountParam{
+				MySQLAccountParam: common.AccountAdminExample,
+			},
+		},
+		Params: &ImportGrantsFileParam{
+			SourceIp:      "1.1.1.1",
+			SourceVersion: "5.7.20-tmysql-3.4.4-log",
+			DestAddress:   "2.2.2.2:20000",
+			Filename:      "source-grants.priv",
+			IgnoreUsers:   []string{"u1", "u2", "u3"},
+			MachineType:   "backend",
+		},
+	}
+}

dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/grant/import_grants_file/generate_final.go

@@ -0,0 +1,79 @@
+package import_grants_file
+
+import (
+	"dbm-services/common/go-pubpkg/logger"
+	"dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/grant/import_grants_file/internal/listener"
+	"fmt"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+)
+
+func (c *ImportGrantsFile) GenerateFinalFile() (err error) {
+	c.finalFilename = fmt.Sprintf("final-%s", c.Params.Filename)
+	fp := filepath.Join(c.workDir, c.finalFilename)
+	f, err := os.OpenFile(fp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = f.Close()
+	}()
+
+	c.lineCount = 0
+	for _, l := range c.finalListeners {
+		var nonSysAuthOptions []*listener.AuthOptionStruct
+
+		logger.Info("filter %s system users", l.ToSql())
+		for _, ap := range l.AuthOptions {
+			fullNameHost := strings.Trim(ap.Username, "'`") // 把两头可能的单引号去掉方便处理
+
+			idx := slices.IndexFunc(c.Params.IgnoreUsers, func(iu string) bool {
+				return strings.HasPrefix(fullNameHost, iu)
+			})
+
+			// 不是系统账号
+			if idx < 0 {
+				// 残留的 J_ 账号
+				// 源 ip 在 ModifyPrivs 里已经提前修改为 dest ip 了
+				if (strings.HasSuffix(fullNameHost, "@'localhost") ||
+					strings.HasSuffix(fullNameHost, "@`localhost") ||
+					strings.HasSuffix(fullNameHost, fmt.Sprintf("@'%s", c.destIp)) ||
+					strings.HasSuffix(fullNameHost, fmt.Sprintf("@`%s", c.destIp))) &&
+					strings.HasPrefix(fullNameHost, "J_") {
+					logger.Info("maybe also system user: %s, ignored", fullNameHost)
+				} else {
+					logger.Info("local normal user: %s", fullNameHost)
+					nonSysAuthOptions = append(nonSysAuthOptions, ap)
+				}
+			} else {
+				logger.Info("%s will be ignored", fullNameHost)
+			}
+		}
+
+		//系统账号不生成 sql
+		if len(nonSysAuthOptions) <= 0 {
+			logger.Info("whole %s ignored", l.ToSql())
+			continue
+		}
+
+		l.AuthOptions = nonSysAuthOptions
+
+		_, err = f.WriteString(strings.Trim(l.ToSql(), ";") + ";\n")
+		if err != nil {
+			return err
+		}
+		c.lineCount++
+	}
+
+	_, err = f.WriteString("FLUSH PRIVILEGES;\n")
+	if err != nil {
+		return err
+	}
+	c.lineCount++
+
+	// 写完文件把内存释放掉
+	c.finalListeners = nil
+	return nil
+}

dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/grant/import_grants_file/import_file.go

@@ -0,0 +1,73 @@
+package import_grants_file
+
+import (
+	"bufio"
+	"context"
+	"dbm-services/common/go-pubpkg/logger"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+func (c *ImportGrantsFile) Import() (err error) {
+	fp := filepath.Join(c.workDir, c.finalFilename)
+	f, err := os.OpenFile(fp, os.O_RDONLY, os.ModePerm)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = f.Close()
+	}()
+
+	conn, err := c.db.Db.Conn(context.Background())
+	if err != nil {
+		return err
+	}
+	_, err = conn.ExecContext(
+		context.Background(),
+		"SET sql_log_bin=0",
+	)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = conn.Close()
+	}()
+	logger.Info("disable binlog")
+
+	doneCh := make(chan struct{})
+	defer close(doneCh)
+	counter := 1
+	// 把日志放到异步打印, 要不然太多了
+	go func() {
+		ticker := time.NewTicker(1 * time.Second)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ticker.C:
+				logger.Info("%d/%d imported", counter, c.lineCount)
+			case <-doneCh:
+				logger.Info("%d/%d imported", counter, c.lineCount)
+				return
+			}
+		}
+	}()
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
+		_, err = conn.ExecContext(context.Background(), line)
+		if err != nil {
+			logger.Error("import failed: %s", line, err.Error())
+			return err
+		}
+		counter++
+	}
+	if err := scanner.Err(); err != nil {
+		return err
+	}
+
+	doneCh <- struct{}{}
+
+	return nil
+}