feat(customize): add CSV import functionality for sprints and issue c…

backend/plugins/customize/api/csv_issue.go

@@ -124,6 +124,99 @@ func (h *Handlers) ImportIssueRepoCommit(input *plugin.ApiResourceInput) (*plugi
 	return nil, h.svc.ImportIssueRepoCommit(boardId, file, incremental)
 }
 
+// ImportSprint accepts a CSV file, parses and saves it to the database
+// @Summary      Upload sprints.csv file
+// @Description  Upload sprints.csv file
+// @Tags 		 plugins/customize
+// @Accept       multipart/form-data
+// @Param        boardId formData string true "the ID of the board"
+// @Param        file formData file true "select file to upload"
+// @Param        incremental formData string true "whether to save only new data"
+// @Produce      json
+// @Success      200
+// @Failure 400  {object} shared.ApiBody "Bad Request"
+// @Failure 500  {object} shared.ApiBody "Internal Error"
+// @Router       /plugins/customize/csvfiles/sprints.csv [post]
+func (h *Handlers) ImportSprint(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	file, err := h.extractFile(input)
+	if err != nil {
+		return nil, err
+	}
+	// nolint
+	defer file.Close()
+	boardId := strings.TrimSpace(input.Request.FormValue("boardId"))
+	if boardId == "" {
+		return nil, errors.Default.New("empty boardId")
+	}
+	incremental := false
+	if input.Request.FormValue("incremental") == "true" {
+		incremental = true
+	}
+	return nil, h.svc.ImportSprint(boardId, file, incremental)
+}
+
+// ImportIssueChangelog accepts a CSV file, parses and saves it to the database
+// @Summary      Upload issue_changelogs.csv file
+// @Description  Upload issue_changelogs.csv file
+// @Tags 		 plugins/customize
+// @Accept       multipart/form-data
+// @Param        boardId formData string true "the ID of the board"
+// @Param        file formData file true "select file to upload"
+// @Param 		 incremental formData boolean false "Whether to incrementally update changelogs" default(false)
+// @Produce      json
+// @Success      200
+// @Failure 400  {object} shared.ApiBody "Bad Request"
+// @Failure 500  {object} shared.ApiBody "Internal Error"
+// @Router       /plugins/customize/csvfiles/issue_changelogs.csv [post]
+func (h *Handlers) ImportIssueChangelog(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	file, err := h.extractFile(input)
+	if err != nil {
+		return nil, err
+	}
+	// nolint
+	defer file.Close()
+	boardId := strings.TrimSpace(input.Request.FormValue("boardId"))
+	if boardId == "" {
+		return nil, errors.Default.New("empty boardId")
+	}
+	incremental := false
+	if input.Request.FormValue("incremental") == "true" {
+		incremental = true
+	}
+	return nil, h.svc.ImportIssueChangelog(boardId, file, incremental)
+}
+
+// ImportIssueWorklog accepts a CSV file, parses and saves it to the database
+// @Summary      Upload issue_worklogs.csv file
+// @Description  Upload issue_worklogs.csv file
+// @Tags 		 plugins/customize
+// @Accept       multipart/form-data
+// @Param        boardId formData string true "the ID of the board"
+// @Param        file formData file true "select file to upload"
+// @Param        incremental formData boolean false "Whether to do incremental sync (default false
+// @Produce      json
+// @Success      200
+// @Failure 400  {object} shared.ApiBody "Bad Request"
+// @Failure 500  {object} shared.ApiBody "Internal Error"
+// @Router       /plugins/customize/csvfiles/issue_worklogs.csv [post]
+func (h *Handlers) ImportIssueWorklog(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	file, err := h.extractFile(input)
+	if err != nil {
+		return nil, err
+	}
+	// nolint
+	defer file.Close()
+	boardId := strings.TrimSpace(input.Request.FormValue("boardId"))
+	if boardId == "" {
+		return nil, errors.Default.New("empty boardId")
+	}
+	incremental := false
+	if input.Request.FormValue("incremental") == "true" {
+		incremental = true
+	}
+	return nil, h.svc.ImportIssueWorklog(boardId, file, incremental)
+}
+
 func (h *Handlers) extractFile(input *plugin.ApiResourceInput) (io.ReadCloser, errors.Error) {
 	if input.Request == nil {
 		return nil, errors.Default.New("request is nil")

backend/plugins/customize/e2e/import_issue_changelogs_test.go

@@ -0,0 +1,101 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"os"
+	"testing"
+
+	"github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
+	"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
+	"github.com/apache/incubator-devlake/helpers/e2ehelper"
+	"github.com/apache/incubator-devlake/plugins/customize/impl"
+	"github.com/apache/incubator-devlake/plugins/customize/service"
+)
+
+func TestImportIssueChangelogDataFlow(t *testing.T) {
+	var plugin impl.Customize
+	dataflowTester := e2ehelper.NewDataFlowTester(t, "customize", plugin)
+
+	// 清空表
+	dataflowTester.FlushTabler(&ticket.IssueChangelogs{})
+    dataflowTester.FlushTabler(&crossdomain.Account{})
+
+	// 初始化服务
+	svc := service.NewService(dataflowTester.Dal)
+
+	// 导入全量数据
+	changelogFile, err := os.Open("raw_tables/issue_changelogs.csv")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer changelogFile.Close()
+	err = svc.ImportIssueChangelog("TEST_BOARD", changelogFile, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// 验证全量导入结果
+	dataflowTester.VerifyTableWithRawData(
+		ticket.IssueChangelogs{},
+		"snapshot_tables/issue_changelogs.csv",
+		[]string{
+			"id",
+			"issue_id",
+			"author_id",
+			"field_name",
+			"original_from_value",
+			"original_to_value",
+			"created_date",
+		})
+
+	// 导入增量数据
+	incrementalFile, err := os.Open("raw_tables/issue_changelogs_incremental.csv")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer incrementalFile.Close()
+	err = svc.ImportIssueChangelog("TEST_BOARD", incrementalFile, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// 验证增量导入结果
+	dataflowTester.VerifyTableWithRawData(
+		ticket.IssueChangelogs{},
+		"snapshot_tables/issue_changelogs_incremental.csv",
+		[]string{
+			"id",
+			"issue_id",
+			"author_id",
+			"field_name",
+			"original_from_value",
+			"original_to_value",
+			"created_date",
+		})
+
+	dataflowTester.VerifyTable(
+		crossdomain.Account{},
+		"snapshot_tables/accounts_from_issue_changelogs.csv",
+		[]string{
+			"id",
+			"full_name",
+			"user_name",
+		},
+	)
+}

backend/plugins/customize/e2e/import_issue_worklogs_test.go

@@ -0,0 +1,101 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"os"
+	"testing"
+
+	"github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
+	"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
+	"github.com/apache/incubator-devlake/helpers/e2ehelper"
+	"github.com/apache/incubator-devlake/plugins/customize/impl"
+	"github.com/apache/incubator-devlake/plugins/customize/service"
+)
+
+func TestImportIssueWorklogDataFlow(t *testing.T) {
+	var plugin impl.Customize
+	dataflowTester := e2ehelper.NewDataFlowTester(t, "customize", plugin)
+
+	// 清空表
+	dataflowTester.FlushTabler(&ticket.IssueWorklog{})
+	dataflowTester.FlushTabler(&crossdomain.Account{})
+
+	// 初始化服务
+	svc := service.NewService(dataflowTester.Dal)
+
+	// 导入全量数据
+	worklogFile, err := os.Open("raw_tables/issue_worklogs.csv")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer worklogFile.Close()
+	err = svc.ImportIssueWorklog("TEST_BOARD", worklogFile, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// 验证全量导入结果
+	dataflowTester.VerifyTableWithRawData(
+		ticket.IssueWorklog{},
+		"snapshot_tables/issue_worklogs.csv",
+		[]string{
+			"id",
+			"issue_id",
+			"author_id",
+			"time_spent_minutes",
+			"started_date",
+            "logged_date",
+			"comment",
+		})
+
+	// 导入增量数据
+	incrementalFile, err := os.Open("raw_tables/issue_worklogs_incremental.csv")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer incrementalFile.Close()
+	err = svc.ImportIssueWorklog("TEST_BOARD", incrementalFile, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// 验证增量导入结果
+	dataflowTester.VerifyTableWithRawData(
+		ticket.IssueWorklog{},
+		"snapshot_tables/issue_worklogs_incremental.csv",
+		[]string{
+			"id",
+			"issue_id",
+			"author_id",
+			"time_spent_minutes",
+			"started_date",
+            "logged_date",
+			"comment",
+		})
+
+	dataflowTester.VerifyTable(
+		crossdomain.Account{},
+		"snapshot_tables/accounts_from_issue_worklogs.csv",
+		[]string{
+			"id",
+			"full_name",
+			"user_name",
+		},
+	)
+}

backend/plugins/customize/e2e/import_issues_test.go

@@ -39,6 +39,7 @@ func TestImportIssueDataFlow(t *testing.T) {
 	dataflowTester.FlushTabler(&ticket.IssueLabel{})
 	dataflowTester.FlushTabler(&ticket.BoardIssue{})
 	dataflowTester.FlushTabler(&crossdomain.Account{})
+	dataflowTester.FlushTabler(&ticket.SprintIssue{})
 	svc := service.NewService(dataflowTester.Dal)
 	err := svc.CreateField(&models.CustomizedField{
 		TbName:      "issues",
@@ -183,4 +184,13 @@ func TestImportIssueDataFlow(t *testing.T) {
 			"user_name",
 		},
 	)
+
+	dataflowTester.VerifyTableWithRawData(
+		&ticket.SprintIssue{},
+		"snapshot_tables/sprint_issues.csv",
+		[]string{
+			"sprint_id",
+			"issue_id",
+		},
+	)
 }