Skip to content

Commit 792f5ee

Browse files
narrowizardnarrowizard
committed
feat(customize): add CSV import functionality for sprints and issue changelogs/worklogs
- Add new API endpoints for importing sprints, issue changelogs, and issue worklogs from CSV files - Implement corresponding handler functions to process the uploaded CSV files - Add e2e tests to verify the import functionality for sprints, issue changelogs, and issue worklogs - Update the plugin's ApiResources map to include the new endpoints #8446
1 parent 4647d3d commit 792f5ee

24 files changed

+637
-8
lines changed

‎backend/plugins/customize/api/csv_issue.go

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,99 @@ func (h *Handlers) ImportIssueRepoCommit(input *plugin.ApiResourceInput) (*plugi
124124
return nil, h.svc.ImportIssueRepoCommit(boardId, file, incremental)
125125
}
126126

127+
// ImportSprint accepts a CSV file, parses and saves it to the database
128+
// @Summary Upload sprints.csv file
129+
// @Description Upload sprints.csv file
130+
// @Tags plugins/customize
131+
// @Accept multipart/form-data
132+
// @Param boardId formData string true "the ID of the board"
133+
// @Param file formData file true "select file to upload"
134+
// @Param incremental formData string true "whether to save only new data"
135+
// @Produce json
136+
// @Success 200
137+
// @Failure 400 {object} shared.ApiBody "Bad Request"
138+
// @Failure 500 {object} shared.ApiBody "Internal Error"
139+
// @Router /plugins/customize/csvfiles/sprints.csv [post]
140+
func (h *Handlers) ImportSprint(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
141+
file, err := h.extractFile(input)
142+
if err != nil {
143+
return nil, err
144+
}
145+
// nolint
146+
defer file.Close()
147+
boardId := strings.TrimSpace(input.Request.FormValue("boardId"))
148+
if boardId == "" {
149+
return nil, errors.Default.New("empty boardId")
150+
}
151+
incremental := false
152+
if input.Request.FormValue("incremental") == "true" {
153+
incremental = true
154+
}
155+
return nil, h.svc.ImportSprint(boardId, file, incremental)
156+
}
157+
158+
// ImportIssueChangelog accepts a CSV file, parses and saves it to the database
159+
// @Summary Upload issue_changelogs.csv file
160+
// @Description Upload issue_changelogs.csv file
161+
// @Tags plugins/customize
162+
// @Accept multipart/form-data
163+
// @Param boardId formData string true "the ID of the board"
164+
// @Param file formData file true "select file to upload"
165+
// @Param incremental formData boolean false "Whether to incrementally update changelogs" default(false)
166+
// @Produce json
167+
// @Success 200
168+
// @Failure 400 {object} shared.ApiBody "Bad Request"
169+
// @Failure 500 {object} shared.ApiBody "Internal Error"
170+
// @Router /plugins/customize/csvfiles/issue_changelogs.csv [post]
171+
func (h *Handlers) ImportIssueChangelog(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
172+
file, err := h.extractFile(input)
173+
if err != nil {
174+
return nil, err
175+
}
176+
// nolint
177+
defer file.Close()
178+
boardId := strings.TrimSpace(input.Request.FormValue("boardId"))
179+
if boardId == "" {
180+
return nil, errors.Default.New("empty boardId")
181+
}
182+
incremental := false
183+
if input.Request.FormValue("incremental") == "true" {
184+
incremental = true
185+
}
186+
return nil, h.svc.ImportIssueChangelog(boardId, file, incremental)
187+
}
188+
189+
// ImportIssueWorklog accepts a CSV file, parses and saves it to the database
190+
// @Summary Upload issue_worklogs.csv file
191+
// @Description Upload issue_worklogs.csv file
192+
// @Tags plugins/customize
193+
// @Accept multipart/form-data
194+
// @Param boardId formData string true "the ID of the board"
195+
// @Param file formData file true "select file to upload"
196+
// @Param incremental formData boolean false "Whether to do incremental sync (default false
197+
// @Produce json
198+
// @Success 200
199+
// @Failure 400 {object} shared.ApiBody "Bad Request"
200+
// @Failure 500 {object} shared.ApiBody "Internal Error"
201+
// @Router /plugins/customize/csvfiles/issue_worklogs.csv [post]
202+
func (h *Handlers) ImportIssueWorklog(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
203+
file, err := h.extractFile(input)
204+
if err != nil {
205+
return nil, err
206+
}
207+
// nolint
208+
defer file.Close()
209+
boardId := strings.TrimSpace(input.Request.FormValue("boardId"))
210+
if boardId == "" {
211+
return nil, errors.Default.New("empty boardId")
212+
}
213+
incremental := false
214+
if input.Request.FormValue("incremental") == "true" {
215+
incremental = true
216+
}
217+
return nil, h.svc.ImportIssueWorklog(boardId, file, incremental)
218+
}
219+
127220
func (h *Handlers) extractFile(input *plugin.ApiResourceInput) (io.ReadCloser, errors.Error) {
128221
if input.Request == nil {
129222
return nil, errors.Default.New("request is nil")

‎backend/plugins/customize/e2e/import_issue_changelogs_test.go

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
/*
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
*/
17+
18+
package e2e
19+
20+
import (
21+
"os"
22+
"testing"
23+
24+
"github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
25+
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
26+
"github.com/apache/incubator-devlake/helpers/e2ehelper"
27+
"github.com/apache/incubator-devlake/plugins/customize/impl"
28+
"github.com/apache/incubator-devlake/plugins/customize/service"
29+
)
30+
31+
func TestImportIssueChangelogDataFlow(t *testing.T) {
32+
var plugin impl.Customize
33+
dataflowTester := e2ehelper.NewDataFlowTester(t, "customize", plugin)
34+
35+
// 清空表
36+
dataflowTester.FlushTabler(&ticket.IssueChangelogs{})
37+
dataflowTester.FlushTabler(&crossdomain.Account{})
38+
39+
// 初始化服务
40+
svc := service.NewService(dataflowTester.Dal)
41+
42+
// 导入全量数据
43+
changelogFile, err := os.Open("raw_tables/issue_changelogs.csv")
44+
if err != nil {
45+
t.Fatal(err)
46+
}
47+
defer changelogFile.Close()
48+
err = svc.ImportIssueChangelog("TEST_BOARD", changelogFile, false)
49+
if err != nil {
50+
t.Fatal(err)
51+
}
52+
53+
// 验证全量导入结果
54+
dataflowTester.VerifyTableWithRawData(
55+
ticket.IssueChangelogs{},
56+
"snapshot_tables/issue_changelogs.csv",
57+
[]string{
58+
"id",
59+
"issue_id",
60+
"author_id",
61+
"field_name",
62+
"original_from_value",
63+
"original_to_value",
64+
"created_date",
65+
})
66+
67+
// 导入增量数据
68+
incrementalFile, err := os.Open("raw_tables/issue_changelogs_incremental.csv")
69+
if err != nil {
70+
t.Fatal(err)
71+
}
72+
defer incrementalFile.Close()
73+
err = svc.ImportIssueChangelog("TEST_BOARD", incrementalFile, true)
74+
if err != nil {
75+
t.Fatal(err)
76+
}
77+
78+
// 验证增量导入结果
79+
dataflowTester.VerifyTableWithRawData(
80+
ticket.IssueChangelogs{},
81+
"snapshot_tables/issue_changelogs_incremental.csv",
82+
[]string{
83+
"id",
84+
"issue_id",
85+
"author_id",
86+
"field_name",
87+
"original_from_value",
88+
"original_to_value",
89+
"created_date",
90+
})
91+
92+
dataflowTester.VerifyTable(
93+
crossdomain.Account{},
94+
"snapshot_tables/accounts_from_issue_changelogs.csv",
95+
[]string{
96+
"id",
97+
"full_name",
98+
"user_name",
99+
},
100+
)
101+
}

‎backend/plugins/customize/e2e/import_issue_worklogs_test.go

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
/*
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
*/
17+
18+
package e2e
19+
20+
import (
21+
"os"
22+
"testing"
23+
24+
"github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
25+
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
26+
"github.com/apache/incubator-devlake/helpers/e2ehelper"
27+
"github.com/apache/incubator-devlake/plugins/customize/impl"
28+
"github.com/apache/incubator-devlake/plugins/customize/service"
29+
)
30+
31+
func TestImportIssueWorklogDataFlow(t *testing.T) {
32+
var plugin impl.Customize
33+
dataflowTester := e2ehelper.NewDataFlowTester(t, "customize", plugin)
34+
35+
// 清空表
36+
dataflowTester.FlushTabler(&ticket.IssueWorklog{})
37+
dataflowTester.FlushTabler(&crossdomain.Account{})
38+
39+
// 初始化服务
40+
svc := service.NewService(dataflowTester.Dal)
41+
42+
// 导入全量数据
43+
worklogFile, err := os.Open("raw_tables/issue_worklogs.csv")
44+
if err != nil {
45+
t.Fatal(err)
46+
}
47+
defer worklogFile.Close()
48+
err = svc.ImportIssueWorklog("TEST_BOARD", worklogFile, false)
49+
if err != nil {
50+
t.Fatal(err)
51+
}
52+
53+
// 验证全量导入结果
54+
dataflowTester.VerifyTableWithRawData(
55+
ticket.IssueWorklog{},
56+
"snapshot_tables/issue_worklogs.csv",
57+
[]string{
58+
"id",
59+
"issue_id",
60+
"author_id",
61+
"time_spent_minutes",
62+
"started_date",
63+
"logged_date",
64+
"comment",
65+
})
66+
67+
// 导入增量数据
68+
incrementalFile, err := os.Open("raw_tables/issue_worklogs_incremental.csv")
69+
if err != nil {
70+
t.Fatal(err)
71+
}
72+
defer incrementalFile.Close()
73+
err = svc.ImportIssueWorklog("TEST_BOARD", incrementalFile, true)
74+
if err != nil {
75+
t.Fatal(err)
76+
}
77+
78+
// 验证增量导入结果
79+
dataflowTester.VerifyTableWithRawData(
80+
ticket.IssueWorklog{},
81+
"snapshot_tables/issue_worklogs_incremental.csv",
82+
[]string{
83+
"id",
84+
"issue_id",
85+
"author_id",
86+
"time_spent_minutes",
87+
"started_date",
88+
"logged_date",
89+
"comment",
90+
})
91+
92+
dataflowTester.VerifyTable(
93+
crossdomain.Account{},
94+
"snapshot_tables/accounts_from_issue_worklogs.csv",
95+
[]string{
96+
"id",
97+
"full_name",
98+
"user_name",
99+
},
100+
)
101+
}

‎backend/plugins/customize/e2e/import_issues_test.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ func TestImportIssueDataFlow(t *testing.T) {
3939
dataflowTester.FlushTabler(&ticket.IssueLabel{})
4040
dataflowTester.FlushTabler(&ticket.BoardIssue{})
4141
dataflowTester.FlushTabler(&crossdomain.Account{})
42+
dataflowTester.FlushTabler(&ticket.SprintIssue{})
4243
svc := service.NewService(dataflowTester.Dal)
4344
err := svc.CreateField(&models.CustomizedField{
4445
TbName: "issues",
@@ -183,4 +184,13 @@ func TestImportIssueDataFlow(t *testing.T) {
183184
"user_name",
184185
},
185186
)
187+
188+
dataflowTester.VerifyTableWithRawData(
189+
&ticket.SprintIssue{},
190+
"snapshot_tables/sprint_issues.csv",
191+
[]string{
192+
"sprint_id",
193+
"issue_id",
194+
},
195+
)
186196
}

0 commit comments

Comments
 (0)