Skip to content

Commit 8073cee

Browse files
DimfacionDimfacion
committed
[backend] Batching execution traces
1 parent 4792f5d commit 8073cee

26 files changed

+1863
-79
lines changed

‎.drone.yml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ steps:
1010
MINIO_ENDPOINT: minio
1111
MINIO_PORT: 9000
1212
ENGINE_URL: http://elastic:9200
13+
OPENBAS_RABBITMQ_HOSTNAME: rabbitmq
1314
commands:
1415
- mvn clean install -q -DskipTests
1516
- mvn spotless:check
@@ -48,6 +49,7 @@ steps:
4849
OPENBAS_ADMIN_PASSWORD: admin
4950
OPENBAS_ADMIN_TOKEN: 0d17ce9a-f3a8-4c6d-9721-c98dc3dc023f
5051
SPRING_PROFILES_ACTIVE: ci
52+
OPENBAS_RABBITMQ_HOSTNAME: rabbitmq-e2e
5153
commands:
5254
- apt update && apt install -y gnupg
5355
- curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
@@ -175,6 +177,11 @@ services:
175177
POSTGRES_USER: openbas
176178
POSTGRES_PASSWORD: openbas
177179
POSTGRES_DB: openbas
180+
- name: rabbitmq
181+
image: rabbitmq:4.1-management
182+
environment:
183+
RABBITMQ_DEFAULT_USER: guest
184+
RABBITMQ_DEFAULT_PASS: guest
178185
- name: minio-e2e
179186
image: minio/minio:RELEASE.2023-12-02T10-51-33Z-cpuv1
180187
environment:
@@ -187,6 +194,11 @@ services:
187194
POSTGRES_USER: openbas
188195
POSTGRES_PASSWORD: openbas
189196
POSTGRES_DB: openbas
197+
- name: rabbitmq-e2e
198+
image: rabbitmq:4.1-management
199+
environment:
200+
RABBITMQ_DEFAULT_USER: guest
201+
RABBITMQ_DEFAULT_PASS: guest
190202
- name: elastic
191203
image: docker.elastic.co/elasticsearch/elasticsearch:8.18.2
192204
environment:

‎openbas-api/src/main/java/io/openbas/rest/finding/FindingService.java

Lines changed: 33 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,12 @@
1212
import io.openbas.database.repository.TeamRepository;
1313
import io.openbas.database.repository.UserRepository;
1414
import io.openbas.injector_contract.outputs.ContractOutputElement;
15-
import io.openbas.injector_contract.outputs.ContractOutputUtils;
1615
import io.openbas.rest.inject.service.InjectService;
1716
import jakarta.annotation.Resource;
1817
import jakarta.persistence.EntityNotFoundException;
1918
import jakarta.validation.constraints.NotBlank;
2019
import java.util.ArrayList;
2120
import java.util.List;
22-
import java.util.Optional;
2321
import java.util.Set;
2422
import lombok.RequiredArgsConstructor;
2523
import lombok.extern.slf4j.Slf4j;
@@ -87,11 +85,11 @@ public void deleteFinding(@NotNull final String id) {
8785
// This structrued output is generated based on injectorcontract where we can find the node
8886
// Outputs and with that the injector generate this structure output--
8987

90-
public void extractFindingsFromInjectorContract(Inject inject, ObjectNode structuredOutput) {
88+
public List<SimpleFinding> extractFindingsFromInjectorContract(
89+
String injectId, InjectorContract injectorContract, ObjectNode structuredOutput) {
9190
// NOTE: do it in every call to callback ? (reflexion on implant mechanism)
92-
List<Finding> findings = new ArrayList<>();
91+
List<SimpleFinding> findings = new ArrayList<>();
9392
// Get the contract
94-
InjectorContract injectorContract = inject.getInjectorContract().orElseThrow();
9593
List<ContractOutputElement> contractOutputs =
9694
getContractOutputs(injectorContract.getConvertedContent(), mapper);
9795
if (!contractOutputs.isEmpty()) {
@@ -105,61 +103,59 @@ public void extractFindingsFromInjectorContract(Inject inject, ObjectNode struct
105103
if (!contractOutput.getType().validate.apply(jsonNode)) {
106104
throw new IllegalArgumentException("Finding not correctly formatted");
107105
}
108-
Finding finding = ContractOutputUtils.createFinding(contractOutput);
106+
SimpleFinding finding = new SimpleFinding();
107+
finding.setType(contractOutput.getType().toString());
108+
finding.setField(contractOutput.getField());
109+
finding.setLabels(contractOutput.getLabels());
109110
finding.setValue(contractOutput.getType().toFindingValue.apply(jsonNode));
110-
Finding linkedFinding = linkFindings(contractOutput, jsonNode, finding);
111-
findings.add(linkedFinding);
111+
linkFindings(contractOutput, jsonNode, finding);
112+
findings.add(finding);
112113
}
113114
}
114115
} else {
115116
JsonNode jsonNode = structuredOutput.get(contractOutput.getField());
116117
if (!contractOutput.getType().validate.apply(jsonNode)) {
117118
throw new IllegalArgumentException("Finding not correctly formatted");
118119
}
119-
Finding finding = ContractOutputUtils.createFinding(contractOutput);
120+
SimpleFinding finding = new SimpleFinding();
121+
finding.setType(contractOutput.getType().toString());
122+
finding.setField(contractOutput.getField());
123+
finding.setLabels(contractOutput.getLabels());
120124
finding.setValue(contractOutput.getType().toFindingValue.apply(jsonNode));
121-
Finding linkedFinding = linkFindings(contractOutput, jsonNode, finding);
122-
findings.add(linkedFinding);
125+
linkFindings(contractOutput, jsonNode, finding);
126+
findings.add(finding);
123127
}
124128
}
125129
});
126130
}
127-
this.createFindings(findings, inject.getId());
131+
findings.forEach((finding) -> finding.setInjectId(injectId));
132+
return findings;
128133
}
129134

130-
private Finding linkFindings(
131-
ContractOutputElement contractOutput, JsonNode jsonNode, Finding finding) {
135+
private void linkFindings(
136+
ContractOutputElement contractOutput, JsonNode jsonNode, SimpleFinding finding) {
132137
// Create links with assets
133138
if (contractOutput.getType().toFindingAssets != null) {
134139
List<String> assetsIds = contractOutput.getType().toFindingAssets.apply(jsonNode);
135-
List<Optional<Asset>> assets =
136-
assetsIds.stream().map(this.assetRepository::findById).toList();
137-
if (!assets.isEmpty()) {
138-
finding.setAssets(assets.stream().filter(Optional::isPresent).map(Optional::get).toList());
139-
}
140+
finding.setAssets(assetsIds);
140141
}
141142
// Create links with teams
142143
if (contractOutput.getType().toFindingTeams != null) {
143144
List<String> teamsIds = contractOutput.getType().toFindingTeams.apply(jsonNode);
144-
List<Optional<Team>> teams = teamsIds.stream().map(this.teamRepository::findById).toList();
145-
if (!teams.isEmpty()) {
146-
finding.setTeams(teams.stream().filter(Optional::isPresent).map(Optional::get).toList());
147-
}
145+
finding.setTeams(teamsIds);
148146
}
149147
// Create links with users
150148
if (contractOutput.getType().toFindingUsers != null) {
151149
List<String> usersIds = contractOutput.getType().toFindingUsers.apply(jsonNode);
152-
List<Optional<User>> users = usersIds.stream().map(this.userRepository::findById).toList();
153-
if (!users.isEmpty()) {
154-
finding.setUsers(users.stream().filter(Optional::isPresent).map(Optional::get).toList());
155-
}
150+
finding.setUsers(usersIds);
156151
}
157-
return finding;
158152
}
159153

160154
/** Extracts findings from structured output that was generated using output parsers. */
161-
public void extractFindingsFromOutputParsers(
162-
Inject inject, Agent agent, Set<OutputParser> outputParsers, JsonNode structuredOutput) {
155+
public List<SimpleFinding> extractFindingsFromOutputParsers(
156+
String injectId, String assetId, Set<OutputParser> outputParsers, JsonNode structuredOutput) {
157+
158+
List<SimpleFinding> results = new ArrayList<>();
163159

164160
outputParsers.forEach(
165161
outputParser -> {
@@ -176,15 +172,17 @@ public void extractFindingsFromOutputParsers(
176172
throw new IllegalArgumentException("Finding not correctly formatted");
177173
}
178174
// Build and save the finding
179-
findingUtils.buildFinding(
180-
inject,
181-
agent.getAsset(),
182-
contractOutputElement,
183-
contractOutputElement.getType().toFindingValue.apply(jsonNode));
175+
results.add(
176+
findingUtils.buildSimplerFinding(
177+
injectId,
178+
assetId,
179+
contractOutputElement,
180+
contractOutputElement.getType().toFindingValue.apply(jsonNode)));
184181
}
185182
}
186183
}
187184
});
188185
});
186+
return results;
189187
}
190188
}

‎openbas-api/src/main/java/io/openbas/rest/finding/FindingUtils.java

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,24 @@ public void buildFinding(
6262
}
6363
}
6464

65+
public SimpleFinding buildSimplerFinding(
66+
String injectId,
67+
String assetId,
68+
io.openbas.database.model.ContractOutputElement contractOutputElement,
69+
String finalValue) {
70+
71+
SimpleFinding newFinding = new SimpleFinding();
72+
newFinding.setInjectId(injectId);
73+
newFinding.setField(contractOutputElement.getKey());
74+
newFinding.setType(contractOutputElement.getType().toString());
75+
newFinding.setValue(finalValue);
76+
newFinding.setName(contractOutputElement.getName());
77+
newFinding.getAssets().add(assetId);
78+
newFinding.setTags(
79+
new HashSet<>(contractOutputElement.getTags().stream().map(Tag::getId).toList()));
80+
return newFinding;
81+
}
82+
6583
private void handleRaceCondition(
6684
Inject inject, Asset asset, ContractOutputElement contractOutputElement, String finalValue) {
6785
Optional<Finding> retryFinding =

0 commit comments

Comments
 (0)