Skip to content

Commit 05b3f63

Browse files
eshiferaxeshiferax
authored
docs: Add some diagrams. (#215)
Adding diagrams for each root/top-level module.
1 parent 799dddf commit 05b3f63

File tree

16 files changed

+637
-0
lines changed

16 files changed

+637
-0
lines changed

‎diagrams/README.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
## Diagrams
2+
3+
The diagrams shown in the READMEs in `modules/*/` are built using [eraser.io](https://eraser.io).
4+
5+
The source for each can be found in this directory. They use the Eraser [cloud architecture diagram syntax](https://docs.eraser.io/docs/syntax).
6+
7+
* [controlplane_rift](./controlplane_rift.txt)
8+
* [dataplane_rift](./dataplane_rift.txt)
9+
* [emr](./emr.txt)
10+
* [databricks](./databricks.txt)
11+
* [dataplane_rift_emr](./dataplane_rift_emr.txt)

‎diagrams/controlplane_rift.txt‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
direction: right
2+
tecton_virtual_private_tecton [label: "Tecton Control-Plane AWS ACcount", color: magenta, icon: aws] {
3+
control_plane_vpc [label: "Control Plane VPC", color: purple, icon: aws-vpc] {
4+
tecton_control_plane_app_metadata [label: "Tecton Control Plane Application & Metadata", color: gray, icon: monitor] {
5+
web_ui [label: "Web UI", icon: monitor]
6+
metadata_api [label: "Metadata API", icon: code]
7+
metrics_api [label: "Metrics API", icon: prometheus]
8+
feature_serving [label: "Feature Serving", icon: service]
9+
orchestration [label: "Orchestration", icon: settings]
10+
stream_ingest_api [icon: aws-ec2, label: "Stream Ingest API"]
11+
}
12+
nlb [label: "NLB", icon: aws-elastic-load-balancing]
13+
ingest_nlb [label: "Ingest NLB", icon: aws-elastic-load-balancing]
14+
vpc_endpoint_service [label: "VPC Endpoint Service (PrivateLink)", icon: aws-vpc-endpoint]
15+
ingest_vpc_endpoint_service [label: "Ingest VPC Endpoint Service (PrivateLink)", icon: aws-vpc-endpoint]
16+
17+
}
18+
19+
Rift VPC [color: blue, icon: aws-vpc] {
20+
rift_compute [label: "Rift Compute", icon: aws-ec2-instance]
21+
}
22+
rift_compute_role [label: "Rift Compute Role", icon: aws-iam-role]
23+
rift_compute_manager_role [label: "Rift Compute Manager Role", icon: aws-iam-role]
24+
rift_environments_repo[label: "Rift Environments Repository", icon: aws-ecr]
25+
}
26+
27+
customer_aws_account [label: "Customer AWS Account", color: orange, icon: aws-account] {
28+
online_feature_store [label: "Online feature storage\nDynamoDB table", icon: aws-dynamodb]
29+
cross_account_role [label: "Cross Account Role", icon: aws-iam-role]
30+
offline_feature_store [label: "Offline feature storage\nS3", icon: aws-s3]
31+
customer_vpc [label: "VPC", color: lightblue, icon: aws-vpc] {
32+
online_feature_consumption [label: "Online Feature consumption\nML App/Service", icon: monitor]
33+
offline_feature_consumption [label: "Offline Feature consumption\nDevelopment tools (Notebook)", icon: book-open]
34+
vpc_endpoint_1 [label: "VPC Endpoint", icon: aws-vpc-endpoint]
35+
}
36+
37+
VPC [label: "VPC", color: lightblue, icon: aws-vpc] {
38+
vpc_endpoint_2 [label: "VPC Endpoint", icon: aws-vpc-endpoint]
39+
streaming_ingest_app [icon: kafka, label: "Streaming / Realtime Application"]
40+
}
41+
42+
}
43+
44+
// connections
45+
web_ui > nlb
46+
metadata_api > nlb
47+
metrics_api > nlb
48+
49+
nlb > vpc_endpoint_service
50+
vpc_endpoint_service -> vpc_endpoint_1: PrivateLink Connection
51+
vpc_endpoint_1 <> online_feature_consumption: Feature Serving
52+
vpc_endpoint_1 > offline_feature_consumption: Tecton API
53+
54+
feature_serving > nlb
55+
orchestration > cross_account_role: AssumeRole
56+
cross_account_role > offline_feature_store
57+
58+
59+
feature_serving > cross_account_role: Online Store Access (AssumeRole) [color: Black]
60+
cross_account_role > online_feature_store
61+
offline_feature_store <> offline_feature_consumption
62+
orchestration > rift_compute_manager_role: AssumeRole
63+
orchestration > rift_environments_repo: Push Environments Images
64+
rift_environments_repo > rift_compute
65+
rift_compute_manager_role > rift_compute
66+
rift_compute > rift_compute_role
67+
rift_compute_role > offline_feature_store: Materialization
68+
rift_compute > rift_compute_role
69+
rift_compute_role > online_feature_store: Materialization
70+
71+
stream_ingest_api > ingest_nlb
72+
ingest_nlb > ingest_vpc_endpoint_service
73+
ingest_vpc_endpoint_service <> vpc_endpoint_2
74+
vpc_endpoint_2 < streaming_ingest_app
75+
stream_ingest_api > offline_feature_store
76+
stream_ingest_api > online_feature_store

‎diagrams/databricks.txt‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
direction: right
2+
tecton_virtual_private_tecton [label: "Tecton Control-Plane AWS ACcount", color: magenta, icon: globe] {
3+
control_plane_vpc [label: "Control Plane VPC", color: purple, icon: aws-vpc] {
4+
tecton_control_plane_app_metadata [label: "Tecton Control Plane Application & Metadata", color: gray, icon: monitor] {
5+
web_ui [label: "Web UI", icon: monitor]
6+
metadata_api [label: "Metadata API", icon: code]
7+
metrics_api [label: "Metrics API", icon: prometheus]
8+
feature_serving [label: "Feature Serving", icon: service]
9+
orchestration [label: "Orchestration", icon: settings]
10+
stream_ingest_api [icon: aws-ec2, label: "Stream Ingest API"]
11+
}
12+
nlb [label: "NLB", icon: aws-elastic-load-balancing]
13+
ingest_nlb [label: "Ingest NLB", icon: aws-elastic-load-balancing]
14+
vpc_endpoint_service [label: "VPC Endpoint Service (PrivateLink)", icon: aws-vpc-endpoint]
15+
ingest_vpc_endpoint_service [label: "Ingest VPC Endpoint Service (PrivateLink)", icon: aws-vpc-endpoint]
16+
}
17+
}
18+
19+
customer_aws_account [label: "Customer AWS Account", color: orange, icon: aws-account] {
20+
online_feature_store [label: "Online feature storage\nDynamoDB tables", icon: aws-dynamodb]
21+
cross_account_role [label: "Cross Account Role", icon: aws-iam-role]
22+
offline_feature_store [label: "Offline feature storage\nS3", icon: aws-s3]
23+
customer_vpc [label: "VPC", color: lightblue, icon: aws-vpc] {
24+
online_feature_consumption [label: "Online Feature consumption\nML App/Service", icon: monitor]
25+
offline_feature_consumption [label: "Offline Feature consumption\nDevelopment tools (Notebook)", icon: book-open]
26+
vpc_endpoint_1 [label: "VPC Endpoint", icon: aws-vpc-endpoint]
27+
}
28+
29+
VPC [label: "VPC", color: lightblue, icon: aws-vpc] {
30+
vpc_endpoint_2 [label: "VPC Endpoint", icon: aws-vpc-endpoint]
31+
streaming_ingest_app [icon: kafka, label: "Streaming / Realtime Application"]
32+
}
33+
34+
35+
databricks_vpc [icon: aws-vpc, color: blue, label: "Databricks VPC"] {
36+
databricks_compute [icon: databricks, label: "Databricks Spark Compute"]
37+
}
38+
39+
emr_spark_role [icon: aws-iam-role, label: "Spark Role"]
40+
41+
42+
}
43+
44+
// connections
45+
databricks_compute > emr_spark_role
46+
emr_spark_role > offline_feature_store: Materialization
47+
emr_spark_role > online_feature_store: Materialization
48+
49+
web_ui > nlb
50+
metadata_api > nlb
51+
metrics_api > nlb
52+
53+
nlb > vpc_endpoint_service
54+
vpc_endpoint_service -> vpc_endpoint_1: PrivateLink Connection
55+
vpc_endpoint_1 <> online_feature_consumption: FeatureServing
56+
vpc_endpoint_1 > offline_feature_consumption: Tecton API
57+
58+
feature_serving > nlb
59+
orchestration > cross_account_role: AssumeRole
60+
cross_account_role > offline_feature_store
61+
62+
online_feature_consumption > online_feature_store
63+
64+
feature_serving > cross_account_role: AssumeRole [color: Black]
65+
cross_account_role > online_feature_store
66+
offline_feature_store <> offline_feature_consumption
67+
cross_account_role > databricks_compute
68+
69+
70+
stream_ingest_api > ingest_nlb
71+
ingest_nlb > ingest_vpc_endpoint_service
72+
ingest_vpc_endpoint_service <> vpc_endpoint_2
73+
vpc_endpoint_2 < streaming_ingest_app
74+
stream_ingest_api > offline_feature_store
75+
stream_ingest_api > online_feature_store

‎diagrams/dataplane_rift.txt‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
direction: right
2+
tecton_virtual_private_tecton [label: "Tecton Control-Plane AWS ACcount", color: magenta, icon: globe] {
3+
control_plane_vpc [label: "Control Plane VPC", color: purple, icon: aws-vpc] {
4+
tecton_control_plane_app_metadata [label: "Tecton Control Plane Applications", color: gray, icon: monitor] {
5+
web_ui [label: "Web UI", icon: monitor]
6+
metadata_api [label: "Metadata API", icon: code]
7+
metrics_api [label: "Metrics API", icon: prometheus]
8+
feature_serving [label: "Feature Serving", icon: service]
9+
orchestration [label: "Orchestration", icon: settings]
10+
stream_ingest_api [icon: aws-ec2, label: "Stream Ingest API"]
11+
}
12+
nlb [label: "NLB", icon: aws-elastic-load-balancing]
13+
ingest_nlb [label: "Ingest NLB", icon: aws-elastic-load-balancing]
14+
vpc_endpoint_service [label: "VPC Endpoint Service (PrivateLink)", icon: aws-vpc-endpoint]
15+
ingest_vpc_endpoint_service [label: "Ingest VPC Endpoint Service (PrivateLink)", icon: aws-vpc-endpoint]
16+
17+
}
18+
}
19+
20+
customer_aws_account [label: "Customer AWS Account", color: orange, icon: aws-account] {
21+
online_feature_store [label: "Online feature storage\nDynamoDB tables", icon: aws-dynamodb]
22+
cross_account_role [label: "Cross Account Role", icon: aws-iam-role]
23+
offline_feature_store [label: "Offline feature storage\nS3", icon: aws-s3]
24+
rift_environments_repo[label: "Rift Environments Repository", icon: aws-ecr]
25+
customer_vpc [label: "VPC", color: lightblue, icon: aws-vpc] {
26+
online_feature_consumption [label: "Online Feature consumption\nML App/Service", icon: monitor]
27+
offline_feature_consumption [label: "Offline Feature consumption\nDevelopment tools (Notebook)", icon: book-open]
28+
vpc_endpoint_1 [label: "VPC Endpoint", icon: aws-vpc-endpoint]
29+
}
30+
31+
VPC [label: "VPC", color: lightblue, icon: aws-vpc] {
32+
vpc_endpoint_2 [label: "VPC Endpoint", icon: aws-vpc-endpoint]
33+
streaming_ingest_app [icon: kafka, label: "Streaming / Realtime Application"]
34+
}
35+
36+
37+
38+
Rift VPC [color: blue, icon: aws-vpc] {
39+
rift_compute [label: "Rift Compute", icon: aws-ec2-instance]
40+
}
41+
rift_compute_role [label: "Rift Compute Role", icon: aws-iam-role]
42+
rift_compute_manager_role [label: "Rift Compute Manager Role", icon: aws-iam-role]
43+
44+
}
45+
46+
// connections
47+
web_ui > nlb
48+
metadata_api > nlb
49+
metrics_api > nlb
50+
51+
nlb > vpc_endpoint_service
52+
vpc_endpoint_service -> vpc_endpoint_1: PrivateLink Connection
53+
vpc_endpoint_1 <> online_feature_consumption: FeatureServing
54+
vpc_endpoint_1 > offline_feature_consumption: Tecton API
55+
56+
feature_serving > nlb
57+
orchestration > cross_account_role: AssumeRole
58+
cross_account_role > offline_feature_store
59+
60+
online_feature_consumption > online_feature_store
61+
62+
feature_serving > cross_account_role: AssumeRole [color: Black]
63+
cross_account_role > online_feature_store
64+
offline_feature_store <> offline_feature_consumption
65+
orchestration -> rift_compute_manager_role: AssumeRole
66+
orchestration > rift_environments_repo: Push Environments Images
67+
rift_environments_repo > rift_compute
68+
rift_compute_manager_role > rift_compute
69+
rift_compute > rift_compute_role
70+
rift_compute_role > offline_feature_store: Materialization
71+
rift_compute > rift_compute_role
72+
rift_compute_role > online_feature_store: Materialization
73+
74+
stream_ingest_api > ingest_nlb
75+
ingest_nlb > ingest_vpc_endpoint_service
76+
ingest_vpc_endpoint_service <> vpc_endpoint_2
77+
vpc_endpoint_2 < streaming_ingest_app
78+
stream_ingest_api > offline_feature_store
79+
stream_ingest_api > online_feature_store
80+
// Dotted lines (conceptual/data flow)

‎diagrams/dataplane_rift_emr.txt‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
direction: right
2+
tecton_virtual_private_tecton [label: "Tecton Control-Plane AWS ACcount", color: magenta, icon: globe] {
3+
control_plane_vpc [label: "Control Plane VPC", color: purple, icon: aws-vpc] {
4+
tecton_control_plane_app_metadata [label: "Tecton Control Plane Applications", color: gray, icon: monitor] {
5+
web_ui [label: "Web UI", icon: monitor]
6+
metadata_api [label: "Metadata API", icon: code]
7+
metrics_api [label: "Metrics API", icon: prometheus]
8+
feature_serving [label: "Feature Serving", icon: service]
9+
orchestration [label: "Orchestration", icon: settings]
10+
stream_ingest_api [icon: aws-ec2, label: "Stream Ingest API"]
11+
}
12+
nlb [label: "NLB", icon: aws-elastic-load-balancing]
13+
ingest_nlb [label: "Ingest NLB", icon: aws-elastic-load-balancing]
14+
vpc_endpoint_service [label: "VPC Endpoint Service (PrivateLink)", icon: aws-vpc-endpoint]
15+
ingest_vpc_endpoint_service [label: "Ingest VPC Endpoint Service (PrivateLink)", icon: aws-vpc-endpoint]
16+
}
17+
}
18+
19+
customer_aws_account [label: "Customer AWS Account", color: orange, icon: aws-account] {
20+
online_feature_store [label: "Online feature storage\nDynamoDB tables", icon: aws-dynamodb]
21+
cross_account_role [label: "Cross Account Role", icon: aws-iam-role]
22+
offline_feature_store [label: "Offline feature storage\nS3", icon: aws-s3]
23+
rift_environments_repo[label: "Rift Environments Repository", icon: aws-ecr]
24+
customer_vpc [label: "VPC", color: lightblue, icon: aws-vpc] {
25+
online_feature_consumption [label: "Online Feature consumption\nML App/Service", icon: monitor]
26+
offline_feature_consumption [label: "Offline Feature consumption\nDevelopment tools (Notebook)", icon: book-open]
27+
vpc_endpoint_1 [label: "VPC Endpoint", icon: aws-vpc-endpoint]
28+
}
29+
30+
Rift VPC [color: blue, icon: aws-vpc] {
31+
rift_compute [label: "Rift Compute", icon: aws-ec2-instance]
32+
}
33+
rift_compute_role [label: "Rift Compute Role", icon: aws-iam-role]
34+
rift_compute_manager_role [label: "Rift Compute Manager Role", icon: aws-iam-role]
35+
36+
VPC [label: "VPC", color: lightblue, icon: aws-vpc] {
37+
vpc_endpoint_2 [label: "VPC Endpoint", icon: aws-vpc-endpoint]
38+
streaming_ingest_app [icon: kafka, label: "Streaming / Realtime Application"]
39+
}
40+
41+
42+
emr_vpc [icon: aws-vpc, color: blue, label: "EMR VPC"] {
43+
emr_compute [icon: aws-emr, label: "EMR Compute"]
44+
}
45+
46+
emr_spark_role [icon: aws-iam-role, label: "Spark Role"]
47+
48+
49+
}
50+
51+
// connections
52+
emr_compute -> emr_spark_role
53+
emr_spark_role > offline_feature_store: Materialization
54+
emr_spark_role > online_feature_store: Materialization
55+
56+
web_ui > nlb
57+
metadata_api > nlb
58+
metrics_api > nlb
59+
60+
nlb > vpc_endpoint_service
61+
vpc_endpoint_service -> vpc_endpoint_1: PrivateLink Connection
62+
vpc_endpoint_1 <> online_feature_consumption: FeatureServing
63+
vpc_endpoint_1 > offline_feature_consumption: Tecton API
64+
65+
feature_serving > nlb
66+
orchestration > cross_account_role: AssumeRole
67+
cross_account_role > offline_feature_store
68+
69+
online_feature_consumption > online_feature_store
70+
71+
feature_serving > cross_account_role: AssumeRole [color: Black]
72+
cross_account_role > online_feature_store
73+
offline_feature_store <> offline_feature_consumption
74+
orchestration > rift_compute_manager_role: AssumeRole
75+
orchestration > rift_environments_repo: Push Environments Images
76+
cross_account_role > emr_compute
77+
rift_environments_repo > rift_compute
78+
rift_compute_manager_role > rift_compute
79+
rift_compute > rift_compute_role
80+
rift_compute_role > offline_feature_store: Materialization
81+
rift_compute > rift_compute_role
82+
rift_compute_role > online_feature_store: Materialization
83+
84+
85+
stream_ingest_api > ingest_nlb
86+
ingest_nlb > ingest_vpc_endpoint_service
87+
ingest_vpc_endpoint_service <> vpc_endpoint_2
88+
vpc_endpoint_2 < streaming_ingest_app
89+
stream_ingest_api > offline_feature_store
90+
stream_ingest_api > online_feature_store

0 commit comments

Comments
 (0)