Skip to content

Commit c341c76

Browse files
isala404isala404
committed
feat(control-plane): Added support for sherlock module
1 parent 432a9df commit c341c76

File tree

9 files changed

+186
-87
lines changed

9 files changed

+186
-87
lines changed

‎control-plane/api/v1alpha1/inspector_types.go

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@ type InspectorSpec struct {
3939
DeploymentRef string `json:"deploymentRef"`
4040
ServiceRef string `json:"serviceRef"`
4141
Namespace string `json:"namespace"`
42-
ModelURI string `json:"modelURI"`
4342
}
4443

4544
type Status string

‎control-plane/config/crd/bases/lazykoala.isala.me_inspectors.yaml

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,15 +55,12 @@ spec:
5555
description: Foo is an example field of Inspector. Edit inspector_types.go
5656
to remove/update
5757
type: string
58-
modelURI:
59-
type: string
6058
namespace:
6159
type: string
6260
serviceRef:
6361
type: string
6462
required:
6563
- deploymentRef
66-
- modelURI
6764
- namespace
6865
- serviceRef
6966
type: object

‎control-plane/config/samples/lazykoala_v1alpha1_inspector.yaml

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -4,42 +4,38 @@ metadata:
44
name: service-1
55
namespace: default
66
spec:
7-
deploymentRef: service-1-0e762959
8-
serviceRef: service-1-0e762959
7+
deploymentRef: service-1-05588114
8+
serviceRef: service-1-05588114
99
namespace: default
10-
modelURI: path/to/checkpoint.ckpt
1110
---
1211
apiVersion: lazykoala.isala.me/v1alpha1
1312
kind: Inspector
1413
metadata:
1514
name: service-2
1615
namespace: default
1716
spec:
18-
deploymentRef: service-2-0e762959
19-
serviceRef: service-2-0e762959
17+
deploymentRef: service-2-05588114
18+
serviceRef: service-2-05588114
2019
namespace: default
21-
modelURI: path/to/checkpoint.ckpt
2220
---
2321
apiVersion: lazykoala.isala.me/v1alpha1
2422
kind: Inspector
2523
metadata:
2624
name: service-3
2725
namespace: default
2826
spec:
29-
deploymentRef: service-3-0e762959
30-
serviceRef: service-3-0e762959
27+
deploymentRef: service-3-05588114
28+
serviceRef: service-3-05588114
3129
namespace: default
32-
modelURI: path/to/checkpoint.ckpt
3330
---
3431
apiVersion: lazykoala.isala.me/v1alpha1
3532
kind: Inspector
3633
metadata:
3734
name: service-4
3835
namespace: default
3936
spec:
40-
deploymentRef: service-4-0e762959
41-
serviceRef: service-4-0e762959
37+
deploymentRef: service-4-05588114
38+
serviceRef: service-4-05588114
4239
namespace: default
43-
modelURI: path/to/checkpoint.ckpt
4440
---
4541

‎control-plane/controllers/inspector_controller.go

Lines changed: 163 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,14 @@ limitations under the License.
1717
package controllers
1818

1919
import (
20+
"bytes"
2021
"context"
2122
"fmt"
23+
appsv1 "k8s.io/api/apps/v1"
24+
"text/template"
2225
"time"
2326

2427
"gopkg.in/yaml.v3"
25-
appsv1 "k8s.io/api/apps/v1"
2628
v1 "k8s.io/api/core/v1"
2729
"k8s.io/apimachinery/pkg/runtime"
2830
"k8s.io/apimachinery/pkg/types"
@@ -89,6 +91,7 @@ func (r *InspectorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
8991
if err := r.Update(ctx, &inspector); err != nil {
9092
return ctrl.Result{}, err
9193
}
94+
return ctrl.Result{Requeue: true}, nil
9295
}
9396
} else {
9497
// The object is being deleted
@@ -100,6 +103,10 @@ func (r *InspectorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
100103
return ctrl.Result{}, err
101104
}
102105

106+
if err := r.configureSherlock(ctx, &inspector, false); err != nil {
107+
return ctrl.Result{}, err
108+
}
109+
103110
// remove our finalizer from the list and update it.
104111
controllerutil.RemoveFinalizer(&inspector, finalizerName)
105112
if err := r.Update(ctx, &inspector); err != nil {
@@ -110,13 +117,80 @@ func (r *InspectorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
110117
// Stop reconciliation as the item is being deleted
111118
return ctrl.Result{}, nil
112119
}
120+
121+
scrapePoints, err := r.configureGazer(ctx, &inspector)
122+
if err != nil {
123+
return ctrl.Result{}, err
124+
}
125+
126+
if err := r.configureSherlock(ctx, &inspector, true); err != nil {
127+
return ctrl.Result{}, err
128+
}
129+
130+
// Update local status
131+
var MonitoredIPs []string
132+
for k := range scrapePoints {
133+
MonitoredIPs = append(MonitoredIPs, k)
134+
}
135+
inspector.Status.MonitoredIPs = MonitoredIPs
136+
inspector.Status.Status = lazykoalav1alpha1.Running
137+
138+
if err := r.Status().Update(ctx, &inspector); err != nil {
139+
return ctrl.Result{}, err
140+
}
141+
142+
return ctrl.Result{RequeueAfter: time.Minute}, nil
143+
}
144+
145+
func (r *InspectorReconciler) removeMonitoredIPs(inspector *lazykoalav1alpha1.Inspector) error {
146+
// --------------------- START OF GAZER CONFIG --------------------- //
147+
// Get the Gazer config file
148+
var configMap v1.ConfigMap
149+
if err := r.Get(context.Background(), types.NamespacedName{
150+
Namespace: "lazy-koala",
151+
Name: "gazer-config",
152+
}, &configMap); err != nil {
153+
return err
154+
}
155+
156+
// Phase the config.yaml
157+
configData := make(map[string]ScrapePoint)
158+
if err := yaml.Unmarshal([]byte(configMap.Data["config.yaml"]), &configData); err != nil {
159+
return err
160+
}
161+
162+
// Remove all the existing scrape points created from this Inspector
163+
for _, ip := range inspector.Status.MonitoredIPs {
164+
if _, ok := configData[ip]; ok {
165+
delete(configData, ip)
166+
}
167+
}
168+
169+
// Encode the config.yaml
170+
encodedConfig, err := yaml.Marshal(&configData)
171+
if err != nil {
172+
return err
173+
}
174+
175+
// Patch the config file
176+
configMap.Data["config.yaml"] = string(encodedConfig)
177+
if err := r.Update(context.Background(), &configMap); err != nil {
178+
return err
179+
}
180+
181+
return nil
182+
}
183+
184+
func (r *InspectorReconciler) configureGazer(ctx context.Context, inspector *lazykoalav1alpha1.Inspector) (map[string]ScrapePoint, error) {
185+
logger := log.FromContext(ctx)
186+
113187
// Get the intended deployment
114188
var deploymentRef appsv1.Deployment
115189
if err := r.Get(ctx, types.NamespacedName{
116190
Namespace: inspector.Spec.Namespace,
117191
Name: inspector.Spec.DeploymentRef,
118192
}, &deploymentRef); err != nil {
119-
return ctrl.Result{}, err
193+
return nil, err
120194
}
121195

122196
scrapePoints := make(map[string]ScrapePoint)
@@ -127,7 +201,7 @@ func (r *InspectorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
127201
var podList v1.PodList
128202
if err := r.List(ctx, &podList, &selector); client.IgnoreNotFound(err) != nil {
129203
logger.Error(err, fmt.Sprintf("failed to pods for deployment %s", deploymentRef.ObjectMeta.Name))
130-
return ctrl.Result{}, err
204+
return nil, err
131205
}
132206

133207
// Create Scrape point for each pod
@@ -147,7 +221,7 @@ func (r *InspectorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
147221
Namespace: inspector.Spec.Namespace,
148222
Name: inspector.Spec.ServiceRef,
149223
}, &serviceRef); err != nil {
150-
return ctrl.Result{}, err
224+
return nil, err
151225
}
152226

153227
scrapePoints[serviceRef.Spec.ClusterIP] = ScrapePoint{
@@ -159,91 +233,89 @@ func (r *InspectorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
159233
}
160234

161235
// Get the Gazer config file
162-
var configMap v1.ConfigMap
236+
var gazerConfigMap v1.ConfigMap
163237
if err := r.Get(ctx, types.NamespacedName{
164238
Namespace: "lazy-koala",
165239
Name: "gazer-config",
166-
}, &configMap); err != nil {
167-
return ctrl.Result{}, err
240+
}, &gazerConfigMap); err != nil {
241+
return nil, err
168242
}
169243

170244
// Phase the config.yaml
171-
configData := make(map[string]ScrapePoint)
172-
if err := yaml.Unmarshal([]byte(configMap.Data["config.yaml"]), &configData); err != nil {
173-
return ctrl.Result{}, err
245+
gazerData := make(map[string]ScrapePoint)
246+
if err := yaml.Unmarshal([]byte(gazerConfigMap.Data["config.yaml"]), &gazerData); err != nil {
247+
return nil, err
174248
}
175249

176250
// Remove all the existing scrape points created from this Inspector
177251
for _, ip := range inspector.Status.MonitoredIPs {
178-
if _, ok := configData[ip]; ok {
179-
delete(configData, ip)
252+
if _, ok := gazerData[ip]; ok {
253+
delete(gazerData, ip)
180254
}
181255
}
182256

183257
// Add the new scrape points
184258
for k, v := range scrapePoints {
185-
configData[k] = v
259+
gazerData[k] = v
186260
}
187261

188262
// Encode the config.yaml
189-
encodedConfig, err := yaml.Marshal(&configData)
263+
encodedConfig, err := yaml.Marshal(&gazerData)
190264
if err != nil {
191-
return ctrl.Result{}, err
265+
return nil, err
192266
}
193267

194268
// Patch the config file
195-
configMap.Data["config.yaml"] = string(encodedConfig)
196-
if err := r.Update(ctx, &configMap); err != nil {
197-
return ctrl.Result{}, err
269+
gazerConfigMap.Data["config.yaml"] = string(encodedConfig)
270+
if err := r.Update(ctx, &gazerConfigMap); err != nil {
271+
return nil, err
198272
}
199273

200-
// Update local status
201-
var MonitoredIPs []string
202-
for k := range scrapePoints {
203-
MonitoredIPs = append(MonitoredIPs, k)
204-
}
205-
inspector.Status.MonitoredIPs = MonitoredIPs
206-
inspector.Status.Status = lazykoalav1alpha1.Running
207-
208-
if err := r.Status().Update(ctx, &inspector); err != nil {
209-
return ctrl.Result{}, err
210-
}
211-
212-
return ctrl.Result{RequeueAfter: time.Minute}, nil
274+
return scrapePoints, nil
213275
}
214276

215-
func (r *InspectorReconciler) removeMonitoredIPs(inspector *lazykoalav1alpha1.Inspector) error {
216-
// Get the Gazer config file
217-
var configMap v1.ConfigMap
218-
if err := r.Get(context.Background(), types.NamespacedName{
277+
func (r *InspectorReconciler) configureSherlock(ctx context.Context, inspector *lazykoalav1alpha1.Inspector, append bool) error {
278+
// Get the Sherlock config file
279+
var sherlockConfigMap v1.ConfigMap
280+
if err := r.Get(ctx, types.NamespacedName{
219281
Namespace: "lazy-koala",
220-
Name: "gazer-config",
221-
}, &configMap); err != nil {
282+
Name: "sherlock-config",
283+
}, &sherlockConfigMap); err != nil {
222284
return err
223285
}
224286

225-
// Phase the config.yaml
226-
configData := make(map[string]ScrapePoint)
227-
if err := yaml.Unmarshal([]byte(configMap.Data["config.yaml"]), &configData); err != nil {
287+
// Phase the services.yaml
288+
var sherlockServiceList []string
289+
if err := yaml.Unmarshal([]byte(sherlockConfigMap.Data["services.yaml"]), &sherlockServiceList); err != nil {
228290
return err
229291
}
230292

231-
// Remove all the existing scrape points created from this Inspector
232-
for _, ip := range inspector.Status.MonitoredIPs {
233-
if _, ok := configData[ip]; ok {
234-
delete(configData, ip)
235-
}
293+
if append {
294+
sherlockServiceList = AppendIfMissing(sherlockServiceList, inspector.Spec.DeploymentRef)
295+
} else {
296+
sherlockServiceList = RemoveIfExists(sherlockServiceList, inspector.Spec.DeploymentRef)
236297
}
237298

238-
// Encode the config.yaml
239-
encodedConfig, err := yaml.Marshal(&configData)
299+
// Generate the Servings Config
300+
servingsConfig, err := createServingsConfig(sherlockServiceList)
240301
if err != nil {
241302
return err
242303
}
243304

244-
// Patch the config file
245-
configMap.Data["config.yaml"] = string(encodedConfig)
246-
if err := r.Update(context.Background(), &configMap); err != nil {
305+
if err := r.Update(ctx, &sherlockConfigMap); err != nil {
306+
return err
307+
}
308+
309+
// Encode the services.yaml
310+
encodedSherlockConfig, err := yaml.Marshal(&sherlockServiceList)
311+
if err != nil {
312+
return err
313+
}
314+
315+
sherlockConfigMap.Data["services.yaml"] = string(encodedSherlockConfig)
316+
sherlockConfigMap.Data["models.config"] = servingsConfig
317+
318+
if err := r.Update(ctx, &sherlockConfigMap); err != nil {
247319
return err
248320
}
249321
return nil
@@ -265,3 +337,44 @@ func eventFilter() predicate.Predicate {
265337
},
266338
}
267339
}
340+
341+
func AppendIfMissing(slice []string, item string) []string {
342+
for _, ele := range slice {
343+
if ele == item {
344+
return slice
345+
}
346+
}
347+
return append(slice, item)
348+
}
349+
350+
func RemoveIfExists(slice []string, item string) []string {
351+
for i, other := range slice {
352+
if other == item {
353+
return append(slice[:i], slice[i+1:]...)
354+
}
355+
}
356+
return slice
357+
}
358+
359+
func createServingsConfig(service []string) (string, error) {
360+
tmpl := template.New("config")
361+
362+
tmpl, err := tmpl.Parse(`model_config_list {
363+
{{range .}}
364+
config {
365+
name: '{{.}}'
366+
base_path: '/models/{{.}}/'
367+
model_platform: 'tensorflow'
368+
}
369+
{{end}}
370+
}`)
371+
if err != nil {
372+
return "", err
373+
}
374+
buf := new(bytes.Buffer)
375+
err = tmpl.Execute(buf, service)
376+
if err != nil {
377+
return "", err
378+
}
379+
return fmt.Sprintf("%+v", buf), nil
380+
}

0 commit comments

Comments
 (0)