|
| 1 | +# usage: ./pipeline-tests.sh [sleepTime] [kubectl|seldon] [namespace] |
| 2 | + |
| 3 | +if [ -z "$1" ] |
| 4 | +then |
| 5 | + sleepTime=5 |
| 6 | +else |
| 7 | + sleepTime=$1 |
| 8 | +fi |
| 9 | + |
| 10 | +if [ -z "$2" ] |
| 11 | +then |
| 12 | + cmd="kubectl" |
| 13 | +else |
| 14 | + cmd=$2 |
| 15 | +fi |
| 16 | + |
| 17 | +if [ -z "$3" ] |
| 18 | +then |
| 19 | + namespace="seldon-mesh" |
| 20 | +else |
| 21 | + namespace=$3 |
| 22 | +fi |
| 23 | + |
| 24 | +function load() { |
| 25 | + if [ $cmd == "kubectl" ] |
| 26 | + then |
| 27 | + kubectl apply -f $2 -n $namespace |
| 28 | + else |
| 29 | + if [ $1 == "model" ] |
| 30 | + then |
| 31 | + seldon model load -f $2 |
| 32 | + elif [ $1 == "pipeline" ] |
| 33 | + then |
| 34 | + seldon pipeline load -f $2 |
| 35 | + elif [ $1 == "experiment" ] |
| 36 | + then |
| 37 | + seldon experiment start -f $2 |
| 38 | + fi |
| 39 | + fi |
| 40 | +} |
| 41 | + |
| 42 | +function unload() { |
| 43 | + # note that in k8s we need to use the filepath (3rd argument) |
| 44 | + if [ $cmd == "kubectl" ] |
| 45 | + then |
| 46 | + kubectl delete -f $3 -n $namespace |
| 47 | + else |
| 48 | + if [ $1 == "model" ] |
| 49 | + then |
| 50 | + seldon model unload $2 |
| 51 | + elif [ $1 == "pipeline" ] |
| 52 | + then |
| 53 | + seldon pipeline unload $2 |
| 54 | + elif [ $1 == "experiment" ] |
| 55 | + then |
| 56 | + seldon experiment stop $2 |
| 57 | + fi |
| 58 | + fi |
| 59 | +} |
| 60 | + |
| 61 | +function status() { |
| 62 | + if [ $cmd == "kubectl" ] |
| 63 | + then |
| 64 | + if [ $1 == "model" ] |
| 65 | + then |
| 66 | + kubectl wait --for condition=ready --timeout=300s model $2 -n $namespace |
| 67 | + elif [ $1 == "pipeline" ] |
| 68 | + then |
| 69 | + kubectl wait --for condition=ready --timeout=300s pipeline $2 -n $namespace |
| 70 | + elif [ $1 == "experiment" ] |
| 71 | + then |
| 72 | + kubectl wait --for condition=ready --timeout=300s experiment $2 -n $namespace |
| 73 | + fi |
| 74 | + else |
| 75 | + if [ $1 == "model" ] |
| 76 | + then |
| 77 | + seldon model status $2 -w ModelAvailable | jq -M . |
| 78 | + elif [ $1 == "pipeline" ] |
| 79 | + then |
| 80 | + seldon pipeline status $2 -w PipelineReady | jq -M . |
| 81 | + elif [ $1 == "experiment" ] |
| 82 | + then |
| 83 | + seldon experiment status $2 -w | jq -M . |
| 84 | + fi |
| 85 | + fi |
| 86 | +} |
| 87 | + |
| 88 | + |
| 89 | +load model ./models/tfsimple1.yaml |
| 90 | +load model ./models/tfsimple2.yaml |
| 91 | +status model tfsimple1 |
| 92 | +status model tfsimple2 |
| 93 | +load "pipeline" ./pipelines/tfsimples.yaml |
| 94 | +status pipeline tfsimples |
| 95 | +seldon pipeline infer tfsimples '{"inputs":[{"name":"INPUT0","data":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],"datatype":"INT32","shape":[1,16]}]}' |
| 96 | +seldon pipeline infer tfsimples --inference-mode grpc '{"model_name":"simple","inputs":[{"name":"INPUT0","contents":{"int_contents":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]},"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","contents":{"int_contents":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]},"datatype":"INT32","shape":[1,16]}]}' |
| 97 | +unload pipeline tfsimples ./pipelines/tfsimples.yaml |
| 98 | +unload model tfsimple1 ./models/tfsimple1.yaml |
| 99 | +unload model tfsimple2 ./models/tfsimple2.yaml |
| 100 | + |
| 101 | +sleep $sleepTime |
| 102 | + |
| 103 | +load model ./models/tfsimple1.yaml |
| 104 | +load model ./models/tfsimple2.yaml |
| 105 | +load model ./models/tfsimple3.yaml |
| 106 | +status model tfsimple1 |
| 107 | +status model tfsimple2 |
| 108 | +status model tfsimple3 |
| 109 | +load pipeline ./pipelines/tfsimples-join.yaml |
| 110 | +status pipeline join |
| 111 | +seldon pipeline infer join --inference-mode grpc '{"model_name":"simple","inputs":[{"name":"INPUT0","contents":{"int_contents":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]},"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","contents":{"int_contents":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]},"datatype":"INT32","shape":[1,16]}]}' |
| 112 | +unload pipeline join ./pipelines/tfsimples-join.yaml |
| 113 | +unload model tfsimple1 ./models/tfsimple1.yaml |
| 114 | +unload model tfsimple2 ./models/tfsimple2.yaml |
| 115 | +unload model tfsimple3 ./models/tfsimple3.yaml |
| 116 | + |
| 117 | +sleep $sleepTime |
| 118 | + |
| 119 | +load model ./models/conditional.yaml |
| 120 | +load model ./models/add10.yaml |
| 121 | +load model ./models/mul10.yaml |
| 122 | +status model conditional |
| 123 | +status model add10 |
| 124 | +status model mul10 |
| 125 | +load pipeline ./pipelines/conditional.yaml |
| 126 | +status pipeline tfsimple-conditional |
| 127 | +seldon pipeline infer tfsimple-conditional --inference-mode grpc '{"model_name":"outlier","inputs":[{"name":"CHOICE","contents":{"int_contents":[0]},"datatype":"INT32","shape":[1]},{"name":"INPUT0","contents":{"fp32_contents":[1,2,3,4]},"datatype":"FP32","shape":[4]},{"name":"INPUT1","contents":{"fp32_contents":[1,2,3,4]},"datatype":"FP32","shape":[4]}]}' |
| 128 | +seldon pipeline infer tfsimple-conditional --inference-mode grpc '{"model_name":"outlier","inputs":[{"name":"CHOICE","contents":{"int_contents":[1]},"datatype":"INT32","shape":[1]},{"name":"INPUT0","contents":{"fp32_contents":[1,2,3,4]},"datatype":"FP32","shape":[4]},{"name":"INPUT1","contents":{"fp32_contents":[1,2,3,4]},"datatype":"FP32","shape":[4]}]}' |
| 129 | +unload pipeline tfsimple-conditional ./pipelines/conditional.yaml |
| 130 | +unload model conditional ./models/conditional.yaml |
| 131 | +unload model add10 ./models/add10.yaml |
| 132 | +unload model mul10 ./models/mul10.yaml |
| 133 | + |
| 134 | +sleep $sleepTime |
| 135 | + |
| 136 | +load model ./models/outlier-error.yaml |
| 137 | +status model outlier-error |
| 138 | +load pipeline ./pipelines/error.yaml |
| 139 | +status pipeline error |
| 140 | +seldon pipeline infer error --inference-mode grpc '{"model_name":"outlier","inputs":[{"name":"INPUT","contents":{"fp32_contents":[1,2,3,4]},"datatype":"FP32","shape":[4]}]}' |
| 141 | +seldon pipeline infer error --inference-mode grpc '{"model_name":"outlier","inputs":[{"name":"INPUT","contents":{"fp32_contents":[100,2,3,4]},"datatype":"FP32","shape":[4]}]}' |
| 142 | +unload pipeline error ./models/outlier-error.yaml |
| 143 | +unload model outlier-error ./pipelines/error.yaml |
| 144 | + |
| 145 | +sleep $sleepTime |
| 146 | + |
| 147 | +load model ./models/tfsimple1.yaml |
| 148 | +load model ./models/tfsimple2.yaml |
| 149 | +load model ./models/tfsimple3.yaml |
| 150 | +load model ./models/check.yaml |
| 151 | +status model tfsimple1 |
| 152 | +status model tfsimple2 |
| 153 | +status model tfsimple3 |
| 154 | +status model check |
| 155 | +load pipeline ./pipelines/tfsimples-join-outlier.yaml |
| 156 | +status pipeline joincheck |
| 157 | +seldon pipeline infer joincheck --inference-mode grpc '{"model_name":"simple","inputs":[{"name":"INPUT0","contents":{"int_contents":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","contents":{"int_contents":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},"datatype":"INT32","shape":[1,16]}]}' |
| 158 | +seldon pipeline infer joincheck --inference-mode grpc '{"model_name":"simple","inputs":[{"name":"INPUT0","contents":{"int_contents":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]},"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","contents":{"int_contents":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]},"datatype":"INT32","shape":[1,16]}]}' |
| 159 | +unload pipeline joincheck ./pipelines/tfsimples-join-outlier.yaml |
| 160 | +unload model tfsimple1 ./models/tfsimple1.yaml |
| 161 | +unload model tfsimple2 ./models/tfsimple2.yaml |
| 162 | +unload model tfsimple3 ./models/tfsimple3.yaml |
| 163 | +unload model check ./models/check.yaml |
| 164 | + |
| 165 | +sleep $sleepTime |
| 166 | + |
| 167 | +load model ./models/mul10.yaml |
| 168 | +load model ./models/add10.yaml |
| 169 | +status model mul10 |
| 170 | +status model add10 |
| 171 | +load pipeline ./pipelines/pipeline-inputs.yaml |
| 172 | +status pipeline pipeline-inputs |
| 173 | +seldon pipeline infer pipeline-inputs --inference-mode grpc '{"model_name":"pipeline","inputs":[{"name":"INPUT0","contents":{"fp32_contents":[1,2,3,4]},"datatype":"FP32","shape":[4]},{"name":"INPUT1","contents":{"fp32_contents":[1,2,3,4]},"datatype":"FP32","shape":[4]}]}' | jq -M . |
| 174 | +unload pipeline pipeline-inputs ./pipelines/pipeline-inputs.yaml |
| 175 | +unload model mul10 ./models/mul10.yaml |
| 176 | +unload model add10 ./models/add10.yaml |
| 177 | + |
| 178 | +sleep $sleepTime |
| 179 | + |
| 180 | +load model ./models/mul10.yaml |
| 181 | +load model ./models/add10.yaml |
| 182 | +status model mul10 |
| 183 | +status model add10 |
| 184 | +load pipeline ./pipelines/trigger-joins.yaml |
| 185 | +status pipeline trigger-joins |
| 186 | +seldon pipeline infer trigger-joins --inference-mode grpc '{"model_name":"pipeline","inputs":[{"name":"ok1","contents":{"fp32_contents":[1]},"datatype":"FP32","shape":[1]},{"name":"INPUT","contents":{"fp32_contents":[1,2,3,4]},"datatype":"FP32","shape":[4]}]}' |
| 187 | +seldon pipeline infer trigger-joins --inference-mode grpc '{"model_name":"pipeline","inputs":[{"name":"ok3","contents":{"fp32_contents":[1]},"datatype":"FP32","shape":[1]},{"name":"INPUT","contents":{"fp32_contents":[1,2,3,4]},"datatype":"FP32","shape":[4]}]}' |
| 188 | +unload pipeline trigger-joins ./pipelines/trigger-joins.yaml |
| 189 | +unload model mul10 ./models/mul10.yaml |
| 190 | +unload model add10 ./models/add10.yaml |
| 191 | + |
| 192 | + |
| 193 | +# MLServer |
| 194 | +sleep $sleepTime |
| 195 | +load model ./models/sklearn-iris-gs.yaml |
| 196 | +status model iris |
| 197 | +seldon model infer iris '{"inputs": [{"name": "predict", "shape": [1, 4], "datatype": "FP32", "data": [[1, 2, 3, 4]]}]}' |
| 198 | +seldon model infer iris --inference-mode grpc \ |
| 199 | + '{"model_name":"iris","inputs":[{"name":"input","contents":{"fp32_contents":[1,2,3,4]},"datatype":"FP32","shape":[1,4]}]}' | jq -M . |
| 200 | +unload model iris ./models/sklearn-iris-gs.yaml |
| 201 | + |
| 202 | + |
| 203 | +# Experiments |
| 204 | +load model ./models/sklearn1.yaml |
| 205 | +load model ./models/sklearn2.yaml |
| 206 | +status model iris |
| 207 | +status model iris2 |
| 208 | +seldon model infer iris -i 50 \ |
| 209 | + '{"inputs": [{"name": "predict", "shape": [1, 4], "datatype": "FP32", "data": [[1, 2, 3, 4]]}]}' |
| 210 | +seldon model infer iris2 -i 50 \ |
| 211 | + '{"inputs": [{"name": "predict", "shape": [1, 4], "datatype": "FP32", "data": [[1, 2, 3, 4]]}]}' |
| 212 | +load experiment ./experiments/ab-default-model.yaml |
| 213 | +status experiment experiment-sample |
| 214 | +seldon model infer iris -i 50 \ |
| 215 | + '{"inputs": [{"name": "predict", "shape": [1, 4], "datatype": "FP32", "data": [[1, 2, 3, 4]]}]}' |
| 216 | +seldon model infer iris --show-headers \ |
| 217 | + '{"inputs": [{"name": "predict", "shape": [1, 4], "datatype": "FP32", "data": [[1, 2, 3, 4]]}]}' |
| 218 | +seldon model infer iris -s -i 50 \ |
| 219 | + '{"inputs": [{"name": "predict", "shape": [1, 4], "datatype": "FP32", "data": [[1, 2, 3, 4]]}]}' |
| 220 | +seldon model infer iris --inference-mode grpc -s -i 50\ |
| 221 | + '{"model_name":"iris","inputs":[{"name":"input","contents":{"fp32_contents":[1,2,3,4]},"datatype":"FP32","shape":[1,4]}]}' |
| 222 | +unload experiment experiment-sample ./experiments/ab-default-model.yaml |
| 223 | +unload model iris ./models/sklearn1.yaml |
| 224 | +unload model iris2 ./models/sklearn2.yaml |
0 commit comments