Skip to content

Commit efc5de8

Browse files
AarononthewebAaronontheweb
authored
DevOps standardization (#13)
* added WebCrawler.Shared.DevOps project to help standardize infrastructure * validated shared DevOps design thus far * added references to WebCrawler.Shared.DevOps in all projects * added Phobos bootstrapping * close #10
1 parent f2b419a commit efc5de8

26 files changed

+348
-21
lines changed

‎RELEASE_NOTES.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
#### 0.2.2 Feb 07 2019 ####
2+
* Added shared DevOps infrastructure.
3+
* Setting the stage for the use of Phobos in future releases.
4+
* Resolved [Install pbm on all nodes](https://github.com/petabridge/Cluster.WebCrawler/issues/10)
5+
16
#### 0.2.1 Feb 07 2019 ####
27
* Made it possible to invoke `CoordinatedShutdown` automatically via the `AppDomain.CurrentDomain.ProcessExit` event in .NET Core on `TrackerService`, `Lighthouse`, `CrawlerService`, and the `Web` application.
38

‎WebCrawler.sln‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "build", "build", "{FEFC4143
2323
build.sh = build.sh
2424
EndProjectSection
2525
EndProject
26+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WebCrawler.Shared.DevOps", "src\WebCrawler.Shared.DevOps\WebCrawler.Shared.DevOps.csproj", "{B6DF5688-61D1-462B-9B29-37E0898E2A94}"
27+
EndProject
28+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WebCrawler.Shared.DevOps.Tests", "src\WebCrawler.Shared.DevOps.Tests\WebCrawler.Shared.DevOps.Tests.csproj", "{FED11EA4-D0F5-4B7D-9696-135E61ED3186}"
29+
EndProject
2630
Global
2731
GlobalSection(SolutionConfigurationPlatforms) = preSolution
2832
Debug|Any CPU = Debug|Any CPU
@@ -53,6 +57,14 @@ Global
5357
{7A2C40A8-F6D1-4A28-8FD4-886818D7CB35}.Debug|Any CPU.Build.0 = Debug|Any CPU
5458
{7A2C40A8-F6D1-4A28-8FD4-886818D7CB35}.Release|Any CPU.ActiveCfg = Release|Any CPU
5559
{7A2C40A8-F6D1-4A28-8FD4-886818D7CB35}.Release|Any CPU.Build.0 = Release|Any CPU
60+
{B6DF5688-61D1-462B-9B29-37E0898E2A94}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
61+
{B6DF5688-61D1-462B-9B29-37E0898E2A94}.Debug|Any CPU.Build.0 = Debug|Any CPU
62+
{B6DF5688-61D1-462B-9B29-37E0898E2A94}.Release|Any CPU.ActiveCfg = Release|Any CPU
63+
{B6DF5688-61D1-462B-9B29-37E0898E2A94}.Release|Any CPU.Build.0 = Release|Any CPU
64+
{FED11EA4-D0F5-4B7D-9696-135E61ED3186}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
65+
{FED11EA4-D0F5-4B7D-9696-135E61ED3186}.Debug|Any CPU.Build.0 = Debug|Any CPU
66+
{FED11EA4-D0F5-4B7D-9696-135E61ED3186}.Release|Any CPU.ActiveCfg = Release|Any CPU
67+
{FED11EA4-D0F5-4B7D-9696-135E61ED3186}.Release|Any CPU.Build.0 = Release|Any CPU
5668
EndGlobalSection
5769
GlobalSection(SolutionProperties) = preSolution
5870
HideSolutionNode = FALSE

‎build.fsx‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,15 @@ Target "Clean" (fun _ ->
5454
CleanDir "docs/_site"
5555
)
5656

57+
(*
58+
59+
Creates an environment variable that can be read by other processes afterwards for deployments and such
60+
61+
*)
62+
Target "SetReleaseVersion" (fun _ ->
63+
setEnvironVar "RELEASE_NUMBER" releaseNotes.AssemblyVersion
64+
)
65+
5766
Target "AssemblyInfo" (fun _ ->
5867
XmlPokeInnerText "./src/common.props" "//Project/PropertyGroup/VersionPrefix" releaseNotes.AssemblyVersion
5968
XmlPokeInnerText "./src/common.props" "//Project/PropertyGroup/PackageReleaseNotes" (releaseNotes.Notes |> String.concat "\n")
@@ -362,9 +371,10 @@ Target "BuildRelease" DoNothing
362371
Target "All" DoNothing
363372
Target "Nuget" DoNothing
364373
Target "Docker" DoNothing
374+
Target "PrepareDeploy" DoNothing
365375

366376
// build dependencies
367-
"Clean" ==> "RestorePackages" ==> "AssemblyInfo" ==> "Build" ==> "BuildRelease"
377+
"Clean" ==> "RestorePackages" ==> "SetReleaseVersion" ==> "AssemblyInfo" ==> "Build" ==> "BuildRelease"
368378

369379
// tests dependencies
370380
"Clean" ==> "RestorePackages" ==> "Build" ==> "RunTests"
@@ -385,4 +395,7 @@ Target "Docker" DoNothing
385395
"NBench" ==> "All"
386396
"Nuget" ==> "All"
387397

398+
// Deploy
399+
"BuildRelease" ==> "Docker" ==> "PrepareDeploy"
400+
388401
RunTargetOrDefault "Help"

‎src/Lighthouse/Lighthouse.csproj‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,13 @@
66
</PropertyGroup>
77
<ItemGroup>
88
<PackageReference Include="Akka.Cluster" Version="$(AkkaVersion)" />
9-
<PackageReference Include="Akka.Cluster.HealthCheck" Version="$(AkkaHealthCheckVersion)" />
10-
<PackageReference Include="Akka.Bootstrap.Docker" Version="$(AkkaBootstrapVersion)" />
9+
1110
<PackageReference Include="Microsoft.Extensions.Configuration" Version="1.1.2" />
1211
<PackageReference Include="Microsoft.Extensions.Configuration.Xml" Version="1.1.2" />
13-
<PackageReference Include="Petabridge.Cmd.Cluster" Version="$(PetabridgeCmdVersion)" />
12+
13+
</ItemGroup>
14+
<ItemGroup>
15+
<ProjectReference Include="..\WebCrawler.Shared.DevOps\WebCrawler.Shared.DevOps.csproj" />
1416
</ItemGroup>
1517
<ItemGroup>
1618
<None Update="akka.hocon">

‎src/Lighthouse/LighthouseHostFactory.cs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using Akka.Actor;
66
using Akka.Bootstrap.Docker;
77
using Akka.Configuration;
8+
using WebCrawler.Shared.DevOps;
89
using ConfigurationException = Akka.Configuration.ConfigurationException;
910

1011
namespace Lighthouse
@@ -18,8 +19,7 @@ public static ActorSystem LaunchLighthouse()
1819
{
1920
var systemName = Environment.GetEnvironmentVariable("ACTORSYSTEM")?.Trim();
2021

21-
var clusterConfig = ConfigurationFactory.ParseString(File.ReadAllText("akka.hocon"))
22-
.BootstrapFromDocker();
22+
var clusterConfig = ConfigurationFactory.ParseString(File.ReadAllText("akka.hocon")).ApplyOpsConfig();
2323

2424
var lighthouseConfig = clusterConfig.GetConfig("lighthouse");
2525
if (lighthouseConfig != null && string.IsNullOrEmpty(systemName))
@@ -66,7 +66,8 @@ public static ActorSystem LaunchLighthouse()
6666
var finalConfig = injectedClusterConfigString != null ? injectedClusterConfigString
6767
.WithFallback(clusterConfig) : clusterConfig;
6868

69-
return ActorSystem.Create(systemName, finalConfig);
69+
return ActorSystem.Create(systemName, finalConfig)
70+
.StartPbm();
7071
}
7172
}
7273
}

‎src/Lighthouse/LighthouseService.cs‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,6 @@ public class LighthouseService
2727
public void Start()
2828
{
2929
_lighthouseSystem = LighthouseHostFactory.LaunchLighthouse();
30-
var pbm = PetabridgeCmd.Get(_lighthouseSystem);
31-
pbm.RegisterCommandPalette(ClusterCommands.Instance); // enable cluster management commands
32-
pbm.Start();
3330
}
3431

3532
/// <summary>

‎src/WebCrawler.CrawlService/CrawlerService.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using Akka.Actor;
66
using Akka.Bootstrap.Docker;
77
using WebCrawler.Shared.Config;
8+
using WebCrawler.Shared.DevOps;
89

910
namespace WebCrawler.CrawlService
1011
{
@@ -18,7 +19,7 @@ public class CrawlerService
1819
public bool Start()
1920
{
2021
var config = HoconLoader.ParseConfig("crawler.hocon");
21-
ClusterSystem = ActorSystem.Create("webcrawler", config.BootstrapFromDocker());
22+
ClusterSystem = ActorSystem.Create("webcrawler", config.ApplyOpsConfig()).StartPbm();
2223
return true;
2324
}
2425

‎src/WebCrawler.CrawlService/WebCrawler.CrawlService.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
</ItemGroup>
2020

2121
<ItemGroup>
22+
<ProjectReference Include="..\WebCrawler.Shared.DevOps\WebCrawler.Shared.DevOps.csproj" />
2223
<ProjectReference Include="..\WebCrawler.Shared.IO\WebCrawler.Shared.IO.csproj" />
2324
<ProjectReference Include="..\WebCrawler.Shared\WebCrawler.Shared.csproj" />
2425
</ItemGroup>

‎src/WebCrawler.Shared.DevOps.Tests/ActorSystemStartupSpecs.cs‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Text;
4+
using Akka.Actor;
5+
using Akka.Cluster.HealthCheck;
6+
using Akka.HealthCheck;
7+
using Akka.HealthCheck.Liveness;
8+
using Akka.HealthCheck.Readiness;
9+
using Akka.TestKit.Xunit2;
10+
using FluentAssertions;
11+
using Xunit;
12+
using Xunit.Abstractions;
13+
14+
namespace WebCrawler.Shared.DevOps.Tests
15+
{
16+
public class ActorSystemStartupSpecs : TestKit
17+
{
18+
public ActorSystemStartupSpecs(ITestOutputHelper helper)
19+
: base(Akka.Configuration.Config.Empty.ApplyOpsConfig(), output: helper) { }
20+
21+
[Fact(DisplayName = "Instrumented ActorSystem should start HealthChecks automatically")]
22+
public void ActorSystem_should_start_HealthChecks_automatically()
23+
{
24+
/*
25+
* Without explicitly invoking the AkkaHealthCheck extension, we should see
26+
* that the actors responsible for publishing that information have already
27+
* started and begun broadcasting readiness / liveness status in the background
28+
* since we enable the extension via the `akka.extensions` HOCON
29+
*/
30+
31+
// Liveness
32+
AwaitAssert(() =>
33+
{
34+
Sys.ActorSelection("/system/healthcheck-live").Tell(GetCurrentLiveness.Instance);
35+
ExpectMsg<LivenessStatus>(TimeSpan.FromMilliseconds(30));
36+
}, TimeSpan.FromSeconds(3), TimeSpan.FromMilliseconds(40));
37+
38+
// Readiness
39+
AwaitAssert(() =>
40+
{
41+
Sys.ActorSelection("/system/healthcheck-readiness").Tell(GetCurrentReadiness.Instance);
42+
ExpectMsg<ReadinessStatus>(TimeSpan.FromMilliseconds(30));
43+
}, TimeSpan.FromSeconds(3), TimeSpan.FromMilliseconds(40));
44+
45+
// should be running with the Akka.Cluster healthcheck probe
46+
//AkkaHealthCheck.For(Sys).ReadinessProvider.Should().BeOfType<ClusterReadinessProbeProvider>();
47+
}
48+
}
49+
}

‎src/WebCrawler.Shared.DevOps.Tests/Config/OpsConfigSpec.cs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
using FluentAssertions;
2+
using WebCrawler.Shared.DevOps.Config;
3+
using Xunit;
4+
5+
namespace WebCrawler.Shared.DevOps.Tests.Config
6+
{
7+
public class OpsConfigSpec
8+
{
9+
[Fact]
10+
public void Should_load_default_OpsConfig()
11+
{
12+
var config = OpsConfig.GetOpsConfig();
13+
config.Should().NotBeNull();
14+
}
15+
}
16+
}

0 commit comments

Comments
 (0)