Skip to content

Commit b9fdab4

Browse files
AarononthewebAaronontheweb
committed
copied over original WebCrawler code
1 parent 9c1be12 commit b9fdab4

File tree

109 files changed

+34491
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

109 files changed

+34491
-0
lines changed

‎.dockerignore‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
.dockerignore
2+
.env
3+
.git
4+
.gitignore
5+
.vs
6+
.vscode
7+
docker-compose.yml
8+
docker-compose.*.yml
9+
*/bin
10+
*/obj
11+
!obj/Docker/publish/*
12+
!obj/Docker/empty/

‎WebCrawler.sln‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
2+
Microsoft Visual Studio Solution File, Format Version 12.00
3+
# Visual Studio 15
4+
VisualStudioVersion = 15.0.27130.2027
5+
MinimumVisualStudioVersion = 10.0.40219.1
6+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebCrawler.Shared", "src\WebCrawler.Shared\WebCrawler.Shared.csproj", "{236B9550-1E56-4893-809E-E2E4E93A8AD5}"
7+
EndProject
8+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebCrawler.Shared.IO", "src\WebCrawler.Shared.IO\WebCrawler.Shared.IO.csproj", "{D3BC07C3-B4E1-47F0-A444-255948A3A0A5}"
9+
EndProject
10+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebCrawler.TrackerService", "src\WebCrawler.TrackerService\WebCrawler.TrackerService.csproj", "{95B49511-2420-4E7A-A52F-15B41146A8BB}"
11+
EndProject
12+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebCrawler.CrawlService", "src\WebCrawler.CrawlService\WebCrawler.CrawlService.csproj", "{E81C11C6-C365-42F0-8430-6DBB53B66C39}"
13+
EndProject
14+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebCrawler.Web", "src\WebCrawler.Web\WebCrawler.Web.csproj", "{297D37F3-5CAA-4B4F-A6EA-0A76AFDDE80C}"
15+
EndProject
16+
Project("{E53339B2-1760-4266-BCC7-CA923CBCF16C}") = "docker-compose", "docker-compose.dcproj", "{188B87E9-7FF0-4488-A340-FE8A62688B27}"
17+
EndProject
18+
Global
19+
GlobalSection(SolutionConfigurationPlatforms) = preSolution
20+
Debug|Any CPU = Debug|Any CPU
21+
Release|Any CPU = Release|Any CPU
22+
EndGlobalSection
23+
GlobalSection(ProjectConfigurationPlatforms) = postSolution
24+
{236B9550-1E56-4893-809E-E2E4E93A8AD5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
25+
{236B9550-1E56-4893-809E-E2E4E93A8AD5}.Debug|Any CPU.Build.0 = Debug|Any CPU
26+
{236B9550-1E56-4893-809E-E2E4E93A8AD5}.Release|Any CPU.ActiveCfg = Release|Any CPU
27+
{236B9550-1E56-4893-809E-E2E4E93A8AD5}.Release|Any CPU.Build.0 = Release|Any CPU
28+
{D3BC07C3-B4E1-47F0-A444-255948A3A0A5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
29+
{D3BC07C3-B4E1-47F0-A444-255948A3A0A5}.Debug|Any CPU.Build.0 = Debug|Any CPU
30+
{D3BC07C3-B4E1-47F0-A444-255948A3A0A5}.Release|Any CPU.ActiveCfg = Release|Any CPU
31+
{D3BC07C3-B4E1-47F0-A444-255948A3A0A5}.Release|Any CPU.Build.0 = Release|Any CPU
32+
{95B49511-2420-4E7A-A52F-15B41146A8BB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
33+
{95B49511-2420-4E7A-A52F-15B41146A8BB}.Debug|Any CPU.Build.0 = Debug|Any CPU
34+
{95B49511-2420-4E7A-A52F-15B41146A8BB}.Release|Any CPU.ActiveCfg = Release|Any CPU
35+
{95B49511-2420-4E7A-A52F-15B41146A8BB}.Release|Any CPU.Build.0 = Release|Any CPU
36+
{E81C11C6-C365-42F0-8430-6DBB53B66C39}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
37+
{E81C11C6-C365-42F0-8430-6DBB53B66C39}.Debug|Any CPU.Build.0 = Debug|Any CPU
38+
{E81C11C6-C365-42F0-8430-6DBB53B66C39}.Release|Any CPU.ActiveCfg = Release|Any CPU
39+
{E81C11C6-C365-42F0-8430-6DBB53B66C39}.Release|Any CPU.Build.0 = Release|Any CPU
40+
{297D37F3-5CAA-4B4F-A6EA-0A76AFDDE80C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
41+
{297D37F3-5CAA-4B4F-A6EA-0A76AFDDE80C}.Debug|Any CPU.Build.0 = Debug|Any CPU
42+
{297D37F3-5CAA-4B4F-A6EA-0A76AFDDE80C}.Release|Any CPU.ActiveCfg = Release|Any CPU
43+
{297D37F3-5CAA-4B4F-A6EA-0A76AFDDE80C}.Release|Any CPU.Build.0 = Release|Any CPU
44+
{188B87E9-7FF0-4488-A340-FE8A62688B27}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
45+
{188B87E9-7FF0-4488-A340-FE8A62688B27}.Debug|Any CPU.Build.0 = Debug|Any CPU
46+
{188B87E9-7FF0-4488-A340-FE8A62688B27}.Release|Any CPU.ActiveCfg = Release|Any CPU
47+
{188B87E9-7FF0-4488-A340-FE8A62688B27}.Release|Any CPU.Build.0 = Release|Any CPU
48+
EndGlobalSection
49+
GlobalSection(SolutionProperties) = preSolution
50+
HideSolutionNode = FALSE
51+
EndGlobalSection
52+
GlobalSection(ExtensibilityGlobals) = postSolution
53+
SolutionGuid = {31B5BCF5-43CA-472A-A27B-2E79702D69E8}
54+
EndGlobalSection
55+
EndGlobal

‎docker-compose.ci.build.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
version: '3.4'
2+
3+
services:
4+
ci-build:
5+
image: microsoft/aspnetcore-build:1.0-2.0
6+
volumes:
7+
- .:/src
8+
working_dir: /src
9+
command: /bin/bash -c "dotnet restore ./WebCrawler.sln && dotnet publish ./WebCrawler.sln -c Release -o ./obj/Docker/publish"

‎docker-compose.dcproj‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<Project ToolsVersion="15.0" Sdk="Microsoft.Docker.Sdk">
3+
<PropertyGroup Label="Globals">
4+
<ProjectVersion>2.1</ProjectVersion>
5+
<DockerTargetOS>Linux</DockerTargetOS>
6+
<ProjectGuid>188b87e9-7ff0-4488-a340-fe8a62688b27</ProjectGuid>
7+
<DockerLaunchBrowser>True</DockerLaunchBrowser>
8+
<DockerServiceUrl>http://localhost:{ServicePort}</DockerServiceUrl>
9+
<DockerServiceName>webcrawler.web</DockerServiceName>
10+
</PropertyGroup>
11+
<ItemGroup>
12+
<None Include="docker-compose.override.yml">
13+
<DependentUpon>docker-compose.yml</DependentUpon>
14+
</None>
15+
<None Include="docker-compose.yml" />
16+
<None Include=".dockerignore" />
17+
<None Include="docker-compose.ci.build.yml" />
18+
</ItemGroup>
19+
</Project>

‎docker-compose.override.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
version: '3.4'
2+
3+
services:
4+
webcrawler.web:
5+
environment:
6+
- ASPNETCORE_ENVIRONMENT=Development
7+
ports:
8+
- "80"

‎docker-compose.yml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
version: '3.4'
2+
3+
services:
4+
lighthouse:
5+
image: petabridge/lighthouse:v0.9.2
6+
ports:
7+
- '9110:9110'
8+
- '4053:4053'
9+
environment:
10+
ACTORSYSTEM: "webcrawler"
11+
CLUSTER_IP: lighthouse
12+
CLUSTER_PORT: 4053
13+
CLUSTER_SEEDS: "akka.tcp://webcrawler@lighthouse:4053"
14+
15+
16+
webcrawler.web:
17+
image: webcrawler.web
18+
build:
19+
context: .
20+
dockerfile: src/WebCrawler.Web/Dockerfile
21+
ports:
22+
- '8080:80'
23+
environment:
24+
CLUSTER_SEEDS: "akka.tcp://webcrawler@lighthouse:4053"
25+
CLUSTER_IP: webcrawler.web
26+
CLUSTER_PORT: 0
27+
depends_on:
28+
- "lighthouse"
29+
30+
webcrawler.crawlservice:
31+
image: webcrawler.crawlservice
32+
build:
33+
context: .
34+
dockerfile: src/WebCrawler.CrawlService/Dockerfile
35+
environment:
36+
CLUSTER_SEEDS: "akka.tcp://webcrawler@lighthouse:4053"
37+
CLUSTER_IP: webcrawler.crawlservice
38+
CLUSTER_PORT: 0
39+
depends_on:
40+
- "lighthouse"
41+
42+
webcrawler.trackerservice:
43+
image: webcrawler.trackerservice
44+
build:
45+
context: .
46+
dockerfile: src/WebCrawler.TrackerService/Dockerfile
47+
environment:
48+
CLUSTER_SEEDS: "akka.tcp://webcrawler@lighthouse:4053"
49+
CLUSTER_IP: webcrawler.trackerservice
50+
CLUSTER_PORT: 0
51+
depends_on:
52+
- "lighthouse"

‎get-dockerip.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/bin/sh
2+
if [ -z "$CLUSTER_IP"]; then
3+
host=$(hostname -i)
4+
echo "Docker container bound on $host"
5+
export CLUSTER_IP="$host"
6+
else
7+
echo "Docker container bound on $CLUSTER_IP"
8+
fi
9+
10+
exec "$@"

‎src/WebCrawler.CrawlService/CrawlerService.cs‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Text;
4+
using System.Threading.Tasks;
5+
using Akka.Actor;
6+
using Akka.Bootstrap.Docker;
7+
using WebCrawler.Shared.Config;
8+
9+
namespace WebCrawler.CrawlService
10+
{
11+
public class CrawlerService
12+
{
13+
protected ActorSystem ClusterSystem;
14+
15+
public Task WhenTerminated => ClusterSystem.WhenTerminated;
16+
17+
18+
public bool Start()
19+
{
20+
var config = HoconLoader.ParseConfig("crawler.hocon");
21+
ClusterSystem = ActorSystem.Create("webcrawler", config.BootstrapFromDocker());
22+
return true;
23+
}
24+
25+
public Task Stop()
26+
{
27+
return CoordinatedShutdown.Get(ClusterSystem).Run(CoordinatedShutdown.ClrExitReason.Instance);
28+
}
29+
}
30+
}

‎src/WebCrawler.CrawlService/Dockerfile‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
FROM microsoft/dotnet:2.0-runtime AS base
2+
WORKDIR /app
3+
4+
# should be a comma-delimited list
5+
ENV CLUSTER_SEEDS "[]"
6+
ENV CLUSTER_IP ""
7+
ENV CLUSTER_PORT "5213"
8+
9+
#Akka.Remote inbound listening endpoint
10+
EXPOSE 5213
11+
12+
FROM microsoft/dotnet:2.0-sdk AS build
13+
WORKDIR /src
14+
COPY *.sln ./
15+
COPY ./get-dockerip.sh ./get-dockerip.sh
16+
COPY src/WebCrawler.CrawlService/WebCrawler.CrawlService.csproj src/WebCrawler.CrawlService/
17+
COPY src/WebCrawler.Shared/WebCrawler.Shared.csproj src/WebCrawler.Shared/
18+
COPY src/WebCrawler.Shared.IO/WebCrawler.Shared.IO.csproj src/WebCrawler.Shared.IO/
19+
RUN dotnet restore
20+
COPY . .
21+
WORKDIR /src/src/WebCrawler.CrawlService
22+
RUN dotnet build -c Release -o /app
23+
24+
FROM build AS publish
25+
RUN dotnet publish -c Release -o /app
26+
27+
FROM base AS final
28+
WORKDIR /app
29+
COPY --from=build /src/get-dockerip.sh ./get-dockerip.sh
30+
COPY --from=publish /app .
31+
32+
ENTRYPOINT ["/bin/bash","get-dockerip.sh"]
33+
34+
CMD ["dotnet", "WebCrawler.CrawlService.dll"]

‎src/WebCrawler.CrawlService/Program.cs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
using System;
2+
3+
namespace WebCrawler.CrawlService
4+
{
5+
class Program
6+
{
7+
static void Main(string[] args)
8+
{
9+
var crawlerService = new CrawlerService();
10+
crawlerService.Start();
11+
12+
Console.CancelKeyPress += (sender, eventArgs) =>
13+
{
14+
crawlerService.Stop();
15+
eventArgs.Cancel = true;
16+
};
17+
18+
crawlerService.WhenTerminated.Wait();
19+
}
20+
}
21+
}

0 commit comments

Comments
 (0)