kbruen/new-infofer-scraper
1
0
Fork 0
Code Pull requests Releases Activity

Run tasks concurrently

Browse source
This commit is contained in:
Kenneth Bruen 2022-11-09 17:28:22 +01:00
parent 0bcaf40ea3
commit 0b5a8bf487
Signed by: kbruen
GPG key ID: C1980A470C3EE5B1
1 changed files with 17 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
server/Services/Implementations/Database.cs
View file

@ -6,6 +6,7 @@ using System.Text.Json;
using System.Text.Json.Nodes; using System.Text.Json.Nodes;
using System.Text.Json.Serialization; using System.Text.Json.Serialization;
using System.Threading.Tasks; using System.Threading.Tasks;
using InfoferScraper.Models.Station;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using MongoDB.Bson; using MongoDB.Bson;
@ -40,7 +41,9 @@ public class Database : Server.Services.Interfaces.IDatabase {
public Database(ILogger<Database> logger, IOptions<MongoSettings> mongoSettings) { public Database(ILogger<Database> logger, IOptions<MongoSettings> mongoSettings) {
Logger = logger; Logger = logger;
MongoClient mongoClient = new(mongoSettings.Value.ConnectionString); var settings = MongoClientSettings.FromConnectionString(mongoSettings.Value.ConnectionString);
settings.MaxConnectionPoolSize = 100000;
MongoClient mongoClient = new(settings);
db = mongoClient.GetDatabase(mongoSettings.Value.DatabaseName) ?? throw new NullReferenceException("Unable to get Mongo database"); db = mongoClient.GetDatabase(mongoSettings.Value.DatabaseName) ?? throw new NullReferenceException("Unable to get Mongo database");
dbRecordCollection = db.GetCollection<DbRecord>("db"); dbRecordCollection = db.GetCollection<DbRecord>("db");
trainListingsCollection = db.GetCollection<TrainListing>("trainListings"); trainListingsCollection = db.GetCollection<TrainListing>("trainListings");
@ -183,11 +186,13 @@ public class Database : Server.Services.Interfaces.IDatabase {
public async Task OnTrainData(InfoferScraper.Models.Train.ITrainScrapeResult trainData) { public async Task OnTrainData(InfoferScraper.Models.Train.ITrainScrapeResult trainData) {
var trainNumber = await FoundTrain(trainData.Rank, trainData.Number, trainData.Operator); var trainNumber = await FoundTrain(trainData.Rank, trainData.Number, trainData.Operator);
foreach (var group in trainData.Groups) { await Task.WhenAll(
foreach (var station in group.Stations) { trainData.Groups
await FoundTrainAtStation(station.Name, trainNumber); .SelectMany(g => g.Stations)
} .Select(trainStop => trainStop.Name)
} .Distinct()
.Select(station => FoundTrainAtStation(station, trainNumber))
);
} }
public async Task OnStationData(InfoferScraper.Models.Station.IStationScrapeResult stationData) { public async Task OnStationData(InfoferScraper.Models.Station.IStationScrapeResult stationData) {
@ -198,22 +203,19 @@ public class Database : Server.Services.Interfaces.IDatabase {
trainNumber = await FoundTrain(train.Train.Rank, trainNumber, train.Train.Operator); trainNumber = await FoundTrain(train.Train.Rank, trainNumber, train.Train.Operator);
await FoundTrainAtStation(stationName, trainNumber); await FoundTrainAtStation(stationName, trainNumber);
if (train.Train.Route.Count != 0) { if (train.Train.Route.Count != 0) {
foreach (var station in train.Train.Route) { await Task.WhenAll(train.Train.Route.Select(station => FoundTrainAtStation(station, trainNumber)));
await FoundTrainAtStation(station, trainNumber);
}
} }
} }
List<IStationArrDep> arrdep = new();
if (stationData.Arrivals != null) { if (stationData.Arrivals != null) {
foreach (var train in stationData.Arrivals) { arrdep.AddRange(stationData.Arrivals);
await ProcessTrain(train);
}
} }
if (stationData.Departures != null) { if (stationData.Departures != null) {
foreach (var train in stationData.Departures) { arrdep.AddRange(stationData.Departures);
await ProcessTrain(train);
}
} }
await Task.WhenAll(arrdep.Select(ProcessTrain));
} }
} }