kbruen/new-infofer-scraper
1
0
Fork 0
Code Pull requests Releases Activity

Add debug logging, update mongodb driver

Browse source
This commit is contained in:
Kenneth Bruen 2023-03-30 04:00:19 +02:00
parent d4ad04b0f3
commit 8421b49834
Signed by: kbruen
GPG key ID: C1980A470C3EE5B1
4 changed files with 13 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
docker-compose.yml
View file

@ -5,8 +5,6 @@ services:
image: new_infofer_scraper
build: .
ports:
- ${PORT:-5000}:80
- ${PORT:-5001}:80
environment:
- DB_DIR=/data
volumes:
- ./data:/data
DB_DIR: /data

11
server/Services/Implementations/DataManager.cs
View file

@ -2,13 +2,13 @@
using System.Collections.Generic;
using System.Diagnostics;
using System.Threading.Tasks;
using InfoferScraper.Models.Train;
using InfoferScraper.Models.Station;
using Server.Services.Interfaces;
using Server.Utils;
using InfoferScraper;
using InfoferScraper.Models.Station;
using InfoferScraper.Models.Train;
using Microsoft.Extensions.Logging;
using scraper.Models.Itinerary;
using Server.Services.Interfaces;
using Server.Utils;
namespace Server.Services.Implementations {
public class DataManager : IDataManager {
@ -25,6 +25,7 @@ namespace Server.Services.Implementations {
stationCache = new(async (t) => {
var (stationName, date) = t;
Logger.LogDebug("Fetching station {StationName} for date {Date}", stationName, date);
var zonedDate = new NodaTime.LocalDate(date.Year, date.Month, date.Day).AtStartOfDayInZone(CfrTimeZone);
var station = await InfoferScraper.Scrapers.StationScraper.Scrape(stationName, zonedDate.ToDateTimeOffset());
@ -40,6 +41,7 @@ namespace Server.Services.Implementations {
}, TimeSpan.FromMinutes(1));
trainCache = new(async (t) => {
var (trainNumber, date) = t;
Logger.LogDebug("Fetching train {TrainNumber} for date {Date}", trainNumber, date);
var zonedDate = new NodaTime.LocalDate(date.Year, date.Month, date.Day).AtStartOfDayInZone(CfrTimeZone);
var train = await InfoferScraper.Scrapers.TrainScraper.Scrape(trainNumber, zonedDate.ToDateTimeOffset());
@ -55,6 +57,7 @@ namespace Server.Services.Implementations {
}, TimeSpan.FromSeconds(30));
itinerariesCache = new(async (t) => {
var (from, to, date) = t;
Logger.LogDebug("Fetching itinerary from {From} to {To} for date {Date}", from, to, date);
var zonedDate = new NodaTime.LocalDate(date.Year, date.Month, date.Day).AtStartOfDayInZone(CfrTimeZone);
var itineraries = await InfoferScraper.Scrapers.RouteScraper.Scrape(from, to, zonedDate.ToDateTimeOffset());

3
server/Services/Implementations/Database.cs
View file

@ -57,10 +57,13 @@ public class Database : Server.Services.Interfaces.IDatabase {
Logger = logger;
var settings = MongoClientSettings.FromConnectionString(mongoSettings.Value.ConnectionString);
settings.ServerApi = new(ServerApiVersion.V1);
settings.MaxConnectionPoolSize = 10000;
MongoClient mongoClient = new(settings);
Logger.LogDebug("Created monogClient");
throttle = new(mongoClient.Settings.MaxConnectionPoolSize / 2);
db = mongoClient.GetDatabase(mongoSettings.Value.DatabaseName) ?? throw new NullReferenceException("Unable to get Mongo database");
Logger.LogDebug("Created db");
dbRecordCollection = db.GetCollection<DbRecord>("db");
trainListingsCollection = db.GetCollection<TrainListing>("trainListings");
stationListingsCollection = db.GetCollection<StationListing>("stationListings");

2
server/server.csproj
View file

@ -12,7 +12,7 @@
<PackageReference Include="Microsoft.Data.Sqlite" Version="6.0.1" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="5.0.13" />
<PackageReference Include="MongoDB.Analyzer" Version="1.1.0" />
<PackageReference Include="MongoDB.Driver" Version="2.18.0" />
<PackageReference Include="MongoDB.Driver" Version="2.19.1" />
<PackageReference Include="Nanoid" Version="2.1.0" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="5.6.3" />
</ItemGroup>