Add debug logging, update mongodb driver
This commit is contained in:
		
							parent
							
								
									d4ad04b0f3
								
							
						
					
					
						commit
						8421b49834
					
				
					 4 changed files with 13 additions and 9 deletions
				
			
		| 
						 | 
					@ -5,8 +5,6 @@ services:
 | 
				
			||||||
    image: new_infofer_scraper
 | 
					    image: new_infofer_scraper
 | 
				
			||||||
    build: .
 | 
					    build: .
 | 
				
			||||||
    ports:
 | 
					    ports:
 | 
				
			||||||
      - ${PORT:-5000}:80
 | 
					      - ${PORT:-5001}:80
 | 
				
			||||||
    environment:
 | 
					    environment:
 | 
				
			||||||
      - DB_DIR=/data
 | 
					      DB_DIR: /data
 | 
				
			||||||
    volumes:
 | 
					 | 
				
			||||||
      - ./data:/data
 | 
					 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
| 
						 | 
					@ -2,13 +2,13 @@
 | 
				
			||||||
using System.Collections.Generic;
 | 
					using System.Collections.Generic;
 | 
				
			||||||
using System.Diagnostics;
 | 
					using System.Diagnostics;
 | 
				
			||||||
using System.Threading.Tasks;
 | 
					using System.Threading.Tasks;
 | 
				
			||||||
using InfoferScraper.Models.Train;
 | 
					 | 
				
			||||||
using InfoferScraper.Models.Station;
 | 
					 | 
				
			||||||
using Server.Services.Interfaces;
 | 
					 | 
				
			||||||
using Server.Utils;
 | 
					 | 
				
			||||||
using InfoferScraper;
 | 
					using InfoferScraper;
 | 
				
			||||||
 | 
					using InfoferScraper.Models.Station;
 | 
				
			||||||
 | 
					using InfoferScraper.Models.Train;
 | 
				
			||||||
using Microsoft.Extensions.Logging;
 | 
					using Microsoft.Extensions.Logging;
 | 
				
			||||||
using scraper.Models.Itinerary;
 | 
					using scraper.Models.Itinerary;
 | 
				
			||||||
 | 
					using Server.Services.Interfaces;
 | 
				
			||||||
 | 
					using Server.Utils;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
namespace Server.Services.Implementations {
 | 
					namespace Server.Services.Implementations {
 | 
				
			||||||
	public class DataManager : IDataManager {
 | 
						public class DataManager : IDataManager {
 | 
				
			||||||
| 
						 | 
					@ -25,6 +25,7 @@ namespace Server.Services.Implementations {
 | 
				
			||||||
 | 
					
 | 
				
			||||||
			stationCache = new(async (t) => {
 | 
								stationCache = new(async (t) => {
 | 
				
			||||||
				var (stationName, date) = t;
 | 
									var (stationName, date) = t;
 | 
				
			||||||
 | 
									Logger.LogDebug("Fetching station {StationName} for date {Date}", stationName, date);
 | 
				
			||||||
				var zonedDate = new NodaTime.LocalDate(date.Year, date.Month, date.Day).AtStartOfDayInZone(CfrTimeZone);
 | 
									var zonedDate = new NodaTime.LocalDate(date.Year, date.Month, date.Day).AtStartOfDayInZone(CfrTimeZone);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
				var station = await InfoferScraper.Scrapers.StationScraper.Scrape(stationName, zonedDate.ToDateTimeOffset());
 | 
									var station = await InfoferScraper.Scrapers.StationScraper.Scrape(stationName, zonedDate.ToDateTimeOffset());
 | 
				
			||||||
| 
						 | 
					@ -40,6 +41,7 @@ namespace Server.Services.Implementations {
 | 
				
			||||||
			}, TimeSpan.FromMinutes(1));
 | 
								}, TimeSpan.FromMinutes(1));
 | 
				
			||||||
			trainCache = new(async (t) => {
 | 
								trainCache = new(async (t) => {
 | 
				
			||||||
				var (trainNumber, date) = t;
 | 
									var (trainNumber, date) = t;
 | 
				
			||||||
 | 
									Logger.LogDebug("Fetching train {TrainNumber} for date {Date}", trainNumber, date);
 | 
				
			||||||
				var zonedDate = new NodaTime.LocalDate(date.Year, date.Month, date.Day).AtStartOfDayInZone(CfrTimeZone);
 | 
									var zonedDate = new NodaTime.LocalDate(date.Year, date.Month, date.Day).AtStartOfDayInZone(CfrTimeZone);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
				var train = await InfoferScraper.Scrapers.TrainScraper.Scrape(trainNumber, zonedDate.ToDateTimeOffset());
 | 
									var train = await InfoferScraper.Scrapers.TrainScraper.Scrape(trainNumber, zonedDate.ToDateTimeOffset());
 | 
				
			||||||
| 
						 | 
					@ -55,6 +57,7 @@ namespace Server.Services.Implementations {
 | 
				
			||||||
			}, TimeSpan.FromSeconds(30));
 | 
								}, TimeSpan.FromSeconds(30));
 | 
				
			||||||
			itinerariesCache = new(async (t) => {
 | 
								itinerariesCache = new(async (t) => {
 | 
				
			||||||
				var (from, to, date) = t;
 | 
									var (from, to, date) = t;
 | 
				
			||||||
 | 
									Logger.LogDebug("Fetching itinerary from {From} to {To} for date {Date}", from, to, date);
 | 
				
			||||||
				var zonedDate = new NodaTime.LocalDate(date.Year, date.Month, date.Day).AtStartOfDayInZone(CfrTimeZone);
 | 
									var zonedDate = new NodaTime.LocalDate(date.Year, date.Month, date.Day).AtStartOfDayInZone(CfrTimeZone);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
				var itineraries = await InfoferScraper.Scrapers.RouteScraper.Scrape(from, to, zonedDate.ToDateTimeOffset());
 | 
									var itineraries = await InfoferScraper.Scrapers.RouteScraper.Scrape(from, to, zonedDate.ToDateTimeOffset());
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
| 
						 | 
					@ -57,10 +57,13 @@ public class Database : Server.Services.Interfaces.IDatabase {
 | 
				
			||||||
		Logger = logger;
 | 
							Logger = logger;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
		var settings = MongoClientSettings.FromConnectionString(mongoSettings.Value.ConnectionString);
 | 
							var settings = MongoClientSettings.FromConnectionString(mongoSettings.Value.ConnectionString);
 | 
				
			||||||
 | 
							settings.ServerApi = new(ServerApiVersion.V1);
 | 
				
			||||||
		settings.MaxConnectionPoolSize = 10000;
 | 
							settings.MaxConnectionPoolSize = 10000;
 | 
				
			||||||
		MongoClient mongoClient = new(settings);
 | 
							MongoClient mongoClient = new(settings);
 | 
				
			||||||
 | 
							Logger.LogDebug("Created monogClient");
 | 
				
			||||||
		throttle = new(mongoClient.Settings.MaxConnectionPoolSize / 2);
 | 
							throttle = new(mongoClient.Settings.MaxConnectionPoolSize / 2);
 | 
				
			||||||
		db = mongoClient.GetDatabase(mongoSettings.Value.DatabaseName) ?? throw new NullReferenceException("Unable to get Mongo database");
 | 
							db = mongoClient.GetDatabase(mongoSettings.Value.DatabaseName) ?? throw new NullReferenceException("Unable to get Mongo database");
 | 
				
			||||||
 | 
							Logger.LogDebug("Created db");
 | 
				
			||||||
		dbRecordCollection = db.GetCollection<DbRecord>("db");
 | 
							dbRecordCollection = db.GetCollection<DbRecord>("db");
 | 
				
			||||||
		trainListingsCollection = db.GetCollection<TrainListing>("trainListings");
 | 
							trainListingsCollection = db.GetCollection<TrainListing>("trainListings");
 | 
				
			||||||
		stationListingsCollection = db.GetCollection<StationListing>("stationListings");
 | 
							stationListingsCollection = db.GetCollection<StationListing>("stationListings");
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
| 
						 | 
					@ -12,7 +12,7 @@
 | 
				
			||||||
        <PackageReference Include="Microsoft.Data.Sqlite" Version="6.0.1" />
 | 
					        <PackageReference Include="Microsoft.Data.Sqlite" Version="6.0.1" />
 | 
				
			||||||
        <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="5.0.13" />
 | 
					        <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="5.0.13" />
 | 
				
			||||||
        <PackageReference Include="MongoDB.Analyzer" Version="1.1.0" />
 | 
					        <PackageReference Include="MongoDB.Analyzer" Version="1.1.0" />
 | 
				
			||||||
        <PackageReference Include="MongoDB.Driver" Version="2.18.0" />
 | 
					        <PackageReference Include="MongoDB.Driver" Version="2.19.1" />
 | 
				
			||||||
        <PackageReference Include="Nanoid" Version="2.1.0" />
 | 
					        <PackageReference Include="Nanoid" Version="2.1.0" />
 | 
				
			||||||
        <PackageReference Include="Swashbuckle.AspNetCore" Version="5.6.3" />
 | 
					        <PackageReference Include="Swashbuckle.AspNetCore" Version="5.6.3" />
 | 
				
			||||||
    </ItemGroup>
 | 
					    </ItemGroup>
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
		Loading…
	
	Add table
		
		Reference in a new issue