kbruen/new-infofer-scraper
1
0
Fork 0
Code Pull requests Releases Activity

Transition from fs+JSON to MongoDB

Browse source
This commit is contained in:
Kenneth Bruen 2022-11-09 05:17:39 +01:00
parent 169e128ca1
commit 145f7b0ee1
Signed by: kbruen
GPG key ID: C1980A470C3EE5B1
13 changed files with 167 additions and 129 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
server/Controllers/V2/StationsController.cs
View file

@ -1,5 +1,6 @@
using System.Collections.Generic;
using Microsoft.AspNetCore.Mvc;
using Server.Models.Database;
using Server.Services.Interfaces;
namespace Server.Controllers.V2;
@ -15,7 +16,7 @@ public class StationsController : Controller {
}
[HttpGet("")]
public ActionResult<IEnumerable<IStationRecord>> ListStations() {
public ActionResult<IEnumerable<StationListing>> ListStations() {
return Ok(Database.Stations);
}
}

3
server/Controllers/V2/TrainsController.cs
View file

@ -1,5 +1,6 @@
using System.Collections.Generic;
using Microsoft.AspNetCore.Mvc;
using Server.Models.Database;
using Server.Services.Interfaces;
namespace Server.Controllers.V2;
@ -15,7 +16,7 @@ public class TrainsController : Controller {
}
[HttpGet("")]
public ActionResult<IEnumerable<ITrainRecord>> ListTrains() {
public ActionResult<IEnumerable<TrainListing>> ListTrains() {
return Ok(Database.Trains);
}
}

3
server/Controllers/V3/StationsController.cs
View file

@ -4,6 +4,7 @@ using System.Threading.Tasks;
using InfoferScraper.Models.Station;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Server.Models.Database;
using Server.Services.Interfaces;
namespace Server.Controllers.V3;
@ -21,7 +22,7 @@ public class StationsController : Controller {
}
[HttpGet("")]
public ActionResult<IEnumerable<IStationRecord>> ListStations() {
public ActionResult<IEnumerable<StationListing>> ListStations() {
return Ok(Database.Stations);
}

3
server/Controllers/V3/TrainsController.cs
View file

@ -5,6 +5,7 @@ using InfoferScraper.Models.Train;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using scraper.Exceptions;
using Server.Models.Database;
using Server.Services.Interfaces;
namespace Server.Controllers.V3;
@ -22,7 +23,7 @@ public class TrainsController : Controller {
}
[HttpGet("")]
public ActionResult<IEnumerable<ITrainRecord>> ListTrains() {
public ActionResult<IEnumerable<TrainListing>> ListTrains() {
return Ok(Database.Trains);
}

5
server/Models/Database/MongoSettings.cs Normal file
View file

@ -0,0 +1,5 @@
namespace Server.Models.Database;
public record MongoSettings(string ConnectionString, string DatabaseName) {
public MongoSettings() : this("", "") { }
}

17
server/Models/Database/StationListing.cs Normal file
View file

@ -0,0 +1,17 @@
using System.Collections.Generic;
using System.Text.Json.Serialization;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
namespace Server.Models.Database;
public record StationListing(
[property: BsonId]
[property: BsonRepresentation(BsonType.ObjectId)]
[property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
string? Id,
string Name,
List<string> StoppedAtBy
) {
public StationListing(string name, List<string> stoppedAtBy) : this(null, name, stoppedAtBy) { }
}

17
server/Models/Database/TrainListing.cs Normal file
View file

@ -0,0 +1,17 @@
using System.Text.Json.Serialization;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
namespace Server.Models.Database;
public record TrainListing(
[property: BsonId]
[property: BsonRepresentation(BsonType.ObjectId)]
[property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
string? Id,
string Rank,
string Number,
string Company
) {
public TrainListing(string rank, string number, string company) : this(null, rank, number, company) { }
}

212
server/Services/Implementations/Database.cs
View file

@ -7,6 +7,11 @@ using System.Text.Json.Nodes;
using System.Text.Json.Serialization;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
using MongoDB.Driver;
using Server.Models.Database;
namespace Server.Services.Implementations;
@ -17,77 +22,47 @@ public class Database : Server.Services.Interfaces.IDatabase {
private ILogger<Database> Logger { get; }
private bool shouldCommitOnEveryChange = true;
private bool dbDataDirty = false;
private bool stationsDirty = false;
private bool trainsDirty = false;
public DbRecord DbData { get; private set; } = new(3);
public DbRecord DbData { get; private set; } = new(2);
private List<StationRecord> stations = new();
private List<TrainRecord> trains = new();
public IReadOnlyList<Server.Services.Interfaces.IStationRecord> Stations => stations;
public IReadOnlyList<Server.Services.Interfaces.ITrainRecord> Trains => trains;
public IReadOnlyList<StationListing> Stations => stationListingsCollection.FindSync(_ => true).ToList();
public IReadOnlyList<TrainListing> Trains => trainListingsCollection.FindSync(_ => true).ToList();
private static readonly string DbDir = Environment.GetEnvironmentVariable("DB_DIR") ?? Path.Join(Environment.CurrentDirectory, "db");
private static readonly string DbFile = Path.Join(DbDir, "db.json");
private static readonly string StationsFile = Path.Join(DbDir, "stations.json");
private static readonly string TrainsFile = Path.Join(DbDir, "trains.json");
public IDisposable MakeDbTransaction() {
shouldCommitOnEveryChange = false;
return new Server.Utils.ActionDisposable(() => {
if (dbDataDirty) File.WriteAllText(DbFile, JsonSerializer.Serialize(DbData, serializerOptions));
if (stationsDirty) {
stations.Sort((s1, s2) => s2.StoppedAtBy.Count.CompareTo(s1.StoppedAtBy.Count));
File.WriteAllText(StationsFile, JsonSerializer.Serialize(stations, serializerOptions));
}
if (trainsDirty) File.WriteAllText(TrainsFile, JsonSerializer.Serialize(trains, serializerOptions));
dbDataDirty = stationsDirty = trainsDirty = false;
shouldCommitOnEveryChange = true;
});
}
private readonly IMongoDatabase db;
private readonly IMongoCollection<DbRecord> dbRecordCollection;
private readonly IMongoCollection<TrainListing> trainListingsCollection;
private readonly IMongoCollection<StationListing> stationListingsCollection;
public Database(ILogger<Database> logger) {
public Database(ILogger<Database> logger, IOptions<MongoSettings> mongoSettings) {
Logger = logger;
if (!Directory.Exists(DbDir)) {
Logger.LogDebug("Creating directory: {DbDir}", DbDir);
Directory.CreateDirectory(DbDir);
}
MongoClient mongoClient = new(mongoSettings.Value.ConnectionString);
db = mongoClient.GetDatabase(mongoSettings.Value.DatabaseName) ?? throw new NullReferenceException("Unable to get Mongo database");
dbRecordCollection = db.GetCollection<DbRecord>("db");
trainListingsCollection = db.GetCollection<TrainListing>("trainListings");
stationListingsCollection = db.GetCollection<StationListing>("stationListings");
Migration();
if (File.Exists(DbFile)) {
DbData = JsonSerializer.Deserialize<DbRecord>(File.ReadAllText(DbFile), serializerOptions)!;
}
else {
File.WriteAllText(DbFile, JsonSerializer.Serialize(DbData, serializerOptions));
}
if (File.Exists(StationsFile)) {
stations = JsonSerializer.Deserialize<List<StationRecord>>(File.ReadAllText(StationsFile), serializerOptions)!;
}
if (File.Exists(TrainsFile)) {
trains = JsonSerializer.Deserialize<List<TrainRecord>>(File.ReadAllText(TrainsFile), serializerOptions)!;
}
}
private void Migration() {
if (!File.Exists(DbFile)) {
// using var _ = Logger.BeginScope("Migrating DB version 1 -> 2");
if (!File.Exists(DbFile) && File.Exists(TrainsFile)) {
Logger.LogInformation("Migrating DB version 1 -> 2");
if (File.Exists(StationsFile)) {
Logger.LogDebug("Converting StationsFile");
var oldStations = JsonNode.Parse(File.ReadAllText(StationsFile));
List<StationListing> stations = new();
if (oldStations != null) {
Logger.LogDebug("Found {StationsCount} stations", oldStations.AsArray().Count);
foreach (var station in oldStations.AsArray()) {
if (station == null) continue;
station["stoppedAtBy"] = new JsonArray(station["stoppedAtBy"]!.AsArray().Select(num => (JsonNode)(num!).ToString()!).ToArray());
}
stations = JsonSerializer.Deserialize<List<StationRecord>>(oldStations, serializerOptions)!;
stations = oldStations.Deserialize<List<StationListing>>(serializerOptions)!;
}
Logger.LogDebug("Rewriting StationsFile");
File.WriteAllText(StationsFile, JsonSerializer.Serialize(stations, serializerOptions));
@ -95,6 +70,7 @@ public class Database : Server.Services.Interfaces.IDatabase {
if (File.Exists(TrainsFile)) {
Logger.LogDebug("Converting TrainsFile");
var oldTrains = JsonNode.Parse(File.ReadAllText(TrainsFile));
List<TrainListing> trains = new();
if (oldTrains != null) {
Logger.LogDebug("Found {TrainsCount} trains", oldTrains.AsArray().Count);
foreach (var train in oldTrains.AsArray()) {
@ -102,7 +78,7 @@ public class Database : Server.Services.Interfaces.IDatabase {
train["number"] = train["numberString"];
train.AsObject().Remove("numberString");
}
trains = JsonSerializer.Deserialize<List<TrainRecord>>(oldTrains, serializerOptions)!;
trains = oldTrains.Deserialize<List<TrainListing>>(serializerOptions)!;
}
Logger.LogDebug("Rewriting TrainsFile");
File.WriteAllText(TrainsFile, JsonSerializer.Serialize(trains, serializerOptions));
@ -111,72 +87,101 @@ public class Database : Server.Services.Interfaces.IDatabase {
File.WriteAllText(DbFile, JsonSerializer.Serialize(DbData, serializerOptions));
Migration();
}
else {
else if (File.Exists(DbFile)) {
var oldDbData = JsonNode.Parse(File.ReadAllText(DbFile));
if (((int?)oldDbData?["version"]) == 2) {
Logger.LogInformation("DB Version: 2; noop");
Logger.LogInformation("Migrating DB version 2 -> 3 (transition from fs+JSON to MongoDB)");
if (File.Exists(StationsFile)) {
Logger.LogDebug("Converting StationsFile");
var stations = JsonSerializer.Deserialize<List<StationListing>>(File.ReadAllText(StationsFile));
stationListingsCollection.InsertMany(stations);
File.Delete(StationsFile);
}
if (File.Exists(TrainsFile)) {
Logger.LogDebug("Converting TrainsFile");
var trains = JsonSerializer.Deserialize<List<TrainListing>>(File.ReadAllText(TrainsFile));
trainListingsCollection.InsertMany(trains);
File.Delete(TrainsFile);
}
File.Delete(DbFile);
try {
Directory.Delete(DbDir);
}
catch (Exception) {
// Deleting of the directory is optional; may not be allowed in Docker or similar
}
var x = dbRecordCollection.FindSync(_ => true).ToList()!;
if (x.Count != 0) {
Logger.LogWarning("db collection contained data when migrating to V3");
using (var _ = Logger.BeginScope("Already existing data:")) {
foreach (var dbRecord in x) {
Logger.LogInformation("Id: {Id}, Version: {Version}", dbRecord.Id, dbRecord.Version);
}
}
Logger.LogInformation("Backing up existing data");
var backupDbRecordCollection = db.GetCollection<DbRecord>("db-backup");
backupDbRecordCollection.InsertMany(x);
Logger.LogDebug("Removing existing data");
dbRecordCollection.DeleteMany(_ => true);
}
dbRecordCollection.InsertOne(new(3));
Migration();
}
else {
throw new Exception("Unexpected Database version");
throw new("Unexpected Database version, only DB Version 2 uses DbFile");
}
}
else {
var datas = dbRecordCollection.FindSync(_ => true).ToList();
if (datas.Count == 0) {
Logger.LogInformation("No db record found, new database");
dbRecordCollection.InsertOne(DbData);
}
else {
DbData = datas[0];
}
if (DbData.Version == 3) {
Logger.LogInformation("Using MongoDB Database Version 3; noop");
}
else {
throw new($"Unexpected Database version: {DbData.Version}");
}
}
}
public async Task<string> FoundTrain(string rank, string number, string company) {
number = string.Join("", number.TakeWhile(c => '0' <= c && c <= '9'));
if (!trains.Where(train => train.Number == number).Any()) {
number = string.Join("", number.TakeWhile(c => c is >= '0' and <= '9'));
if (!await (await trainListingsCollection.FindAsync(Builders<TrainListing>.Filter.Eq("number", number))).AnyAsync()) {
Logger.LogDebug("Found train {Rank} {Number} from {Company}", rank, number, company);
trains.Add(new(number, rank, company));
if (shouldCommitOnEveryChange) {
await File.WriteAllTextAsync(TrainsFile, JsonSerializer.Serialize(trains, serializerOptions));
}
else {
trainsDirty = true;
}
await trainListingsCollection.InsertOneAsync(new(number: number, rank: rank, company: company));
}
return number;
}
public async Task FoundStation(string name) {
if (!stations.Where(station => station.Name == name).Any()) {
if (!await (stationListingsCollection.Find(Builders<StationListing>.Filter.Eq("name", name))).AnyAsync()) {
Logger.LogDebug("Found station {StationName}", name);
stations.Add(new(name, new()));
if (shouldCommitOnEveryChange) {
await File.WriteAllTextAsync(StationsFile, JsonSerializer.Serialize(stations, serializerOptions));
}
else {
stationsDirty = true;
}
await stationListingsCollection.InsertOneAsync(new(name, new()));
}
}
public async Task FoundTrainAtStation(string stationName, string trainNumber) {
trainNumber = string.Join("", trainNumber.TakeWhile(c => '0' <= c && c <= '9'));
trainNumber = string.Join("", trainNumber.TakeWhile(c => c is >= '0' and <= '9'));
await FoundStation(stationName);
var dirty = false;
for (var i = 0; i < stations.Count; i++) {
if (stations[i].Name == stationName) {
if (!stations[i].StoppedAtBy.Contains(trainNumber)) {
Logger.LogDebug("Found train {TrainNumber} at station {StationName}", trainNumber, stationName);
stations[i].ActualStoppedAtBy.Add(trainNumber);
dirty = true;
}
break;
}
}
if (dirty) {
if (shouldCommitOnEveryChange) {
stations.Sort((s1, s2) => s2.StoppedAtBy.Count.CompareTo(s1.StoppedAtBy.Count));
await File.WriteAllTextAsync(StationsFile, JsonSerializer.Serialize(stations, serializerOptions));
}
else {
stationsDirty = true;
}
var updateResult = await stationListingsCollection.UpdateOneAsync(
Builders<StationListing>.Filter.Eq("name", stationName),
Builders<StationListing>.Update.AddToSet("stoppedAtBy", trainNumber)
);
if (updateResult.IsAcknowledged && updateResult.ModifiedCount > 0) {
Logger.LogDebug("Found train {TrainNumber} at station {StationName}", trainNumber, stationName);
}
}
public async Task OnTrainData(InfoferScraper.Models.Train.ITrainScrapeResult trainData) {
using var _ = MakeDbTransaction();
var trainNumber = await FoundTrain(trainData.Rank, trainData.Number, trainData.Operator);
foreach (var group in trainData.Groups) {
foreach (var station in group.Stations) {
@ -199,8 +204,6 @@ public class Database : Server.Services.Interfaces.IDatabase {
}
}
using var _ = MakeDbTransaction();
if (stationData.Arrivals != null) {
foreach (var train in stationData.Arrivals) {
await ProcessTrain(train);
@ -214,25 +217,12 @@ public class Database : Server.Services.Interfaces.IDatabase {
}
}
public record DbRecord(int Version);
public record StationRecord : Server.Services.Interfaces.IStationRecord {
[JsonPropertyName("stoppedAtBy")]
public List<string> ActualStoppedAtBy { get; init; }
public string Name { get; init; }
[JsonIgnore]
public IReadOnlyList<string> StoppedAtBy => ActualStoppedAtBy;
public StationRecord() {
Name = "";
ActualStoppedAtBy = new();
}
public StationRecord(string name, List<string> stoppedAtBy) {
Name = name;
ActualStoppedAtBy = stoppedAtBy;
}
public record DbRecord(
[property: BsonId]
[property: BsonRepresentation(BsonType.ObjectId)]
[property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
string? Id,
int Version
) {
public DbRecord(int version) : this(null, version) { }
}
public record TrainRecord(string Number, string Rank, string Company) : Server.Services.Interfaces.ITrainRecord;

16
server/Services/Interfaces/IDatabase.cs
View file

@ -2,12 +2,13 @@ using System.Collections.Generic;
using System.Threading.Tasks;
using InfoferScraper.Models.Train;
using InfoferScraper.Models.Station;
using Server.Models.Database;
namespace Server.Services.Interfaces;
public interface IDatabase {
public IReadOnlyList<IStationRecord> Stations { get; }
public IReadOnlyList<ITrainRecord> Trains { get; }
public IReadOnlyList<StationListing> Stations { get; }
public IReadOnlyList<TrainListing> Trains { get; }
public Task<string> FoundTrain(string rank, string number, string company);
public Task FoundStation(string name);
@ -15,14 +16,3 @@ public interface IDatabase {
public Task OnTrainData(ITrainScrapeResult trainData);
public Task OnStationData(IStationScrapeResult stationData);
}
public interface IStationRecord {
public string Name { get; }
public IReadOnlyList<string> StoppedAtBy { get; }
}
public interface ITrainRecord {
public string Rank { get; }
public string Number { get; }
public string Company { get; }
}

6
server/Startup.cs
View file

@ -8,6 +8,8 @@ using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.OpenApi.Models;
using MongoDB.Bson.Serialization.Conventions;
using Server.Models.Database;
using Server.Services.Implementations;
using Server.Services.Interfaces;
@ -26,6 +28,10 @@ namespace Server {
options.KnownProxies.Add(Dns.GetHostAddresses("host.docker.internal")[0]);
});
}
services.Configure<MongoSettings>(Configuration.GetSection("TrainDataMongo"));
var conventionPack = new ConventionPack { new CamelCaseElementNameConvention() };
ConventionRegistry.Register("camelCase", conventionPack, _ => true);
services.AddSingleton<IDataManager, DataManager>();
services.AddSingleton<IDatabase, Database>();
services.AddSingleton<NodaTime.IDateTimeZoneProvider>(NodaTime.DateTimeZoneProviders.Tzdb);

6
server/appsettings.Development.json
View file

@ -5,5 +5,9 @@
"Microsoft": "Warning",
"Microsoft.Hosting.Lifetime": "Information"
}
}
},
"TrainDataMongo": {
"ConnectionString": "mongodb://localhost:27017",
"DatabaseName": "NewInfoferScraper"
},
}

4
server/appsettings.json
View file

@ -9,5 +9,9 @@
"Microsoft.Hosting.Lifetime": "Information"
}
},
"TrainDataMongo": {
"ConnectionString": "mongodb://mongo:27017",
"DatabaseName": "NewInfoferScraper"
},
"AllowedHosts": "*"
}

1
server/server.csproj
View file

@ -10,6 +10,7 @@
<ItemGroup>
<PackageReference Include="Microsoft.Data.Sqlite" Version="6.0.1" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="5.0.13" />
<PackageReference Include="MongoDB.Driver" Version="2.18.0" />
<PackageReference Include="Nanoid" Version="2.1.0" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="5.6.3" />
</ItemGroup>