kbruen/new-infofer-scraper
1
0
Fork 0
Code Pull requests Releases Activity

Use Newtonsoft.Json instead of System.Text.Json

Browse source 
Fixed bug related to serializing properties inherited from interface
This commit is contained in:
Kenneth Bruen 2023-09-03 04:13:26 +03:00
parent 8421b49834
commit 9e680e3bb2
Signed by: kbruen
GPG key ID: C1980A470C3EE5B1
8 changed files with 42 additions and 109 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
scraper/scraper.csproj
View file

@ -9,6 +9,7 @@
<PackageReference Include="AngleSharp" Version="0.16.0" />
<PackageReference Include="Flurl" Version="3.0.2" />
<PackageReference Include="Jetbrains.Annotations" Version="2021.2.0" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="NodaTime" Version="3.0.5" />
</ItemGroup>

82
scraper/src/Models/Train.cs
View file

@ -1,9 +1,9 @@
using System;
using System.Collections.Generic;
using System.Text.Json;
using System.Text.Json.Serialization;
using InfoferScraper.Models.Status;
using InfoferScraper.Models.Train.JsonConverters;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using Newtonsoft.Json.Serialization;
namespace InfoferScraper.Models.Train {
#region Interfaces
@ -89,14 +89,14 @@ namespace InfoferScraper.Models.Train {
#endregion
[JsonConverter(typeof(StatusKindConverter))]
[JsonConverter(typeof(StringEnumConverter), typeof(CamelCaseNamingStrategy))]
public enum StatusKind {
Passing,
Arrival,
Departure,
}
[JsonConverter(typeof(NoteKindConverter))]
[JsonConverter(typeof(StringEnumConverter), typeof(CamelCaseNamingStrategy))]
public enum NoteKind {
TrainNumberChange,
DetachingWagons,
@ -243,76 +243,4 @@ namespace InfoferScraper.Models.Train {
}
#endregion
#region JSON Converters
namespace JsonConverters {
internal class StatusKindConverter : JsonConverterFactory {
public override bool CanConvert(Type typeToConvert) {
return typeToConvert == typeof(StatusKind);
}
public override JsonConverter? CreateConverter(Type typeToConvert, JsonSerializerOptions options) {
return new Converter();
}
private class Converter : JsonConverter<StatusKind> {
public override StatusKind Read(
ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options
) {
return reader.GetString() switch {
"arrival" => StatusKind.Arrival,
"departure" => StatusKind.Departure,
"passing" => StatusKind.Passing,
_ => throw new NotImplementedException()
};
}
public override void Write(Utf8JsonWriter writer, StatusKind value, JsonSerializerOptions options) {
writer.WriteStringValue(value switch {
StatusKind.Passing => "passing",
StatusKind.Arrival => "arrival",
StatusKind.Departure => "departure",
_ => throw new NotImplementedException()
});
}
}
}
internal class NoteKindConverter : JsonConverterFactory {
public override bool CanConvert(Type typeToConvert) {
return typeToConvert == typeof(NoteKind);
}
public override JsonConverter? CreateConverter(Type typeToConvert, JsonSerializerOptions options) {
return new Converter();
}
private class Converter : JsonConverter<NoteKind> {
public override NoteKind Read(
ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options
) {
return reader.GetString() switch {
"departsAs" => NoteKind.DepartsAs,
"trainNumberChange" => NoteKind.TrainNumberChange,
"receivingWagons" => NoteKind.ReceivingWagons,
"detachingWagons" => NoteKind.DetachingWagons,
_ => throw new NotImplementedException()
};
}
public override void Write(Utf8JsonWriter writer, NoteKind value, JsonSerializerOptions options) {
writer.WriteStringValue(value switch {
NoteKind.DepartsAs => "departsAs",
NoteKind.TrainNumberChange => "trainNumberChange",
NoteKind.DetachingWagons => "detachingWagons",
NoteKind.ReceivingWagons => "receivingWagons",
_ => throw new NotImplementedException()
});
}
}
}
}
#endregion
}

4
server/Models/Database/StationAlias.cs
View file

@ -1,13 +1,13 @@
using System.Text.Json.Serialization;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
using Newtonsoft.Json;
namespace Server.Models.Database;
public record StationAlias(
[property: BsonId]
[property: BsonRepresentation(BsonType.ObjectId)]
[property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
[property: JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
string? Id,
string Name,
[property: BsonRepresentation(BsonType.ObjectId)]

4
server/Models/Database/StationListing.cs
View file

@ -1,14 +1,14 @@
using System.Collections.Generic;
using System.Text.Json.Serialization;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
using Newtonsoft.Json;
namespace Server.Models.Database;
public record StationListing(
[property: BsonId]
[property: BsonRepresentation(BsonType.ObjectId)]
[property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
[property: JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
string? Id,
string Name,
List<string> StoppedAtBy

5
server/Models/Database/TrainListing.cs
View file

@ -1,14 +1,13 @@
using System.Collections.Generic;
using System.Text.Json.Serialization;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
using Newtonsoft.Json;
namespace Server.Models.Database;
public record TrainListing(
[property: BsonId]
[property: BsonRepresentation(BsonType.ObjectId)]
[property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
[property: JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
string? Id,
string Rank,
string Number,

46
server/Services/Implementations/Database.cs
View file

@ -2,9 +2,6 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Nodes;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using InfoferScraper.Models.Station;
@ -13,6 +10,9 @@ using Microsoft.Extensions.Options;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
using MongoDB.Driver;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Serialization;
using scraper.Models.Itinerary;
using Server.Models.Database;
using Server.Utils;
@ -20,8 +20,10 @@ using Server.Utils;
namespace Server.Services.Implementations;
public class Database : Server.Services.Interfaces.IDatabase {
private static readonly JsonSerializerOptions serializerOptions = new() {
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
private static readonly JsonSerializerSettings jsonSerializerSettings = new() {
ContractResolver = new DefaultContractResolver {
NamingStrategy = new CamelCaseNamingStrategy(),
},
};
private ILogger<Database> Logger { get; }
@ -79,54 +81,54 @@ public class Database : Server.Services.Interfaces.IDatabase {
Logger.LogInformation("Migrating DB version 1 -> 2");
if (File.Exists(StationsFile)) {
Logger.LogDebug("Converting StationsFile");
var oldStations = JsonNode.Parse(File.ReadAllText(StationsFile));
var oldStations = JToken.Parse(File.ReadAllText(StationsFile));
List<StationListing> stations = new();
if (oldStations != null) {
Logger.LogDebug("Found {StationsCount} stations", oldStations.AsArray().Count);
foreach (var station in oldStations.AsArray()) {
Logger.LogDebug("Found {StationsCount} stations", oldStations.Children().Count());
foreach (var station in oldStations.Children()) {
if (station == null) continue;
station["stoppedAtBy"] = new JsonArray(station["stoppedAtBy"]!.AsArray().Select(num => (JsonNode)(num!).ToString()!).ToArray());
station["stoppedAtBy"] = new JArray(station["stoppedAtBy"]!.Children().Select(num => (JToken)(num!).ToString()!).ToArray());
}
stations = oldStations.Deserialize<List<StationListing>>(serializerOptions)!;
stations = oldStations.ToObject<List<StationListing>>(JsonSerializer.Create(jsonSerializerSettings))!;
}
Logger.LogDebug("Rewriting StationsFile");
File.WriteAllText(StationsFile, JsonSerializer.Serialize(stations, serializerOptions));
File.WriteAllText(StationsFile, JsonConvert.SerializeObject(stations, jsonSerializerSettings));
}
if (File.Exists(TrainsFile)) {
Logger.LogDebug("Converting TrainsFile");
var oldTrains = JsonNode.Parse(File.ReadAllText(TrainsFile));
var oldTrains = JToken.Parse(File.ReadAllText(TrainsFile));
List<TrainListing> trains = new();
if (oldTrains != null) {
Logger.LogDebug("Found {TrainsCount} trains", oldTrains.AsArray().Count);
foreach (var train in oldTrains.AsArray()) {
Logger.LogDebug("Found {TrainsCount} trains", oldTrains.Children().Count());
foreach (var train in oldTrains.Children()) {
if (train == null) continue;
train["number"] = train["numberString"];
train.AsObject().Remove("numberString");
train["numberString"]?.Remove();
}
trains = oldTrains.Deserialize<List<TrainListing>>(serializerOptions)!;
trains = oldTrains.ToObject<List<TrainListing>>(JsonSerializer.Create(jsonSerializerSettings))!;
}
Logger.LogDebug("Rewriting TrainsFile");
File.WriteAllText(TrainsFile, JsonSerializer.Serialize(trains, serializerOptions));
File.WriteAllText(TrainsFile, JsonConvert.SerializeObject(trains, jsonSerializerSettings));
}
DbData = new(2);
File.WriteAllText(DbFile, JsonSerializer.Serialize(DbData, serializerOptions));
File.WriteAllText(DbFile, JsonConvert.SerializeObject(DbData, jsonSerializerSettings));
Migration();
}
else if (File.Exists(DbFile)) {
var oldDbData = JsonNode.Parse(File.ReadAllText(DbFile));
var oldDbData = JToken.Parse(File.ReadAllText(DbFile));
if (((int?)oldDbData?["version"]) == 2) {
Logger.LogInformation("Migrating DB version 2 -> 3 (transition from fs+JSON to MongoDB)");
if (File.Exists(StationsFile)) {
Logger.LogDebug("Converting StationsFile");
var stations = JsonSerializer.Deserialize<List<StationListing>>(File.ReadAllText(StationsFile));
var stations = JsonConvert.DeserializeObject<List<StationListing>>(File.ReadAllText(StationsFile));
stationListingsCollection.InsertMany(stations);
File.Delete(StationsFile);
}
if (File.Exists(TrainsFile)) {
Logger.LogDebug("Converting TrainsFile");
var trains = JsonSerializer.Deserialize<List<TrainListing>>(File.ReadAllText(TrainsFile));
var trains = JsonConvert.DeserializeObject<List<TrainListing>>(File.ReadAllText(TrainsFile));
trainListingsCollection.InsertMany(trains);
File.Delete(TrainsFile);
}
@ -380,7 +382,7 @@ public class Database : Server.Services.Interfaces.IDatabase {
public record DbRecord(
[property: BsonId]
[property: BsonRepresentation(BsonType.ObjectId)]
[property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
[property: JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
string? Id,
int Version
) {

8
server/Startup.cs
View file

@ -1,6 +1,5 @@
using System;
using System.Net;
using System.Text.Json;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Http;
@ -10,6 +9,7 @@ using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.OpenApi.Models;
using MongoDB.Bson.Serialization.Conventions;
using Newtonsoft.Json.Serialization;
using Server.Models.Database;
using Server.Services.Implementations;
using Server.Services.Interfaces;
@ -37,8 +37,10 @@ namespace Server {
services.AddSingleton<IDatabase, Database>();
services.AddSingleton<NodaTime.IDateTimeZoneProvider>(NodaTime.DateTimeZoneProviders.Tzdb);
services.AddControllers()
.AddJsonOptions(options => {
options.JsonSerializerOptions.PropertyNamingPolicy = JsonNamingPolicy.CamelCase;
.AddNewtonsoftJson(options => {
options.SerializerSettings.ContractResolver = new DefaultContractResolver {
NamingStrategy = new CamelCaseNamingStrategy(),
};
});
services.AddSwaggerGen(c => {
c.SwaggerDoc("v1", new OpenApiInfo { Title = "InfoTren Scraper", Version = "v1" });

1
server/server.csproj
View file

@ -9,6 +9,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="6.0.21" />
<PackageReference Include="Microsoft.Data.Sqlite" Version="6.0.1" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="5.0.13" />
<PackageReference Include="MongoDB.Analyzer" Version="1.1.0" />