Compare commits
No commits in common. "2da10fe7b2d4591fccc366c85d6f81ed8e064812" and "88dd5a53148aef8d88095d62d8fb991fadaff7fd" have entirely different histories.
2da10fe7b2
...
88dd5a5314
8 changed files with 63 additions and 165 deletions
|
|
@ -6,7 +6,7 @@
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<OutputType>Exe</OutputType>
|
<OutputType>Exe</OutputType>
|
||||||
<TargetFrameworks>net6.0;net7.0;net8.0;net9.0</TargetFrameworks>
|
<TargetFrameworks>net6.0;net7.0;net8.0</TargetFrameworks>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,7 @@
|
||||||
using System;
|
using System;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
|
using System.Text.Json;
|
||||||
using System.Threading.Tasks;
|
using System.Threading.Tasks;
|
||||||
using Newtonsoft.Json;
|
|
||||||
using Newtonsoft.Json.Serialization;
|
|
||||||
using InfoferScraper;
|
using InfoferScraper;
|
||||||
using InfoferScraper.Scrapers;
|
using InfoferScraper.Scrapers;
|
||||||
|
|
||||||
|
|
@ -40,13 +39,11 @@ async Task PrintTrain() {
|
||||||
}
|
}
|
||||||
|
|
||||||
Console.WriteLine(
|
Console.WriteLine(
|
||||||
JsonConvert.SerializeObject(
|
JsonSerializer.Serialize(
|
||||||
await new TrainScraper().Scrape(trainNumber),
|
await new TrainScraper().Scrape(trainNumber),
|
||||||
new JsonSerializerSettings {
|
new JsonSerializerOptions {
|
||||||
Formatting = Formatting.Indented,
|
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||||
ContractResolver = new DefaultContractResolver {
|
WriteIndented = true,
|
||||||
NamingStrategy = new CamelCaseNamingStrategy(),
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
|
|
@ -60,13 +57,11 @@ async Task PrintStation() {
|
||||||
}
|
}
|
||||||
|
|
||||||
Console.WriteLine(
|
Console.WriteLine(
|
||||||
JsonConvert.SerializeObject(
|
JsonSerializer.Serialize(
|
||||||
await new StationScraper().Scrape(stationName),
|
await new StationScraper().Scrape(stationName),
|
||||||
new JsonSerializerSettings {
|
new JsonSerializerOptions {
|
||||||
Formatting = Formatting.Indented,
|
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||||
ContractResolver = new DefaultContractResolver {
|
WriteIndented = true,
|
||||||
NamingStrategy = new CamelCaseNamingStrategy(),
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
# https://hub.docker.com/_/microsoft-dotnet
|
# https://hub.docker.com/_/microsoft-dotnet
|
||||||
FROM mcr.microsoft.com/dotnet/sdk:9.0 AS build
|
FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
|
||||||
WORKDIR /source
|
WORKDIR /source
|
||||||
|
|
||||||
# copy csproj and restore as distinct layers
|
# copy csproj and restore as distinct layers
|
||||||
|
|
@ -14,10 +14,10 @@ COPY server/. ./server/
|
||||||
COPY scraper/. ./scraper/
|
COPY scraper/. ./scraper/
|
||||||
COPY ConsoleTest/. ./ConsoleTest/
|
COPY ConsoleTest/. ./ConsoleTest/
|
||||||
WORKDIR /source/server
|
WORKDIR /source/server
|
||||||
RUN dotnet publish -f net9.0 -c release -o /app --no-restore
|
RUN dotnet publish -f net8.0 -c release -o /app --no-restore
|
||||||
|
|
||||||
# final stage/image
|
# final stage/image
|
||||||
FROM mcr.microsoft.com/dotnet/aspnet:9.0
|
FROM mcr.microsoft.com/dotnet/aspnet:8.0
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
COPY --from=build /app ./
|
COPY --from=build /app ./
|
||||||
ENV INSIDE_DOCKER=true
|
ENV INSIDE_DOCKER=true
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
<TargetFrameworks>net6.0;net7.0;net8.0;net9.0</TargetFrameworks>
|
<TargetFrameworks>net6.0;net7.0;net8.0</TargetFrameworks>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
|
|
||||||
40
scraper/src/Models/Train.cs
Executable file → Normal file
40
scraper/src/Models/Train.cs
Executable file → Normal file
|
|
@ -40,11 +40,6 @@ namespace InfoferScraper.Models.Train {
|
||||||
public int Delay { get; }
|
public int Delay { get; }
|
||||||
public string Station { get; }
|
public string Station { get; }
|
||||||
public StatusKind State { get; }
|
public StatusKind State { get; }
|
||||||
/// <summary>
|
|
||||||
/// The time when the real time report was introduced in the system
|
|
||||||
/// </summary>
|
|
||||||
public DateTimeOffset? ReportTime { get; }
|
|
||||||
public ITrainRoute? Between { get; }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public interface ITrainStopDescription {
|
public interface ITrainStopDescription {
|
||||||
|
|
@ -107,9 +102,6 @@ namespace InfoferScraper.Models.Train {
|
||||||
DetachingWagons,
|
DetachingWagons,
|
||||||
ReceivingWagons,
|
ReceivingWagons,
|
||||||
DepartsAs,
|
DepartsAs,
|
||||||
BusReplacementStartingHere,
|
|
||||||
BusReplacement,
|
|
||||||
BusReplacementEndingHere,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#region Implementations
|
#region Implementations
|
||||||
|
|
@ -174,14 +166,6 @@ namespace InfoferScraper.Models.Train {
|
||||||
public int Delay { get; set; }
|
public int Delay { get; set; }
|
||||||
public string Station { get; set; } = "";
|
public string Station { get; set; } = "";
|
||||||
public StatusKind State { get; set; }
|
public StatusKind State { get; set; }
|
||||||
public DateTimeOffset? ReportTime { get; set; }
|
|
||||||
public ITrainRoute? Between { get; set; }
|
|
||||||
|
|
||||||
internal void MakeBetween(Action<TrainRoute> configurator) {
|
|
||||||
TrainRoute newRoute = new();
|
|
||||||
configurator(newRoute);
|
|
||||||
Between = newRoute;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
internal record TrainStopDescription : ITrainStopDescription {
|
internal record TrainStopDescription : ITrainStopDescription {
|
||||||
|
|
@ -230,18 +214,6 @@ namespace InfoferScraper.Models.Train {
|
||||||
public string Station { get; set; } = "";
|
public string Station { get; set; } = "";
|
||||||
}
|
}
|
||||||
|
|
||||||
class BusReplacementStartingHereNote : ITrainStopNote {
|
|
||||||
public NoteKind Kind => NoteKind.BusReplacementStartingHere;
|
|
||||||
}
|
|
||||||
|
|
||||||
class BusReplacementNote : ITrainStopNote {
|
|
||||||
public NoteKind Kind => NoteKind.BusReplacement;
|
|
||||||
}
|
|
||||||
|
|
||||||
class BusReplacementEndingHereNote : ITrainStopNote {
|
|
||||||
public NoteKind Kind => NoteKind.BusReplacementEndingHere;
|
|
||||||
}
|
|
||||||
|
|
||||||
internal void AddDepartsAsNote(string rank, string number, DateTimeOffset departureDate) {
|
internal void AddDepartsAsNote(string rank, string number, DateTimeOffset departureDate) {
|
||||||
ModifyableNotes.Add(new DepartsAsNote { Rank = rank, Number = number, DepartureDate = departureDate });
|
ModifyableNotes.Add(new DepartsAsNote { Rank = rank, Number = number, DepartureDate = departureDate });
|
||||||
}
|
}
|
||||||
|
|
@ -257,18 +229,6 @@ namespace InfoferScraper.Models.Train {
|
||||||
internal void AddDetachingWagonsNote(string station) {
|
internal void AddDetachingWagonsNote(string station) {
|
||||||
ModifyableNotes.Add(new DetachingWagonsNote { Station = station });
|
ModifyableNotes.Add(new DetachingWagonsNote { Station = station });
|
||||||
}
|
}
|
||||||
|
|
||||||
internal void AddBusReplacementStartingHereNote() {
|
|
||||||
ModifyableNotes.Add(new BusReplacementStartingHereNote {});
|
|
||||||
}
|
|
||||||
|
|
||||||
internal void AddBusReplacementNote() {
|
|
||||||
ModifyableNotes.Add(new BusReplacementNote {});
|
|
||||||
}
|
|
||||||
|
|
||||||
internal void AddBusReplacementEndingHereNote() {
|
|
||||||
ModifyableNotes.Add(new BusReplacementEndingHereNote {});
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public record TrainStopArrDep : ITrainStopArrDep {
|
public record TrainStopArrDep : ITrainStopArrDep {
|
||||||
|
|
|
||||||
90
scraper/src/Scrapers/Train.cs
Executable file → Normal file
90
scraper/src/Scrapers/Train.cs
Executable file → Normal file
|
|
@ -1,4 +1,4 @@
|
||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
using System.Net;
|
using System.Net;
|
||||||
|
|
@ -28,10 +28,6 @@ namespace InfoferScraper.Scrapers {
|
||||||
new(
|
new(
|
||||||
@"^(?:Fără|([0-9]+)\smin)\s(întârziere|mai\sdevreme)\sla\s(trecerea\sfără\soprire\sprin|sosirea\sîn|plecarea\sdin)\s(.+)(?:\s\(Raportat\sla\s([0-9]+):([0-9]+)\))?\.");
|
@"^(?:Fără|([0-9]+)\smin)\s(întârziere|mai\sdevreme)\sla\s(trecerea\sfără\soprire\sprin|sosirea\sîn|plecarea\sdin)\s(.+)(?:\s\(Raportat\sla\s([0-9]+):([0-9]+)\))?\.");
|
||||||
|
|
||||||
private static readonly Regex SlExpandedRegex =
|
|
||||||
new(
|
|
||||||
@"^(?:Fără|([0-9]+)\smin)\s(întârziere|mai\sdevreme)\sla\s(trecerea\sfără\soprire\sprin|sosirea\sîn|plecarea\sdin)\s(.+)(?:\s\(Raportat\sla\s([0-9]+):([0-9]+)\))?\.\sConform\sitinerariului,\strenul\sse\saflă\sîntre\sstațiile\s(.+)\s-\s(.+)\.\sPuteți\sapăsa\spe\sbutonul\s”Hartă”\spentru\sa\svedea\slocația.");
|
|
||||||
|
|
||||||
private static readonly Dictionary<char, StatusKind> SlStateMap = new() {
|
private static readonly Dictionary<char, StatusKind> SlStateMap = new() {
|
||||||
{ 't', StatusKind.Passing },
|
{ 't', StatusKind.Passing },
|
||||||
{ 's', StatusKind.Arrival },
|
{ 's', StatusKind.Arrival },
|
||||||
|
|
@ -53,12 +49,6 @@ namespace InfoferScraper.Scrapers {
|
||||||
new(@"^Trenul primește vagoane de la\s(.+)\.$");
|
new(@"^Trenul primește vagoane de la\s(.+)\.$");
|
||||||
private static readonly Regex DetachingWagonsNoteRegex =
|
private static readonly Regex DetachingWagonsNoteRegex =
|
||||||
new(@"^Trenul detașează vagoane pentru stația\s(.+)\.$");
|
new(@"^Trenul detașează vagoane pentru stația\s(.+)\.$");
|
||||||
private static readonly Regex BusReplacementStartingHereNoteRegex =
|
|
||||||
new(@"^\s*Transfer\scu\sautobuzul\sîncepând\scu\saceastă\sstație\s*$");
|
|
||||||
private static readonly Regex BusReplacementNoteRegex =
|
|
||||||
new(@"^Transfer cu autobuzul$");
|
|
||||||
private static readonly Regex BusReplacementEndingHereNoteRegex =
|
|
||||||
new(@"^\s*Transfer\scu\sautobuzul\spână\sla\saceastă\sstație\s*$");
|
|
||||||
|
|
||||||
private static readonly DateTimeZone BucharestTz = DateTimeZoneProviders.Tzdb["Europe/Bucharest"];
|
private static readonly DateTimeZone BucharestTz = DateTimeZoneProviders.Tzdb["Europe/Bucharest"];
|
||||||
|
|
||||||
|
|
@ -154,6 +144,22 @@ namespace InfoferScraper.Scrapers {
|
||||||
.Select(group => group.Value);
|
.Select(group => group.Value);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
var statusLineMatch =
|
||||||
|
SlRegex.Match(statusDiv.QuerySelector(":scope > div")!.Text().WithCollapsedSpaces());
|
||||||
|
var (slmDelay, (slmLate, (slmArrival, (slmStation, _)))) =
|
||||||
|
(statusLineMatch.Groups as IEnumerable<Group>).Skip(1).Select(group => group.Value);
|
||||||
|
group.MakeStatus(status => {
|
||||||
|
status.Delay = string.IsNullOrEmpty(slmDelay) ? 0 :
|
||||||
|
slmLate == "întârziere" ? int.Parse(slmDelay) : -int.Parse(slmDelay);
|
||||||
|
status.Station = slmStation;
|
||||||
|
status.State = SlStateMap[slmArrival[0]];
|
||||||
|
});
|
||||||
|
}
|
||||||
|
catch {
|
||||||
|
// ignored
|
||||||
|
}
|
||||||
|
|
||||||
Utils.DateTimeSequencer dtSeq = new(date.Year, date.Month, date.Day);
|
Utils.DateTimeSequencer dtSeq = new(date.Year, date.Month, date.Day);
|
||||||
var stations = statusDiv.QuerySelectorAll(":scope > ul > li");
|
var stations = statusDiv.QuerySelectorAll(":scope > ul > li");
|
||||||
foreach (var station in stations) {
|
foreach (var station in stations) {
|
||||||
|
|
@ -236,7 +242,7 @@ namespace InfoferScraper.Scrapers {
|
||||||
|
|
||||||
foreach (var noteDiv in stopNotes.QuerySelectorAll(":scope > div > div")) {
|
foreach (var noteDiv in stopNotes.QuerySelectorAll(":scope > div > div")) {
|
||||||
var noteText = noteDiv.Text().WithCollapsedSpaces();
|
var noteText = noteDiv.Text().WithCollapsedSpaces();
|
||||||
Match trainNumberChangeMatch, departsAsMatch, detachingWagons, receivingWagons, busReplacementStart, busReplacement, busReplacementEnd;
|
Match trainNumberChangeMatch, departsAsMatch, detachingWagons, receivingWagons;
|
||||||
if ((trainNumberChangeMatch = TrainNumberChangeNoteRegex.Match(noteText)).Success) {
|
if ((trainNumberChangeMatch = TrainNumberChangeNoteRegex.Match(noteText)).Success) {
|
||||||
stopDescription.AddTrainNumberChangeNote(trainNumberChangeMatch.Groups[1].Value, trainNumberChangeMatch.Groups[2].Value);
|
stopDescription.AddTrainNumberChangeNote(trainNumberChangeMatch.Groups[1].Value, trainNumberChangeMatch.Groups[2].Value);
|
||||||
}
|
}
|
||||||
|
|
@ -251,69 +257,9 @@ namespace InfoferScraper.Scrapers {
|
||||||
else if ((receivingWagons = ReceivingWagonsNoteRegex.Match(noteText)).Success) {
|
else if ((receivingWagons = ReceivingWagonsNoteRegex.Match(noteText)).Success) {
|
||||||
stopDescription.AddReceivingWagonsNote(receivingWagons.Groups[1].Value);
|
stopDescription.AddReceivingWagonsNote(receivingWagons.Groups[1].Value);
|
||||||
}
|
}
|
||||||
else if ((busReplacementStart = BusReplacementStartingHereNoteRegex.Match(noteText)).Success) {
|
|
||||||
stopDescription.AddBusReplacementStartingHereNote();
|
|
||||||
}
|
|
||||||
else if ((busReplacement = BusReplacementNoteRegex.Match(noteText)).Success) {
|
|
||||||
stopDescription.AddBusReplacementNote();
|
|
||||||
}
|
|
||||||
else if ((busReplacementEnd = BusReplacementEndingHereNoteRegex.Match(noteText)).Success) {
|
|
||||||
stopDescription.AddBusReplacementEndingHereNote();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
|
||||||
var statusLineText = statusDiv.QuerySelector(":scope > div")!.Text().WithCollapsedSpaces();
|
|
||||||
var statusLineExpandedMatch = SlExpandedRegex.Match(statusLineText);
|
|
||||||
var statusLineMatch = SlRegex.Match(statusLineText);
|
|
||||||
|
|
||||||
var realMatch = statusLineExpandedMatch.Success ? statusLineExpandedMatch
|
|
||||||
: statusLineMatch.Success ? statusLineMatch
|
|
||||||
: null;
|
|
||||||
|
|
||||||
if (realMatch != null) {
|
|
||||||
var (slmDelay, (slmLate, (slmArrival, (slmStation,
|
|
||||||
(slmReportH, (slmReportM, (slmBetweenF, (slmBetweenT, _)))))))) =
|
|
||||||
(realMatch.Groups as IEnumerable<Group>).Skip(1).Select(group => group.Value);
|
|
||||||
group.MakeStatus(status => {
|
|
||||||
status.Delay = string.IsNullOrEmpty(slmDelay) ? 0 :
|
|
||||||
slmLate == "întârziere" ? int.Parse(slmDelay) : -int.Parse(slmDelay);
|
|
||||||
status.Station = slmStation;
|
|
||||||
status.State = SlStateMap[slmArrival[0]];
|
|
||||||
if (!string.IsNullOrEmpty(slmReportH) && !string.IsNullOrEmpty(slmReportM)) {
|
|
||||||
var firstDeparture = group.Stations[0].Departure!.ScheduleTime;
|
|
||||||
var potentialReportTime = BucharestTz
|
|
||||||
.AtLeniently(
|
|
||||||
new DateTime(firstDeparture.Year, firstDeparture.Month, firstDeparture.Day, int.Parse(slmReportH), int.Parse(slmReportM), 0)
|
|
||||||
.ToLocalDateTime()
|
|
||||||
)
|
|
||||||
.ToDateTimeOffset();
|
|
||||||
if (potentialReportTime < firstDeparture) {
|
|
||||||
// Assume no reports come in before the train departs
|
|
||||||
var nextDay = firstDeparture.AddDays(1);
|
|
||||||
potentialReportTime = BucharestTz
|
|
||||||
.AtLeniently(
|
|
||||||
new DateTime(nextDay.Year, nextDay.Month, nextDay.Day, potentialReportTime.Hour, potentialReportTime.Minute, 0)
|
|
||||||
.ToLocalDateTime()
|
|
||||||
)
|
|
||||||
.ToDateTimeOffset();
|
|
||||||
}
|
|
||||||
status.ReportTime = potentialReportTime;
|
|
||||||
}
|
|
||||||
if (!string.IsNullOrEmpty(slmBetweenF) && !string.IsNullOrEmpty(slmBetweenT)) {
|
|
||||||
status.MakeBetween(between => {
|
|
||||||
between.From = slmBetweenF;
|
|
||||||
between.To = slmBetweenT;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch {
|
|
||||||
// ignored
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
return result;
|
return result;
|
||||||
|
|
|
||||||
5
scraper/src/Utils/DateTimeSequencer.cs
Executable file → Normal file
5
scraper/src/Utils/DateTimeSequencer.cs
Executable file → Normal file
|
|
@ -16,10 +16,7 @@ namespace InfoferScraper {
|
||||||
|
|
||||||
public DateTime Next(int hour, int minute = 0, int second = 0) {
|
public DateTime Next(int hour, int minute = 0, int second = 0) {
|
||||||
DateTime potentialNewDate = new(_current.Year, _current.Month, _current.Day, hour, minute, second);
|
DateTime potentialNewDate = new(_current.Year, _current.Month, _current.Day, hour, minute, second);
|
||||||
if (_current >= potentialNewDate) {
|
if (_current > potentialNewDate) potentialNewDate = potentialNewDate.AddDays(1);
|
||||||
_current = potentialNewDate.AddDays(1);
|
|
||||||
potentialNewDate = new(_current.Year, _current.Month, _current.Day, hour, minute, second);
|
|
||||||
}
|
|
||||||
_current = potentialNewDate;
|
_current = potentialNewDate;
|
||||||
return _current;
|
return _current;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@
|
||||||
<AssemblyName>Server</AssemblyName>
|
<AssemblyName>Server</AssemblyName>
|
||||||
<RootNamespace>Server</RootNamespace>
|
<RootNamespace>Server</RootNamespace>
|
||||||
<LangVersion>11</LangVersion>
|
<LangVersion>11</LangVersion>
|
||||||
<TargetFrameworks>net6.0;net7.0;net8.0;net9.0</TargetFrameworks>
|
<TargetFrameworks>net6.0;net7.0;net8.0</TargetFrameworks>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue