kbruen/new-infofer-scraper
1
0
Fork 0
Code Pull requests Releases Activity

Compare commits

base: kbruen:c77168b393b3027934354c113124d91293d5117f
Branches Tags
kbruen:master
kbruen:python3
kbruen:python3
..
compare: kbruen:a9f1838fd68d9019f309254f8e696853846b9e1a
Branches Tags
kbruen:master
kbruen:python3
kbruen:python3

2 commits
c77168b393 ... a9f1838fd6

Author SHA1 Message Date
Dan Cojocaru
a9f1838fd6
Support new status line format 2025-01-03 06:51:16 +01:00
Dan Cojocaru
6c6d69ee26
Add UserAgent 2025-01-03 06:50:58 +01:00
3 changed files with 250 additions and 229 deletions
Show stats Expand all files Collapse all files

7
scraper/src/Scrapers/Route.cs
View file

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using AngleSharp;
@ -57,6 +58,12 @@ public class RouteScraper {
httpClient = new HttpClient(httpClientHandler) {
BaseAddress = new Uri(BaseUrl),
DefaultRequestVersion = new Version(2, 0),
DefaultRequestHeaders = {
UserAgent = {
new ProductInfoHeaderValue("new-infofer-scraper", "0.0.1"),
new ProductInfoHeaderValue("(developed by DC Dev as a hobby, get in touch at webmaster<at>dcdev.ro for any issues)"),
},
},
};
}

7
scraper/src/Scrapers/Station.cs
View file

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using AngleSharp;
@ -51,6 +52,12 @@ namespace InfoferScraper.Scrapers {
httpClient = new HttpClient(httpClientHandler) {
BaseAddress = new Uri(BaseUrl),
DefaultRequestVersion = new Version(2, 0),
DefaultRequestHeaders = {
UserAgent = {
new ProductInfoHeaderValue("new-infofer-scraper", "0.0.1"),
new ProductInfoHeaderValue("(developed by DC Dev as a hobby, get in touch at webmaster<at>dcdev.ro for any issues)"),
},
},
};
}

9
scraper/src/Scrapers/Train.cs
View file

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using AngleSharp;
@ -25,7 +26,7 @@ namespace InfoferScraper.Scrapers {
private static readonly Regex SlRegex =
new(
@"^(?:Fără|([0-9]+)\smin)\s(întârziere|mai\sdevreme)\sla\s(trecerea\sfără\soprire\sprin|sosirea\sîn|plecarea\sdin)\s(.+)\.$");
@"^(?:Fără|([0-9]+)\smin)\s(întârziere|mai\sdevreme)\sla\s(trecerea\sfără\soprire\sprin|sosirea\sîn|plecarea\sdin)\s(.+)(?:\s\(Raportat\sla\s([0-9]+):([0-9]+)\))?\.");
private static readonly Dictionary<char, StatusKind> SlStateMap = new() {
{ 't', StatusKind.Passing },
@ -69,6 +70,12 @@ namespace InfoferScraper.Scrapers {
httpClient = new HttpClient(httpClientHandler) {
BaseAddress = new Uri(BaseUrl),
DefaultRequestVersion = new Version(2, 0),
DefaultRequestHeaders = {
UserAgent = {
new ProductInfoHeaderValue("new-infofer-scraper", "0.0.1"),
new ProductInfoHeaderValue("(developed by DC Dev as a hobby, get in touch at webmaster<at>dcdev.ro for any issues)"),
},
},
};
}