request() via profile 💥

This commit is contained in:
Jannis R 2019-10-31 18:48:11 +01:00
parent 4652c1694e
commit 6d5c6081ce
No known key found for this signature in database
GPG key ID: 0FE83946296A88A5
8 changed files with 67 additions and 82 deletions

View file

@ -21,13 +21,12 @@
There's opt-in support for throttling requests to the endpoint. There's opt-in support for throttling requests to the endpoint.
```js ```js
const withThrottling = require('hafas-client/throttle')
const createClient = require('hafas-client') const createClient = require('hafas-client')
const withThrottling = require('hafas-client/throttle')
const dbProfile = require('hafas-client/p/db') const dbProfile = require('hafas-client/p/db')
// create a throttled HAFAS client with Deutsche Bahn profile // create a throttled HAFAS client with Deutsche Bahn profile
const createThrottledClient = withThrottling(createClient) const client = createClient(withThrottling(dbProfile), 'my-awesome-program')
const client = createThrottledClient(dbProfile, 'my-awesome-program')
// Berlin Jungfernheide to München Hbf // Berlin Jungfernheide to München Hbf
client.journeys('8011167', '8000261', {results: 1}) client.journeys('8011167', '8000261', {results: 1})
@ -39,8 +38,8 @@ You can pass custom values for the nr of requests (`limit`) per interval into `w
```js ```js
// 2 requests per second // 2 requests per second
const createThrottledClient = withThrottling(createClient, 2, 1000) const throttledDbProfile = withThrottling(dbProfile, 2, 1000)
const client = createThrottledClient(dbProfile, 'my-awesome-program') const client = createClient(throttledDbProfile, 'my-awesome-program')
``` ```
## Retrying failed requests ## Retrying failed requests
@ -48,13 +47,12 @@ const client = createThrottledClient(dbProfile, 'my-awesome-program')
There's opt-in support for retrying failed requests to the endpoint. There's opt-in support for retrying failed requests to the endpoint.
```js ```js
const withRetrying = require('hafas-client/retry')
const createClient = require('hafas-client') const createClient = require('hafas-client')
const withRetrying = require('hafas-client/retry')
const dbProfile = require('hafas-client/p/db') const dbProfile = require('hafas-client/p/db')
// create a client with Deutsche Bahn profile that will retry on HAFAS errors // create a client with Deutsche Bahn profile that will retry on HAFAS errors
const createRetryingClient = withRetrying(createClient) const client = createClient(withRetrying(dbProfile), 'my-awesome-program')
const client = createRetryingClient(dbProfile, 'my-awesome-program')
// Berlin Jungfernheide to München Hbf // Berlin Jungfernheide to München Hbf
client.journeys('8011167', '8000261', {results: 1}) client.journeys('8011167', '8000261', {results: 1})
@ -66,12 +64,12 @@ You can pass custom options into `withRetrying`. They will be passed into [`retr
```js ```js
// retry 2 times, after 10 seconds & 30 seconds // retry 2 times, after 10 seconds & 30 seconds
const createRetryingClient = withRetrying(createClient, { const retryingDbProfile = withRetrying(dbProfile, {
retries: 2, retries: 2,
minTimeout: 10 * 1000, minTimeout: 10 * 1000,
factor: 3 factor: 3
}) })
const client = createRetryingClient(dbProfile, 'my-awesome-program') const client = createClient(retryingDbProfile, 'my-awesome-program')
``` ```
## Writing a profile ## Writing a profile

View file

@ -9,7 +9,6 @@ const pRetry = require('p-retry')
const defaultProfile = require('./lib/default-profile') const defaultProfile = require('./lib/default-profile')
const createFormatProductsFilter = require('./format/products-filter') const createFormatProductsFilter = require('./format/products-filter')
const validateProfile = require('./lib/validate-profile') const validateProfile = require('./lib/validate-profile')
const _request = require('./lib/request')
const isNonEmptyString = str => 'string' === typeof str && str.length > 0 const isNonEmptyString = str => 'string' === typeof str && str.length > 0
@ -25,10 +24,6 @@ const validateLocation = (loc, name = 'location') => {
} }
} }
const defaults = {
request: _request
}
const createClient = (profile, userAgent, opt = {}) => { const createClient = (profile, userAgent, opt = {}) => {
profile = Object.assign({}, defaultProfile, profile) profile = Object.assign({}, defaultProfile, profile)
if (!profile.formatProductsFilter) { if (!profile.formatProductsFilter) {
@ -40,10 +35,6 @@ const createClient = (profile, userAgent, opt = {}) => {
throw new TypeError('userAgent must be a string'); throw new TypeError('userAgent must be a string');
} }
const {
request
} = Object.assign({}, defaults, opt)
const _stationBoard = (station, type, parse, opt = {}) => { const _stationBoard = (station, type, parse, opt = {}) => {
if (isObj(station)) station = profile.formatStation(station.id) if (isObj(station)) station = profile.formatStation(station.id)
else if ('string' === typeof station) station = profile.formatStation(station) else if ('string' === typeof station) station = profile.formatStation(station)
@ -88,7 +79,7 @@ const createClient = (profile, userAgent, opt = {}) => {
if (profile.departuresGetPasslist) req.getPasslist = !!opt.stopovers if (profile.departuresGetPasslist) req.getPasslist = !!opt.stopovers
if (profile.departuresStbFltrEquiv) req.stbFltrEquiv = !opt.includeRelatedStations if (profile.departuresStbFltrEquiv) req.stbFltrEquiv = !opt.includeRelatedStations
return request({profile, opt}, userAgent, { return profile.request({profile, opt}, userAgent, {
meth: 'StationBoard', meth: 'StationBoard',
req req
}) })
@ -230,7 +221,7 @@ const createClient = (profile, userAgent, opt = {}) => {
if (profile.journeysNumF && opt.results !== null) query.numF = opt.results if (profile.journeysNumF && opt.results !== null) query.numF = opt.results
if (profile.journeysOutFrwd) query.outFrwd = outFrwd if (profile.journeysOutFrwd) query.outFrwd = outFrwd
return request({profile, opt}, userAgent, { return profile.request({profile, opt}, userAgent, {
cfg: {polyEnc: 'GPA'}, cfg: {polyEnc: 'GPA'},
meth: 'TripSearch', meth: 'TripSearch',
req: profile.transformJourneysQuery({profile, opt}, query) req: profile.transformJourneysQuery({profile, opt}, query)
@ -277,7 +268,7 @@ const createClient = (profile, userAgent, opt = {}) => {
remarks: true // parse & expose hints & warnings? remarks: true // parse & expose hints & warnings?
}, opt) }, opt)
return request({profile, opt}, userAgent, { return profile.request({profile, opt}, userAgent, {
meth: 'Reconstruction', meth: 'Reconstruction',
req: { req: {
ctxRecon: refreshToken, ctxRecon: refreshToken,
@ -311,7 +302,7 @@ const createClient = (profile, userAgent, opt = {}) => {
}, opt) }, opt)
const f = profile.formatLocationFilter(opt.stops, opt.addresses, opt.poi) const f = profile.formatLocationFilter(opt.stops, opt.addresses, opt.poi)
return request({profile, opt}, userAgent, { return profile.request({profile, opt}, userAgent, {
cfg: {polyEnc: 'GPA'}, cfg: {polyEnc: 'GPA'},
meth: 'LocMatch', meth: 'LocMatch',
req: {input: { req: {input: {
@ -339,7 +330,7 @@ const createClient = (profile, userAgent, opt = {}) => {
opt = Object.assign({ opt = Object.assign({
linesOfStops: false // parse & expose lines at the stop/station? linesOfStops: false // parse & expose lines at the stop/station?
}, opt) }, opt)
return request({profile, opt}, userAgent, { return profile.request({profile, opt}, userAgent, {
meth: 'LocDetails', meth: 'LocDetails',
req: { req: {
locL: [stop] locL: [stop]
@ -367,7 +358,7 @@ const createClient = (profile, userAgent, opt = {}) => {
linesOfStops: false // parse & expose lines at each stop/station? linesOfStops: false // parse & expose lines at each stop/station?
}, opt) }, opt)
return request({profile, opt}, userAgent, { return profile.request({profile, opt}, userAgent, {
cfg: {polyEnc: 'GPA'}, cfg: {polyEnc: 'GPA'},
meth: 'LocGeoPos', meth: 'LocGeoPos',
req: { req: {
@ -405,7 +396,7 @@ const createClient = (profile, userAgent, opt = {}) => {
remarks: true // parse & expose hints & warnings? remarks: true // parse & expose hints & warnings?
}, opt) }, opt)
return request({profile, opt}, userAgent, { return profile.request({profile, opt}, userAgent, {
cfg: {polyEnc: 'GPA'}, cfg: {polyEnc: 'GPA'},
meth: 'JourneyDetails', meth: 'JourneyDetails',
req: { req: {
@ -454,7 +445,7 @@ const createClient = (profile, userAgent, opt = {}) => {
if (Number.isNaN(+opt.when)) throw new TypeError('opt.when is invalid') if (Number.isNaN(+opt.when)) throw new TypeError('opt.when is invalid')
const durationPerStep = opt.duration / Math.max(opt.frames, 1) * 1000 const durationPerStep = opt.duration / Math.max(opt.frames, 1) * 1000
return request({profile, opt}, userAgent, { return profile.request({profile, opt}, userAgent, {
meth: 'JourneyGeoPos', meth: 'JourneyGeoPos',
req: { req: {
maxJny: opt.results, maxJny: opt.results,
@ -492,7 +483,7 @@ const createClient = (profile, userAgent, opt = {}) => {
if (Number.isNaN(+opt.when)) throw new TypeError('opt.when is invalid') if (Number.isNaN(+opt.when)) throw new TypeError('opt.when is invalid')
const refetch = () => { const refetch = () => {
return request({profile, opt}, userAgent, { return profile.request({profile, opt}, userAgent, {
meth: 'LocGeoReach', meth: 'LocGeoReach',
req: { req: {
loc: profile.formatLocation(profile, address, 'address'), loc: profile.formatLocation(profile, address, 'address'),

View file

@ -1,5 +1,7 @@
'use strict' 'use strict'
const request = require('../lib/request')
const parseDateTime = require('../parse/date-time') const parseDateTime = require('../parse/date-time')
const parsePlatform = require('../parse/platform') const parsePlatform = require('../parse/platform')
const parseProductsBitmask = require('../parse/products-bitmask') const parseProductsBitmask = require('../parse/products-bitmask')
@ -34,13 +36,13 @@ const filters = require('../format/filters')
const id = (ctx, x) => x const id = (ctx, x) => x
const defaultProfile = { const defaultProfile = {
request,
transformReqBody: id,
transformReq: id,
salt: null, salt: null,
addChecksum: false, addChecksum: false,
addMicMac: false, addMicMac: false,
transformReqBody: id,
transformReq: id,
transformJourneysQuery: id, transformJourneysQuery: id,
parseDateTime, parseDateTime,

View file

@ -3,6 +3,8 @@
const types = { const types = {
locale: 'string', locale: 'string',
timezone: 'string', timezone: 'string',
request: 'function',
transformReq: 'function', transformReq: 'function',
transformReqBody: 'function', transformReqBody: 'function',
transformJourneysQuery: 'function', transformJourneysQuery: 'function',

View file

@ -2,42 +2,36 @@
const retry = require('p-retry') const retry = require('p-retry')
const _request = require('./lib/request')
const retryDefaults = { const retryDefaults = {
retries: 3, retries: 3,
factor: 3, factor: 3,
minTimeout: 5 * 1000 minTimeout: 5 * 1000
} }
const withRetrying = (createClient, retryOpts = {}) => { const withRetrying = (profile, retryOpts = {}) => {
retryOpts = Object.assign({}, retryDefaults, retryOpts) retryOpts = Object.assign({}, retryDefaults, retryOpts)
const {request} = profile
const createRetryingClient = (profile, userAgent, opt = {}) => { const retryingRequest = (...args) => {
const request = 'request' in opt ? opt.request : _request const attempt = () => {
return request(...args)
const retryingRequest = (profile, userAgent, opt, data) => { .catch((err) => {
const attempt = () => { if (err.isHafasError) throw err // continue
return request(profile, userAgent, opt, data) if (err.code === 'ENOTFOUND') { // abort
.catch((err) => { const abortErr = new retry.AbortError(err)
if (err.isHafasError) throw err // continue Object.assign(abortErr, err)
if (err.code === 'ENOTFOUND') { // abort throw abortErr
const abortErr = new retry.AbortError(err) }
Object.assign(abortErr, err) throw err // continue
throw abortErr })
}
throw err // continue
})
}
return retry(attempt, retryOpts)
} }
return retry(attempt, retryOpts)
return createClient(profile, userAgent, { }
...opt,
request: retryingRequest return {
}) ...profile,
request: retryingRequest
} }
return createRetryingClient
} }
module.exports = withRetrying module.exports = withRetrying

View file

@ -2,8 +2,8 @@
const test = require('tape') const test = require('tape')
const withRetrying = require('../retry')
const createClient = require('..') const createClient = require('..')
const withRetrying = require('../retry')
const vbbProfile = require('../p/vbb') const vbbProfile = require('../p/vbb')
const userAgent = 'public-transport/hafas-client:test' const userAgent = 'public-transport/hafas-client:test'
@ -12,7 +12,7 @@ const spichernstr = '900000042101'
test('withRetrying works', (t) => { test('withRetrying works', (t) => {
// for the first 3 calls, return different kinds of errors // for the first 3 calls, return different kinds of errors
let calls = 0 let calls = 0
const failingRequest = (profile, userAgent, opt, data) => { const failingRequest = async (ctx, userAgent, reqData) => {
calls++ calls++
if (calls === 1) { if (calls === 1) {
const err = new Error('HAFAS error') const err = new Error('HAFAS error')
@ -25,18 +25,22 @@ test('withRetrying works', (t) => {
return Promise.reject(err) return Promise.reject(err)
} }
if (calls < 4) return Promise.reject(new Error('generic error')) if (calls < 4) return Promise.reject(new Error('generic error'))
return Promise.resolve([]) return {
res: [],
common: {}
}
} }
const createRetryingClient = withRetrying(createClient, { const profile = withRetrying({
...vbbProfile,
request: failingRequest
}, {
retries: 3, retries: 3,
minTimeout: 100, minTimeout: 100,
factor: 2, factor: 2,
randomize: false randomize: false
}) })
const client = createRetryingClient(vbbProfile, userAgent, { const client = createClient(profile, userAgent)
request: failingRequest
})
t.plan(1 + 4) t.plan(1 + 4)
client.departures(spichernstr, {duration: 1}) client.departures(spichernstr, {duration: 1})

View file

@ -3,8 +3,8 @@
const tapePromise = require('tape-promise').default const tapePromise = require('tape-promise').default
const tape = require('tape') const tape = require('tape')
const withThrottling = require('../throttle')
const createClient = require('..') const createClient = require('..')
const withThrottling = require('../throttle')
const vbbProfile = require('../p/vbb') const vbbProfile = require('../p/vbb')
const depsRes = require('./fixtures/vbb-departures.json') const depsRes = require('./fixtures/vbb-departures.json')
@ -14,10 +14,8 @@ const spichernstr = '900000042101'
const test = tapePromise(tape) const test = tapePromise(tape)
test('withThrottling works', async (t) => { test('withThrottling works', async (t) => {
const ctx = {profile: vbbProfile, opt: {}}
let calls = 0 let calls = 0
const mockedRequest = async (ctx, _, reqData) => { const mockedRequest = async (ctx, userAgent, reqData) => {
calls++ calls++
return { return {
res: depsRes, res: depsRes,
@ -25,8 +23,11 @@ test('withThrottling works', async (t) => {
} }
} }
const createThrottledClient = withThrottling(createClient, 2, 1000) const profile = withThrottling({
const client = createThrottledClient(vbbProfile, ua, mockedRequest) ...vbbProfile,
request: mockedRequest
}, 2, 1000)
const client = createClient(profile, ua)
t.plan(3) t.plan(3)
for (let i = 0; i < 10; i++) { for (let i = 0; i < 10; i++) {

View file

@ -2,20 +2,13 @@
const throttle = require('p-throttle') const throttle = require('p-throttle')
const _request = require('./lib/request') const withThrottling = (profile, limit = 5, interval = 1000) => {
const {request} = profile
const withThrottling = (createClient, limit = 5, interval = 1000) => { return {
const createThrottledClient = (profile, userAgent, opt = {}) => { ...profile,
const request = 'request' in opt ? opt.request : _request request: throttle(request, limit, interval)
const throttledRequest = throttle(request, limit, interval)
return createClient(profile, userAgent, {
...opt,
request: throttledRequest
})
} }
return createThrottledClient
} }
module.exports = withThrottling module.exports = withThrottling