From 0f95b5777e5997ae8080e77dd3814b160fae665c Mon Sep 17 00:00:00 2001 From: Johannes Loher Date: Tue, 11 Apr 2017 18:57:10 +0200 Subject: [PATCH 1/3] switched api to http://mobile.defas-fgi.de/beg/ and added tests for the corresponding functions --- source/app.d | 40 +++-- source/fahrplanparser.d | 314 +++++++++++++++++++++++----------------- 2 files changed, 210 insertions(+), 144 deletions(-) diff --git a/source/app.d b/source/app.d index 83ec0a1..ba5a3ed 100644 --- a/source/app.d +++ b/source/app.d @@ -6,22 +6,29 @@ import std.getopt : defaultGetoptPrinter, getopt; import std.json : JSONValue; import std.stdio : File, writeln; -import requests : postContent; +import requests : getContent; import fahrplanparser; import substitution; +// find stops: http://mobile.defas-fgi.de/beg/XML_STOPFINDER_REQUEST?outputFormat=XML&stateless=1&locationServerActive=1®ionID_sf=1&type_sf=stop&name_sf=Regensburg%20Universit%C3%A4t + +enum baseURL = "http://mobile.defas-fgi.de/beg/"; +enum departureMonitorRequest = "XML_DM_REQUEST"; +enum stopFinderRequest = "XML_STOPFINDER_REQUEST"; + void main(string[] args) { string fileName; string busStop = "4014080"; string substitutionFileName = "replacement.txt"; - + // dfmt off auto helpInformation = getopt(args, - "file|f", "The file that the data is written to.", &fileName, - "stop|s", "The bus stop for which to fetch data.", &busStop, - "replacement-file|r", "The file that contais the direction name replacement info.", &substitutionFileName); + "file|f", "The file that the data is written to.", &fileName, + "stop|s", "The bus stop for which to fetch data.", &busStop, + "replacement-file|r", "The file that contais the direction name replacement info.", &substitutionFileName); + // dfmt on if (helpInformation.helpWanted) { @@ -29,13 +36,20 @@ void main(string[] args) return; } - auto content = postContent("http://txt.bayern-fahrplan.de/textversion/bcl_abfahrtstafel", - ["limit" : "20", - "useRealtime" : "1", - "name_dm" : busStop, - "mode" : "direct", - "type_dm" : "any", - "itdLPxx_bcl" : "true"]); + // dfmt off + auto content = getContent(baseURL ~ departureMonitorRequest, + ["outputFormat" : "XML", + "language" : "de", + "stateless" : "1", + "type_dm" : "stop", + "name_dm" : busStop, + "useRealtime" : "1", + "mode" : "direct", + "ptOptionActive" : "1", + "mergeDep" : "1", + "limit" : "20", + "deleteAssignedStops_dm" : "1"]); + // dfmt on if (substitutionFileName.exists && substitutionFileName.isFile) { @@ -44,6 +58,7 @@ void main(string[] args) auto currentTime = Clock.currTime; JSONValue j = ["time" : "%02s:%02s".format(currentTime.hour, currentTime.minute)]; + j.object["departures"] = (cast(string) content.data).parsedFahrplan.array.JSONValue; auto output = j.toPrettyString.replace("\\/", "/"); if (fileName !is null) @@ -57,5 +72,4 @@ void main(string[] args) { output.writeln; } - } diff --git a/source/fahrplanparser.d b/source/fahrplanparser.d index 7b0e9d3..e92cb34 100644 --- a/source/fahrplanparser.d +++ b/source/fahrplanparser.d @@ -1,12 +1,10 @@ module fahrplanparser; -import std.algorithm : filter, map; -import std.array : empty, front, replace; +import std.algorithm : map; +import std.array : front; import std.conv : to; -import std.datetime : dur, TimeOfDay; -import std.regex : ctRegex, matchAll; -import std.string : strip; -import std.typecons : tuple, Tuple; +import std.datetime : dur, TimeOfDay, DateTimeException; +import std.string : format; import kxml.xml : readDocument, XmlNode; @@ -14,14 +12,18 @@ import substitution; private: -enum ScheduleHeadings +enum departureNodeName = "dp"; +enum timeNodeName = "t"; +enum realTimeNodeName = "rt"; + +enum departuresXPath = "/efa/dps/" ~ departureNodeName; +template timeXPath(string _timeNodeName = timeNodeName) { - date, - departure, - line, - direction, - platform + enum timeXPath = "/st/" ~ _timeNodeName; } +enum useRealTimeXPath = "/realtime"; +enum lineXPath = "/m/nu"; +enum directionXPath = "/m/des"; public: @@ -29,132 +31,182 @@ auto parsedFahrplan(in string data) { // dfmt off return data.readDocument - .parseXPath(`//table[@id="departureMonitor"]/tbody/tr`)[1 .. $] - .getRowContents - .filter!(row => !row.empty) - .map!(a => ["departure" : a[0].parseTime[0].to!string[0 .. $ - 3], - "delay" : a[0].parseTime[1].total!"minutes".to!string, - "line" : a[1], - "direction" : a[2].substitute]); + .parseXPath(departuresXPath) + .map!(dp => ["departure" : "%02s:%02s".format(dp.departureTime.hour, dp.departureTime.minute), + "delay" : dp.delay.total!"minutes".to!string, + "line": dp.parseXPath(lineXPath).front.getCData, + "direction": dp.parseXPath(directionXPath).front.getCData.substitute]); // dfmt on } -private: - -class BadTimeInputException : Exception -{ - this(string msg) @safe pure nothrow @nogc - { - super(msg); - } - - this() @safe pure nothrow @nogc - { - this(""); - } -} - -auto parseTime(in string input) @safe -{ - auto matches = matchAll(input, ctRegex!(`(?P\d{1,2}):(?P\d{2})`)); - if (matches.empty) - throw new BadTimeInputException(); - auto actualTime = TimeOfDay(matches.front["hours"].to!int, matches.front["minutes"].to!int); - matches.popFront; - if (!matches.empty) - { - auto expectedTime = TimeOfDay(matches.front["hours"].to!int, - matches.front["minutes"].to!int); - auto timeDiff = actualTime - expectedTime; - - if (timeDiff < dur!"minutes"(0)) - timeDiff = dur!"hours"(24) + timeDiff; - - return tuple(expectedTime, timeDiff); - } - return tuple(actualTime, dur!"minutes"(0)); -} - -@safe unittest -{ - import std.exception : assertThrown; - - assertThrown(parseTime("")); - assertThrown(parseTime("lkeqf")); - assertThrown(parseTime(":")); - assertThrown(parseTime("00:0")); - - assert("00:00".parseTime == tuple(TimeOfDay(0, 0), dur!"minutes"(0))); - assert("0:00".parseTime == tuple(TimeOfDay(0, 0), dur!"minutes"(0))); - - assert("00:00 00:00".parseTime == tuple(TimeOfDay(0, 0), dur!"minutes"(0))); - - assert("00:00 00:00 12:00".parseTime == tuple(TimeOfDay(0, 0), dur!"minutes"(0))); - - assert("12:3412:34".parseTime == tuple(TimeOfDay(12, 34), dur!"minutes"(0))); - - assert("ölqjfo12:34oieqf12:31ölqjf".parseTime == tuple(TimeOfDay(12, 31), dur!"minutes"(3))); - - assert("17:53 (planmäßig 17:51 Uhr)".parseTime == tuple(TimeOfDay(17, 51), dur!"minutes"(2))); - - assert("00:00 23:59".parseTime == tuple(TimeOfDay(23, 59), dur!"minutes"(1))); -} - -auto getRowContents(XmlNode[] rows) -{ - return rows.map!(x => getRowContent(x)); -} - -auto getRowContent(XmlNode row) -{ - return row.parseXPath("//td")[ScheduleHeadings.departure .. ScheduleHeadings.direction + 1].map!( - cell => stripLinks(cell)); -} - -auto stripLinks(XmlNode cell) -{ - auto links = cell.parseXPath("//a"); - if (links.empty) - { - return cell.getCData; - } - else - { - return links.front.getCData.replace("...", ""); - } -} - @system unittest { - auto foo = new XmlNode("foo"); - assert(foo.stripLinks == ""); + import std.stdio; + import std.array: array; + auto xml = ""; + assert(xml.parsedFahrplan.array == []); - auto link = new XmlNode("a"); - link.setCData("test"); - foo.addChild(link); - assert(foo.stripLinks == "test"); + xml = ""; + assert(xml.parsedFahrplan.array == []); - link.setCData("test2..."); - assert(foo.stripLinks == "test2"); + xml = "1122412426Wernerwerkstraße"; + assert(xml.parsedFahrplan.array == [["direction":"Wernerwerkstraße", "line":"6", "departure":"12:24", "delay":"18"]]); - auto bar = new XmlNode("bar"); - bar.setCData("test3"); - assert(bar.stripLinks == "test3"); + xml = "012246Wernerwerkstraße"; + assert(xml.parsedFahrplan.array == [["direction":"Wernerwerkstraße", "line":"6", "departure":"12:24", "delay":"0"]]); - bar.addChild(link); - assert(bar.stripLinks == "test2"); - - auto baz = new XmlNode("baz"); - auto subNode = new XmlNode("subNode"); - baz.addChild(subNode); - assert(baz.stripLinks == ""); - - baz.addChild(link); - assert(baz.stripLinks == "test2"); - - baz.addCData("test4"); - assert(baz.stripLinks == "test2"); - - baz.removeChild(link); - assert(baz.stripLinks == "test4"); + xml = "012246Wernerwerkstraße11353135611Burgweinting"; + assert(xml.parsedFahrplan.array == [["direction":"Wernerwerkstraße", "line":"6", "departure":"12:24", "delay":"0"], ["direction":"Burgweinting", "line":"11", "departure":"13:53", "delay":"3"]]); +} + +private: + +class UnexpectedValueException(T) : Exception +{ + this(T t, string node) @safe pure + { + super(`Unexpected value "%s" for node "%s"`.format(t, node)); + } +} + +auto departureTime(string _timeNodeName = timeNodeName)(XmlNode dp) +in +{ + assert(dp.getName == departureNodeName); +} +body +{ + return TimeOfDay.fromISOString(dp.parseXPath(timeXPath!_timeNodeName).front.getCData ~ "00"); +} + +@system unittest +{ + import std.exception : assertThrown; + + auto xml = "0000".readDocument.parseXPath("/dp").front; + assert(xml.departureTime == TimeOfDay(0, 0)); + + xml = "0013".readDocument.parseXPath("/dp").front; + assert(xml.departureTime == TimeOfDay(0, 13)); + + xml = "1100".readDocument.parseXPath("/dp").front; + assert(xml.departureTime == TimeOfDay(11, 00)); + + xml = "1242".readDocument.parseXPath("/dp").front; + assert(xml.departureTime == TimeOfDay(12, 42)); + + xml = "2359".readDocument.parseXPath("/dp").front; + assert(xml.departureTime == TimeOfDay(23, 59)); + + assertThrown!DateTimeException("2400".readDocument.parseXPath("/dp") + .front.departureTime); + assertThrown!DateTimeException("0061".readDocument.parseXPath("/dp") + .front.departureTime); + assertThrown!DateTimeException("2567".readDocument.parseXPath("/dp") + .front.departureTime); + assertThrown!DateTimeException("".readDocument.parseXPath("/dp") + .front.departureTime); + assertThrown!DateTimeException("0".readDocument.parseXPath("/dp") + .front.departureTime); + assertThrown!DateTimeException("00".readDocument.parseXPath("/dp") + .front.departureTime); + assertThrown!DateTimeException("000000".readDocument.parseXPath("/dp") + .front.departureTime); + assertThrown!DateTimeException("00:00".readDocument.parseXPath("/dp") + .front.departureTime); + assertThrown!DateTimeException("abcd".readDocument.parseXPath("/dp") + .front.departureTime); +} + +auto delay(XmlNode dp) +in +{ + assert(dp.getName == departureNodeName); +} +body +{ + auto useRealtimeString = dp.parseXPath(useRealTimeXPath).front.getCData; + if (useRealtimeString == "0") + return dur!"minutes"(0); + else if (useRealtimeString == "1") + { + auto expectedTime = dp.departureTime; + auto realTime = dp.departureTime!realTimeNodeName; + auto timeDiff = realTime - expectedTime; + if (timeDiff < dur!"minutes"(0)) + timeDiff = dur!"hours"(24) + timeDiff; + return timeDiff; + } + else + throw new UnexpectedValueException!string(useRealtimeString, "realtime"); +} + +@system unittest +{ + import std.exception : assertThrown; + import core.exception : AssertError; + + auto xml = "0".readDocument.parseXPath("/dp").front; + assert(xml.delay == dur!"minutes"(0)); + + xml = "".readDocument.parseXPath("/dp").front; + assertThrown!(UnexpectedValueException!string)(xml.delay); + + xml = "2".readDocument.parseXPath("/dp").front; + assertThrown!(UnexpectedValueException!string)(xml.delay); + + xml = "a".readDocument.parseXPath("/dp").front; + assertThrown!(UnexpectedValueException!string)(xml.delay); + + xml = "1".readDocument.parseXPath("/dp").front; + assertThrown!AssertError(xml.delay); + + xml = "1".readDocument.parseXPath("/dp").front; + assertThrown!DateTimeException(xml.delay); + + xml = "1".readDocument.parseXPath("/dp").front; + assertThrown!AssertError(xml.delay); + + xml = "".readDocument.parseXPath("/dp").front; + assertThrown!AssertError(xml.delay); + + xml = "1".readDocument.parseXPath("/dp") + .front; + assertThrown!DateTimeException(xml.delay); + + xml = "10000".readDocument.parseXPath("/dp") + .front; + assertThrown!DateTimeException(xml.delay); + + xml = "10000".readDocument.parseXPath("/dp") + .front; + assertThrown!DateTimeException(xml.delay); + + xml = "100000000" + .readDocument.parseXPath("/dp").front; + assert(xml.delay == dur!"minutes"(0)); + + xml = "100010000" + .readDocument.parseXPath("/dp").front; + assert(xml.delay == dur!"minutes"(1)); + + xml = "117531751" + .readDocument.parseXPath("/dp").front; + assert(xml.delay == dur!"minutes"(2)); + + xml = "110101000" + .readDocument.parseXPath("/dp").front; + assert(xml.delay == dur!"minutes"(10)); + + xml = "113011242" + .readDocument.parseXPath("/dp").front; + assert(xml.delay == dur!"minutes"(19)); + + xml = "100001242" + .readDocument.parseXPath("/dp").front; + assert(xml.delay == dur!"minutes"(678)); + + xml = "100002359" + .readDocument.parseXPath("/dp").front; + assert(xml.delay == dur!"minutes"(1)); } From a6aeaa336e503eedddbddb77aceabcdac253111d Mon Sep 17 00:00:00 2001 From: Johannes Loher Date: Mon, 8 May 2017 22:38:31 +0200 Subject: [PATCH 2/3] use name of bus stop instead of id --- source/app.d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/app.d b/source/app.d index ba5a3ed..a407ff6 100644 --- a/source/app.d +++ b/source/app.d @@ -21,7 +21,7 @@ enum stopFinderRequest = "XML_STOPFINDER_REQUEST"; void main(string[] args) { string fileName; - string busStop = "4014080"; + string busStop = "Regensburg Universität"; string substitutionFileName = "replacement.txt"; // dfmt off auto helpInformation = getopt(args, From c7cf7fedfd321f4866040e910d6ae857739ac18a Mon Sep 17 00:00:00 2001 From: Johannes Loher Date: Sun, 14 May 2017 11:29:19 +0200 Subject: [PATCH 3/3] cleanup --- dub.json | 2 +- source/app.d | 3 --- source/fahrplanparser.d | 21 +++++++++++++-------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/dub.json b/dub.json index d5076dd..23a066c 100644 --- a/dub.json +++ b/dub.json @@ -5,7 +5,7 @@ "Oliver Rümpelein" ], "dependencies": { - "requests": "~>0.3.1", + "requests": "~>0.4.1", "kxml": "~>1.0.1" }, "description": "A minimal D application.", diff --git a/source/app.d b/source/app.d index a407ff6..ce064ab 100644 --- a/source/app.d +++ b/source/app.d @@ -12,11 +12,8 @@ import fahrplanparser; import substitution; -// find stops: http://mobile.defas-fgi.de/beg/XML_STOPFINDER_REQUEST?outputFormat=XML&stateless=1&locationServerActive=1®ionID_sf=1&type_sf=stop&name_sf=Regensburg%20Universit%C3%A4t - enum baseURL = "http://mobile.defas-fgi.de/beg/"; enum departureMonitorRequest = "XML_DM_REQUEST"; -enum stopFinderRequest = "XML_STOPFINDER_REQUEST"; void main(string[] args) { diff --git a/source/fahrplanparser.d b/source/fahrplanparser.d index e92cb34..7860e86 100644 --- a/source/fahrplanparser.d +++ b/source/fahrplanparser.d @@ -21,6 +21,7 @@ template timeXPath(string _timeNodeName = timeNodeName) { enum timeXPath = "/st/" ~ _timeNodeName; } + enum useRealTimeXPath = "/realtime"; enum lineXPath = "/m/nu"; enum directionXPath = "/m/des"; @@ -41,8 +42,8 @@ auto parsedFahrplan(in string data) @system unittest { - import std.stdio; - import std.array: array; + import std.array : array; + auto xml = ""; assert(xml.parsedFahrplan.array == []); @@ -50,13 +51,17 @@ auto parsedFahrplan(in string data) assert(xml.parsedFahrplan.array == []); xml = "1122412426Wernerwerkstraße"; - assert(xml.parsedFahrplan.array == [["direction":"Wernerwerkstraße", "line":"6", "departure":"12:24", "delay":"18"]]); + assert(xml.parsedFahrplan.array == [["direction" : "Wernerwerkstraße", + "line" : "6", "departure" : "12:24", "delay" : "18"]]); xml = "012246Wernerwerkstraße"; - assert(xml.parsedFahrplan.array == [["direction":"Wernerwerkstraße", "line":"6", "departure":"12:24", "delay":"0"]]); + assert(xml.parsedFahrplan.array == [["direction" : "Wernerwerkstraße", + "line" : "6", "departure" : "12:24", "delay" : "0"]]); xml = "012246Wernerwerkstraße11353135611Burgweinting"; - assert(xml.parsedFahrplan.array == [["direction":"Wernerwerkstraße", "line":"6", "departure":"12:24", "delay":"0"], ["direction":"Burgweinting", "line":"11", "departure":"13:53", "delay":"3"]]); + assert(xml.parsedFahrplan.array == [["direction" : "Wernerwerkstraße", "line" : "6", + "departure" : "12:24", "delay" : "0"], ["direction" : "Burgweinting", + "line" : "11", "departure" : "13:53", "delay" : "3"]]); } private: @@ -125,13 +130,13 @@ in } body { - auto useRealtimeString = dp.parseXPath(useRealTimeXPath).front.getCData; + immutable useRealtimeString = dp.parseXPath(useRealTimeXPath).front.getCData; if (useRealtimeString == "0") return dur!"minutes"(0); else if (useRealtimeString == "1") { - auto expectedTime = dp.departureTime; - auto realTime = dp.departureTime!realTimeNodeName; + immutable expectedTime = dp.departureTime; + immutable realTime = dp.departureTime!realTimeNodeName; auto timeDiff = realTime - expectedTime; if (timeDiff < dur!"minutes"(0)) timeDiff = dur!"hours"(24) + timeDiff;