From 027df3f7ba2d9f1ba4692af66b2d553cb22b4bd1 Mon Sep 17 00:00:00 2001 From: Johannes Loher Date: Thu, 29 Dec 2016 13:14:48 +0100 Subject: [PATCH 01/21] removed unneccesary code --- source/substitution.d | 6 ------ 1 file changed, 6 deletions(-) diff --git a/source/substitution.d b/source/substitution.d index 0937dc2..e10bf65 100644 --- a/source/substitution.d +++ b/source/substitution.d @@ -5,7 +5,6 @@ public: void loadSubstitutionFile(string fileName) { import std.file : slurp, exists, isFile; - import std.array : assocArray; import std.algorithm.iteration : each; if (fileName.exists && fileName.isFile) @@ -13,11 +12,6 @@ void loadSubstitutionFile(string fileName) auto data = slurp!(string, string)(fileName, `"%s" = "%s"`); map = (string[string]).init; data.each!(pair => map[pair[0]] = pair[1]); - foreach (pair; data) - { - map[pair[0]] = pair[1]; - } - map.rehash; } else { From 11286050ca6d97766853f52bf553e45e4f4b8d53 Mon Sep 17 00:00:00 2001 From: Johannes Loher Date: Thu, 29 Dec 2016 13:19:56 +0100 Subject: [PATCH 02/21] Added a simple readme --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..8f0e99a --- /dev/null +++ b/README.md @@ -0,0 +1,8 @@ +bayernfahrplan +============== + +A JSON-Getter written in D for bayernfahrplan +--------------------------------------------- + +bayernfahrplan is a tool wich lets you fetch departure tables from +bayernfahrplan.de and output them as JSON. \ No newline at end of file From 7f78ea64c93b0c0bbb2f7582089129fefe9b8ac5 Mon Sep 17 00:00:00 2001 From: Johannes Loher Date: Thu, 29 Dec 2016 13:33:41 +0100 Subject: [PATCH 03/21] Added unitests for substitution.substiute --- source/substitution.d | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/source/substitution.d b/source/substitution.d index e10bf65..d743fe9 100644 --- a/source/substitution.d +++ b/source/substitution.d @@ -19,11 +19,30 @@ void loadSubstitutionFile(string fileName) } } -auto substitute(string s) +auto substitute(string s) @safe nothrow { return s in map ? map[s] : s; } +@safe unittest +{ + map[""] = ""; + assert(substitute("") == ""); + + map["a"] = "b"; + assert(substitute("a") == "b"); + + map["Regensburg Danziger Freiheit"] = "Danziger Freiheit"; + assert(substitute("Regensburg Danziger Freiheit") == "Danziger Freiheit"); + + map["Regensburg Danziger Freiheit"] = "Anderer Test"; + assert(substitute("Regensburg Danziger Freiheit") == "Anderer Test"); + + assert(substitute("z") == "z"); + + assert(substitute("Regensburg Hauptbahnhof") == "Regensburg Hauptbahnhof"); +} + private: string[string] map; From 1cc8840127885393625c2d48d02219f6fe9a1862 Mon Sep 17 00:00:00 2001 From: Johannes Loher Date: Thu, 29 Dec 2016 18:20:08 +0100 Subject: [PATCH 04/21] restructred the program a bit and added unittests for fahrplanparser.parseTime. Also fixed some stuff I fould while doing that --- source/app.d | 57 +++-------------------- source/fahrplanparser.d | 100 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 50 deletions(-) create mode 100644 source/fahrplanparser.d diff --git a/source/app.d b/source/app.d index e3b76eb..4865685 100644 --- a/source/app.d +++ b/source/app.d @@ -1,45 +1,12 @@ -import std.algorithm : filter, map, startsWith; -import std.array : array, empty, front, replace; -import std.conv : to; -import std.datetime : dur, TimeOfDay, Clock; import std.getopt : defaultGetoptPrinter, getopt; -import std.json : JSONValue; -import std.regex : ctRegex, matchAll; import std.stdio : File, stdout, writeln; -import std.string : strip; -import std.typecons : tuple; -import std.format : format; - -import kxml.xml : readDocument, XmlNode; import requests : postContent; +import fahrplanparser; + import substitution; -auto parseTime(in string input) -{ - auto matches = matchAll(input, ctRegex!(`(?P\d+):(?P\d+)`)); - auto actualTime = TimeOfDay(matches.front["hours"].to!int, matches.front["minutes"].to!int); - matches.popFront; - if (!matches.empty) - { - auto expectedTime = TimeOfDay(matches.front["hours"].to!int, - matches.front["minutes"].to!int); - return tuple(expectedTime, actualTime - expectedTime); - - } - return tuple(actualTime, dur!"minutes"(0)); -} - -auto getRowContents(XmlNode[] rows) -{ - return rows.map!(row => row.parseXPath("//td")[1 .. $ - 1].map!((column) { - auto link = column.parseXPath("//a"); - if (!link.empty) - return link.front.getCData.replace("...", ""); - return column.getCData;})); -} - void main(string[] args) { string fileName; @@ -63,28 +30,18 @@ void main(string[] args) "type_dm" : "any", "itdLPxx_bcl" : "true"]); - auto currentTime = Clock.currTime; loadSubstitutionFile(substitutionFileName); - JSONValue j = ["time" : "%02s:%02s".format(currentTime.hour, currentTime.minute)]; - j.object["departures"] = readDocument(cast(string) content.data) - .parseXPath(`//table[@id="departureMonitor"]/tbody/tr`)[1 .. $] - .getRowContents - .filter!(row => !row.empty) - .map!(a => ["departure" : a[0].parseTime[0].to!string[0 .. $ - 3], - "delay" : a[0].parseTime[1].total!"minutes".to!string, - "line" : a[1], - "direction" : a[2].substitute]) - .array.JSONValue; + auto output = (cast(string) content.data).parseFahrplan; if (fileName !is null) { - auto output = File(fileName, "w"); - scope(exit) output.close; - output.writeln(j.toPrettyString.replace("\\/", "/")); + auto outfile = File(fileName, "w"); + scope(exit) outfile.close; + outfile.writeln(output); } else { - j.toPrettyString.replace("\\/", "/").writeln; + output.writeln; } } diff --git a/source/fahrplanparser.d b/source/fahrplanparser.d new file mode 100644 index 0000000..258e39f --- /dev/null +++ b/source/fahrplanparser.d @@ -0,0 +1,100 @@ +module fahrplanparser; + +import std.algorithm : filter, map, startsWith; +import std.array : array, empty, front, replace; +import std.conv : to; +import std.datetime : dur, TimeOfDay, Clock; +import std.format : format; +import std.json : JSONValue; +import std.regex : ctRegex, matchAll; +import std.string : strip; +import std.typecons : tuple; + +import kxml.xml : readDocument, XmlNode; + +import substitution; + +public: + +auto parseFahrplan(in string data) +{ + auto currentTime = Clock.currTime; + JSONValue j = ["time" : "%02s:%02s".format(currentTime.hour, currentTime.minute)]; + j.object["departures"] = data.readDocument + .parseXPath(`//table[@id="departureMonitor"]/tbody/tr`)[1 .. $] + .getRowContents + .filter!(row => !row.empty) + .map!(a => ["departure" : a[0].parseTime[0].to!string[0 .. $ - 3], + "delay" : a[0].parseTime[1].total!"minutes".to!string, + "line" : a[1], + "direction" : a[2].substitute]) + .array.JSONValue; + return j.toPrettyString.replace("\\/", "/"); +} + +private: + +class BadTimeInputException : Exception +{ + this(string msg) @safe pure nothrow @nogc + { + super(msg); + } + + this() @safe pure nothrow @nogc + { + this(""); + } +} + +auto parseTime(in string input) @safe +{ + auto matches = matchAll(input, ctRegex!(`(?P\d{1,2}):(?P\d{2})`)); + if (matches.empty) + throw new BadTimeInputException(); + auto actualTime = TimeOfDay(matches.front["hours"].to!int, matches.front["minutes"].to!int); + matches.popFront; + if (!matches.empty) + { + auto expectedTime = TimeOfDay(matches.front["hours"].to!int, + matches.front["minutes"].to!int); + auto timeDiff = actualTime - expectedTime; + + if(timeDiff < dur!"minutes"(0)) + timeDiff = dur!"hours"(24) + timeDiff; + + return tuple(expectedTime, timeDiff); + } + return tuple(actualTime, dur!"minutes"(0)); +} + +@safe unittest +{ + import std.exception : assertThrown; + assertThrown(parseTime("")); + assertThrown(parseTime("lkeqf")); + assertThrown(parseTime(":")); + assertThrown(parseTime("00:0")); + + assert("00:00".parseTime == tuple(TimeOfDay(0, 0), dur!"minutes"(0))); + assert("0:00".parseTime == tuple(TimeOfDay(0, 0), dur!"minutes"(0))); + + assert("00:00 00:00".parseTime == tuple(TimeOfDay(0, 0), dur!"minutes"(0))); + + assert("12:3412:34".parseTime == tuple(TimeOfDay(12, 34), dur!"minutes"(0))); + + assert("ölqjfo12:34oieqf12:31ölqjf".parseTime == tuple(TimeOfDay(12, 31), dur!"minutes"(3))); + + assert("17:53 (planmäßig 17:51 Uhr)".parseTime == tuple(TimeOfDay(17, 51), dur!"minutes"(2))); + + assert("00:00 23:59".parseTime == tuple(TimeOfDay(23,59), dur!"minutes"(1))); +} + +auto getRowContents(XmlNode[] rows) +{ + return rows.map!(row => row.parseXPath("//td")[1 .. $ - 1].map!((column) { + auto link = column.parseXPath("//a"); + if (!link.empty) + return link.front.getCData.replace("...", ""); + return column.getCData;})); +} From 960b4d3f6fd3bb52046957cecd0e193a481486b5 Mon Sep 17 00:00:00 2001 From: Johannes Loher Date: Thu, 29 Dec 2016 18:27:30 +0100 Subject: [PATCH 05/21] Added another test for fahrplanparser.parseTime --- source/fahrplanparser.d | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/fahrplanparser.d b/source/fahrplanparser.d index 258e39f..b546f52 100644 --- a/source/fahrplanparser.d +++ b/source/fahrplanparser.d @@ -81,6 +81,8 @@ auto parseTime(in string input) @safe assert("00:00 00:00".parseTime == tuple(TimeOfDay(0, 0), dur!"minutes"(0))); + assert("00:00 00:00 12:00".parseTime == tuple(TimeOfDay(0, 0), dur!"minutes"(0))); + assert("12:3412:34".parseTime == tuple(TimeOfDay(12, 34), dur!"minutes"(0))); assert("ölqjfo12:34oieqf12:31ölqjf".parseTime == tuple(TimeOfDay(12, 31), dur!"minutes"(3))); From caab61ddb8827c6dc13de6e9b184129709b3adb0 Mon Sep 17 00:00:00 2001 From: Johannes Loher Date: Thu, 29 Dec 2016 18:39:10 +0100 Subject: [PATCH 06/21] Restructured a bit again --- source/app.d | 10 +++++++++- source/fahrplanparser.d | 16 ++++++---------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/source/app.d b/source/app.d index 4865685..089c145 100644 --- a/source/app.d +++ b/source/app.d @@ -1,5 +1,10 @@ import std.getopt : defaultGetoptPrinter, getopt; import std.stdio : File, stdout, writeln; +import std.datetime : Clock; +import std.json : JSONValue; +import std.format : format; +import std.array: array, replace; + import requests : postContent; @@ -32,7 +37,10 @@ void main(string[] args) loadSubstitutionFile(substitutionFileName); - auto output = (cast(string) content.data).parseFahrplan; + auto currentTime = Clock.currTime; + JSONValue j = ["time" : "%02s:%02s".format(currentTime.hour, currentTime.minute)]; + j.object["departures"] = (cast(string) content.data).parsedFahrplan.array.JSONValue; + auto output = j.toPrettyString.replace("\\/", "/"); if (fileName !is null) { auto outfile = File(fileName, "w"); diff --git a/source/fahrplanparser.d b/source/fahrplanparser.d index b546f52..caae9bc 100644 --- a/source/fahrplanparser.d +++ b/source/fahrplanparser.d @@ -1,11 +1,11 @@ module fahrplanparser; +private: + import std.algorithm : filter, map, startsWith; -import std.array : array, empty, front, replace; +import std.array : empty, front, replace; import std.conv : to; import std.datetime : dur, TimeOfDay, Clock; -import std.format : format; -import std.json : JSONValue; import std.regex : ctRegex, matchAll; import std.string : strip; import std.typecons : tuple; @@ -16,20 +16,16 @@ import substitution; public: -auto parseFahrplan(in string data) +auto parsedFahrplan(in string data) { - auto currentTime = Clock.currTime; - JSONValue j = ["time" : "%02s:%02s".format(currentTime.hour, currentTime.minute)]; - j.object["departures"] = data.readDocument + return data.readDocument .parseXPath(`//table[@id="departureMonitor"]/tbody/tr`)[1 .. $] .getRowContents .filter!(row => !row.empty) .map!(a => ["departure" : a[0].parseTime[0].to!string[0 .. $ - 3], "delay" : a[0].parseTime[1].total!"minutes".to!string, "line" : a[1], - "direction" : a[2].substitute]) - .array.JSONValue; - return j.toPrettyString.replace("\\/", "/"); + "direction" : a[2].substitute]); } private: From f7b67f6d09b20e1a28e2da8dbd893330684ee00a Mon Sep 17 00:00:00 2001 From: Johannes Loher Date: Thu, 29 Dec 2016 18:43:54 +0100 Subject: [PATCH 07/21] Some small cleanup --- source/app.d | 11 +++++------ source/fahrplanparser.d | 6 ++---- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/source/app.d b/source/app.d index 089c145..58b28b5 100644 --- a/source/app.d +++ b/source/app.d @@ -1,10 +1,9 @@ -import std.getopt : defaultGetoptPrinter, getopt; -import std.stdio : File, stdout, writeln; -import std.datetime : Clock; -import std.json : JSONValue; -import std.format : format; import std.array: array, replace; - +import std.datetime : Clock; +import std.format : format; +import std.getopt : defaultGetoptPrinter, getopt; +import std.json : JSONValue; +import std.stdio : File, writeln; import requests : postContent; diff --git a/source/fahrplanparser.d b/source/fahrplanparser.d index caae9bc..6b64bac 100644 --- a/source/fahrplanparser.d +++ b/source/fahrplanparser.d @@ -1,11 +1,9 @@ module fahrplanparser; -private: - -import std.algorithm : filter, map, startsWith; +import std.algorithm : filter, map; import std.array : empty, front, replace; import std.conv : to; -import std.datetime : dur, TimeOfDay, Clock; +import std.datetime : dur, TimeOfDay; import std.regex : ctRegex, matchAll; import std.string : strip; import std.typecons : tuple; From 8c8d587c5429d6ad4a3f8d75b9a515842bcfd567 Mon Sep 17 00:00:00 2001 From: Johannes Loher Date: Wed, 11 Jan 2017 14:42:04 +0100 Subject: [PATCH 08/21] restructured fahrplanparser.getRowContents and added some tests --- source/app.d | 7 ++- source/fahrplanparser.d | 94 ++++++++++++++++++++++++++++++++++------- 2 files changed, 84 insertions(+), 17 deletions(-) diff --git a/source/app.d b/source/app.d index 58b28b5..32a0650 100644 --- a/source/app.d +++ b/source/app.d @@ -1,4 +1,4 @@ -import std.array: array, replace; +import std.array : array, replace; import std.datetime : Clock; import std.format : format; import std.getopt : defaultGetoptPrinter, getopt; @@ -16,10 +16,12 @@ void main(string[] args) string fileName; string busStop = "Universität Regensburg"; string substitutionFileName = "replacement.txt"; + auto helpInformation = getopt(args, "file|f", "The file that the data is written to.", &fileName, "stop|s", "The bus stop for which to fetch data.", &busStop, "replacement-file|r", "The file that contais the direction name replacement info.", &substitutionFileName); + if (helpInformation.helpWanted) { defaultGetoptPrinter("Some information about the program.", helpInformation.options); @@ -43,7 +45,8 @@ void main(string[] args) if (fileName !is null) { auto outfile = File(fileName, "w"); - scope(exit) outfile.close; + scope (exit) + outfile.close; outfile.writeln(output); } else diff --git a/source/fahrplanparser.d b/source/fahrplanparser.d index 6b64bac..7b0e9d3 100644 --- a/source/fahrplanparser.d +++ b/source/fahrplanparser.d @@ -6,24 +6,37 @@ import std.conv : to; import std.datetime : dur, TimeOfDay; import std.regex : ctRegex, matchAll; import std.string : strip; -import std.typecons : tuple; +import std.typecons : tuple, Tuple; import kxml.xml : readDocument, XmlNode; import substitution; +private: + +enum ScheduleHeadings +{ + date, + departure, + line, + direction, + platform +} + public: auto parsedFahrplan(in string data) { + // dfmt off return data.readDocument - .parseXPath(`//table[@id="departureMonitor"]/tbody/tr`)[1 .. $] - .getRowContents - .filter!(row => !row.empty) - .map!(a => ["departure" : a[0].parseTime[0].to!string[0 .. $ - 3], - "delay" : a[0].parseTime[1].total!"minutes".to!string, - "line" : a[1], - "direction" : a[2].substitute]); + .parseXPath(`//table[@id="departureMonitor"]/tbody/tr`)[1 .. $] + .getRowContents + .filter!(row => !row.empty) + .map!(a => ["departure" : a[0].parseTime[0].to!string[0 .. $ - 3], + "delay" : a[0].parseTime[1].total!"minutes".to!string, + "line" : a[1], + "direction" : a[2].substitute]); + // dfmt on } private: @@ -54,7 +67,7 @@ auto parseTime(in string input) @safe matches.front["minutes"].to!int); auto timeDiff = actualTime - expectedTime; - if(timeDiff < dur!"minutes"(0)) + if (timeDiff < dur!"minutes"(0)) timeDiff = dur!"hours"(24) + timeDiff; return tuple(expectedTime, timeDiff); @@ -65,6 +78,7 @@ auto parseTime(in string input) @safe @safe unittest { import std.exception : assertThrown; + assertThrown(parseTime("")); assertThrown(parseTime("lkeqf")); assertThrown(parseTime(":")); @@ -83,14 +97,64 @@ auto parseTime(in string input) @safe assert("17:53 (planmäßig 17:51 Uhr)".parseTime == tuple(TimeOfDay(17, 51), dur!"minutes"(2))); - assert("00:00 23:59".parseTime == tuple(TimeOfDay(23,59), dur!"minutes"(1))); + assert("00:00 23:59".parseTime == tuple(TimeOfDay(23, 59), dur!"minutes"(1))); } auto getRowContents(XmlNode[] rows) { - return rows.map!(row => row.parseXPath("//td")[1 .. $ - 1].map!((column) { - auto link = column.parseXPath("//a"); - if (!link.empty) - return link.front.getCData.replace("...", ""); - return column.getCData;})); + return rows.map!(x => getRowContent(x)); +} + +auto getRowContent(XmlNode row) +{ + return row.parseXPath("//td")[ScheduleHeadings.departure .. ScheduleHeadings.direction + 1].map!( + cell => stripLinks(cell)); +} + +auto stripLinks(XmlNode cell) +{ + auto links = cell.parseXPath("//a"); + if (links.empty) + { + return cell.getCData; + } + else + { + return links.front.getCData.replace("...", ""); + } +} + +@system unittest +{ + auto foo = new XmlNode("foo"); + assert(foo.stripLinks == ""); + + auto link = new XmlNode("a"); + link.setCData("test"); + foo.addChild(link); + assert(foo.stripLinks == "test"); + + link.setCData("test2..."); + assert(foo.stripLinks == "test2"); + + auto bar = new XmlNode("bar"); + bar.setCData("test3"); + assert(bar.stripLinks == "test3"); + + bar.addChild(link); + assert(bar.stripLinks == "test2"); + + auto baz = new XmlNode("baz"); + auto subNode = new XmlNode("subNode"); + baz.addChild(subNode); + assert(baz.stripLinks == ""); + + baz.addChild(link); + assert(baz.stripLinks == "test2"); + + baz.addCData("test4"); + assert(baz.stripLinks == "test2"); + + baz.removeChild(link); + assert(baz.stripLinks == "test4"); } From d657873f9f29ea4c5aa4b241491528b38954490b Mon Sep 17 00:00:00 2001 From: Johannes Loher Date: Mon, 10 Apr 2017 20:35:41 +0200 Subject: [PATCH 09/21] It seems we now need to use the id for the bus stop for it to be uniquely identified. --- source/app.d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/app.d b/source/app.d index 32a0650..1c4b745 100644 --- a/source/app.d +++ b/source/app.d @@ -14,7 +14,7 @@ import substitution; void main(string[] args) { string fileName; - string busStop = "Universität Regensburg"; + string busStop = "4014080"; string substitutionFileName = "replacement.txt"; auto helpInformation = getopt(args, From 99d36745ebc841311ccc7a51c03855b76ec84ed0 Mon Sep 17 00:00:00 2001 From: Johannes Loher Date: Mon, 10 Apr 2017 21:07:53 +0200 Subject: [PATCH 10/21] Moved check for the existance of the substitution file from loadSubstitutionFile to main --- source/app.d | 8 ++++++-- source/substitution.d | 16 ++++------------ 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/source/app.d b/source/app.d index 1c4b745..604f62d 100644 --- a/source/app.d +++ b/source/app.d @@ -1,5 +1,6 @@ import std.array : array, replace; import std.datetime : Clock; +import std.file : exists, isFile; import std.format : format; import std.getopt : defaultGetoptPrinter, getopt; import std.json : JSONValue; @@ -35,8 +36,11 @@ void main(string[] args) "mode" : "direct", "type_dm" : "any", "itdLPxx_bcl" : "true"]); - - loadSubstitutionFile(substitutionFileName); + + if (substitutionFileName.exists && substitutionFileName.isFile) + { + loadSubstitutionFile(substitutionFileName); + } auto currentTime = Clock.currTime; JSONValue j = ["time" : "%02s:%02s".format(currentTime.hour, currentTime.minute)]; diff --git a/source/substitution.d b/source/substitution.d index d743fe9..9275705 100644 --- a/source/substitution.d +++ b/source/substitution.d @@ -4,19 +4,11 @@ public: void loadSubstitutionFile(string fileName) { - import std.file : slurp, exists, isFile; + import std.file : slurp; import std.algorithm.iteration : each; - - if (fileName.exists && fileName.isFile) - { - auto data = slurp!(string, string)(fileName, `"%s" = "%s"`); - map = (string[string]).init; - data.each!(pair => map[pair[0]] = pair[1]); - } - else - { - map = (string[string]).init; - } + auto data = slurp!(string, string)(fileName, `"%s" = "%s"`); + map = (string[string]).init; + data.each!(pair => map[pair[0]] = pair[1]); } auto substitute(string s) @safe nothrow From 86d88607670e373088eb9df06861a7f9db008a75 Mon Sep 17 00:00:00 2001 From: Johannes Loher Date: Mon, 10 Apr 2017 22:08:46 +0200 Subject: [PATCH 11/21] made slurp a template parameter of loadSubstitutionFile and created tests for loadSubstitutionFile --- source/app.d | 2 +- source/substitution.d | 55 ++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/source/app.d b/source/app.d index 604f62d..db49108 100644 --- a/source/app.d +++ b/source/app.d @@ -36,7 +36,7 @@ void main(string[] args) "mode" : "direct", "type_dm" : "any", "itdLPxx_bcl" : "true"]); - + if (substitutionFileName.exists && substitutionFileName.isFile) { loadSubstitutionFile(substitutionFileName); diff --git a/source/substitution.d b/source/substitution.d index 9275705..deb90b6 100644 --- a/source/substitution.d +++ b/source/substitution.d @@ -1,16 +1,65 @@ module substitution; +import std.file: slurp; + public: -void loadSubstitutionFile(string fileName) +void loadSubstitutionFile(alias slurpFun = slurp)(string fileName) { - import std.file : slurp; import std.algorithm.iteration : each; - auto data = slurp!(string, string)(fileName, `"%s" = "%s"`); + auto data = slurpFun!(string, string)(fileName, `"%s" = "%s"`); map = (string[string]).init; data.each!(pair => map[pair[0]] = pair[1]); } +@safe unittest +{ + import std.algorithm: canFind; + import std.typecons: Tuple, tuple; + + static Tuple!(string, string)[] mockSlurpEmpty(Type1, Type2)(string filename, in char[] format) + { + return []; + } + + loadSubstitutionFile!mockSlurpEmpty(""); + assert(map.length == 0); + + static Tuple!(string, string)[] mockSlurpEmptyEntry(Type1, Type2)(string filename, in char[] format) + { + return [tuple("", "")]; + } + + loadSubstitutionFile!mockSlurpEmptyEntry(""); + assert("" in map); + assert(map.length == 1); + assert(map[""] == ""); + + static Tuple!(string, string)[] mockSlurpSingleEntry(Type1, Type2)(string filename, in char[] format) + { + return [tuple("foo", "bar")]; + } + + loadSubstitutionFile!mockSlurpSingleEntry(""); + assert("foo" in map); + assert(map.length == 1); + assert(map["foo"] == "bar"); + + static Tuple!(string, string)[] mockSlurpMultipleEntries(Type1, Type2)(string filename, in char[] format) + { + return [tuple("", ""), tuple("0", "1"), tuple("Text in", "wird durch diesen ersetzt")]; + } + + loadSubstitutionFile!mockSlurpMultipleEntries(""); + assert("" in map); + assert("0" in map); + assert("Text in" in map); + assert(map.length == 3); + assert(map[""] == ""); + assert(map["0"] == "1"); + assert(map["Text in"] == "wird durch diesen ersetzt"); +} + auto substitute(string s) @safe nothrow { return s in map ? map[s] : s; From 446f9a096b227ed99d1e54038d0a31efe16224a9 Mon Sep 17 00:00:00 2001 From: Johannes Loher Date: Mon, 10 Apr 2017 22:49:20 +0200 Subject: [PATCH 12/21] added template constraint to loadSubstitutionFile --- source/substitution.d | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/source/substitution.d b/source/substitution.d index deb90b6..315b868 100644 --- a/source/substitution.d +++ b/source/substitution.d @@ -1,12 +1,16 @@ module substitution; -import std.file: slurp; +import std.file : slurp; +import std.meta : AliasSeq; +import std.traits : Parameters; public: void loadSubstitutionFile(alias slurpFun = slurp)(string fileName) + if (is(Parameters!(slurpFun!(string, string)) == AliasSeq!(string, const char[]))) { import std.algorithm.iteration : each; + auto data = slurpFun!(string, string)(fileName, `"%s" = "%s"`); map = (string[string]).init; data.each!(pair => map[pair[0]] = pair[1]); @@ -14,8 +18,7 @@ void loadSubstitutionFile(alias slurpFun = slurp)(string fileName) @safe unittest { - import std.algorithm: canFind; - import std.typecons: Tuple, tuple; + import std.typecons : Tuple, tuple; static Tuple!(string, string)[] mockSlurpEmpty(Type1, Type2)(string filename, in char[] format) { @@ -25,7 +28,8 @@ void loadSubstitutionFile(alias slurpFun = slurp)(string fileName) loadSubstitutionFile!mockSlurpEmpty(""); assert(map.length == 0); - static Tuple!(string, string)[] mockSlurpEmptyEntry(Type1, Type2)(string filename, in char[] format) + static Tuple!(string, string)[] mockSlurpEmptyEntry(Type1, Type2)(string filename, + in char[] format) { return [tuple("", "")]; } @@ -35,7 +39,8 @@ void loadSubstitutionFile(alias slurpFun = slurp)(string fileName) assert(map.length == 1); assert(map[""] == ""); - static Tuple!(string, string)[] mockSlurpSingleEntry(Type1, Type2)(string filename, in char[] format) + static Tuple!(string, string)[] mockSlurpSingleEntry(Type1, Type2)(string filename, + in char[] format) { return [tuple("foo", "bar")]; } @@ -45,7 +50,8 @@ void loadSubstitutionFile(alias slurpFun = slurp)(string fileName) assert(map.length == 1); assert(map["foo"] == "bar"); - static Tuple!(string, string)[] mockSlurpMultipleEntries(Type1, Type2)(string filename, in char[] format) + static Tuple!(string, string)[] mockSlurpMultipleEntries(Type1, Type2)( + string filename, in char[] format) { return [tuple("", ""), tuple("0", "1"), tuple("Text in", "wird durch diesen ersetzt")]; } From ff4c72628fdbe750e7e290c1a3acdd4150d57b24 Mon Sep 17 00:00:00 2001 From: Johannes Loher Date: Fri, 14 Apr 2017 14:33:50 +0200 Subject: [PATCH 13/21] got rid of unnecessary temporary variable --- source/substitution.d | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/substitution.d b/source/substitution.d index 315b868..66e7b9d 100644 --- a/source/substitution.d +++ b/source/substitution.d @@ -11,9 +11,8 @@ void loadSubstitutionFile(alias slurpFun = slurp)(string fileName) { import std.algorithm.iteration : each; - auto data = slurpFun!(string, string)(fileName, `"%s" = "%s"`); map = (string[string]).init; - data.each!(pair => map[pair[0]] = pair[1]); + slurpFun!(string, string)(fileName, `"%s" = "%s"`).each!(pair => map[pair[0]] = pair[1]); } @safe unittest From 94053e443a5b9ba5dbeaa7e2f3598648313aa3f6 Mon Sep 17 00:00:00 2001 From: Johannes Loher Date: Sun, 14 May 2017 11:33:10 +0200 Subject: [PATCH 14/21] Added templates for issues and merge requests --- .gitlab/issue_templates/Bug.md | 25 +++++++++++++++++++ .gitlab/issue_templates/Enhancement.md | 16 ++++++++++++ .../merge_request_templates/Ready_to_Merge.md | 5 ++++ .../Work_In_Progress.md | 9 +++++++ 4 files changed, 55 insertions(+) create mode 100644 .gitlab/issue_templates/Bug.md create mode 100644 .gitlab/issue_templates/Enhancement.md create mode 100644 .gitlab/merge_request_templates/Ready_to_Merge.md create mode 100644 .gitlab/merge_request_templates/Work_In_Progress.md diff --git a/.gitlab/issue_templates/Bug.md b/.gitlab/issue_templates/Bug.md new file mode 100644 index 0000000..406ba16 --- /dev/null +++ b/.gitlab/issue_templates/Bug.md @@ -0,0 +1,25 @@ +Please add the following, if applicable, remove otherwise: + +## Summary ## + +(max. 60 words) + +## Basic information ## + + - Version: + - Component: + - Platform: (RasPi, x86_64, i686) + +## Description ## + +### Expected behaviour ### + +### Actual behaviour ### + +### Reproduction steps ## + +(Please attach files if necessary) + +## Other things to be mentioned ## + +(Ideas for fixing, requests for comments, …) diff --git a/.gitlab/issue_templates/Enhancement.md b/.gitlab/issue_templates/Enhancement.md new file mode 100644 index 0000000..0f25bc4 --- /dev/null +++ b/.gitlab/issue_templates/Enhancement.md @@ -0,0 +1,16 @@ +## Summary ## + +(max 60 words) + +## Basic information ## + + - Component: + - New node needed: (Yes/No) + +## Description ## + +As a , I want to + +## Acceptance criteria ## + +(How should this be in the end?) diff --git a/.gitlab/merge_request_templates/Ready_to_Merge.md b/.gitlab/merge_request_templates/Ready_to_Merge.md new file mode 100644 index 0000000..254b63e --- /dev/null +++ b/.gitlab/merge_request_templates/Ready_to_Merge.md @@ -0,0 +1,5 @@ +As mentioned in , this implements . + +This includes the following changes: + - … + - … diff --git a/.gitlab/merge_request_templates/Work_In_Progress.md b/.gitlab/merge_request_templates/Work_In_Progress.md new file mode 100644 index 0000000..dabf399 --- /dev/null +++ b/.gitlab/merge_request_templates/Work_In_Progress.md @@ -0,0 +1,9 @@ +As mentioned in , this implements + +For this, the following steps are needed: + + - [ ] List + - [ ] of + - [ ] steps + - [ ] and + - [ ] substeps From e112fbe7ec92d77fda496fe5e7a2a45db388897c Mon Sep 17 00:00:00 2001 From: Johannes Loher Date: Sun, 14 May 2017 11:50:22 +0200 Subject: [PATCH 15/21] updated README and improved help message --- README.md | 15 ++++++++++++++- source/app.d | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8f0e99a..012cc00 100644 --- a/README.md +++ b/README.md @@ -5,4 +5,17 @@ A JSON-Getter written in D for bayernfahrplan --------------------------------------------- bayernfahrplan is a tool wich lets you fetch departure tables from -bayernfahrplan.de and output them as JSON. \ No newline at end of file +bayernfahrplan.de and output them as JSON. + +Usage +----- + +``` +Usage: ./learncrypt [options] + + Options: +-f --file The file that the data is written to. +-s --stop The bus stop for which to fetch data. +-r --replacement-file The file that contais the direction name replacement info. +-h --help This help information. +``` \ No newline at end of file diff --git a/source/app.d b/source/app.d index db49108..0c638f7 100644 --- a/source/app.d +++ b/source/app.d @@ -25,7 +25,7 @@ void main(string[] args) if (helpInformation.helpWanted) { - defaultGetoptPrinter("Some information about the program.", helpInformation.options); + defaultGetoptPrinter("Usage: ./learncrypt [options]\n\n Options:", helpInformation.options); return; } From 82f134f565a2ad85647c273fd8787b35c0140a09 Mon Sep 17 00:00:00 2001 From: Johannes Loher Date: Sun, 14 May 2017 11:54:09 +0200 Subject: [PATCH 16/21] fixed stupid copy/paste mistake --- README.md | 2 +- source/app.d | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 012cc00..ed1cb4c 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Usage ----- ``` -Usage: ./learncrypt [options] +Usage: bayernfahrplan [options] Options: -f --file The file that the data is written to. diff --git a/source/app.d b/source/app.d index 0c638f7..83ec0a1 100644 --- a/source/app.d +++ b/source/app.d @@ -25,7 +25,7 @@ void main(string[] args) if (helpInformation.helpWanted) { - defaultGetoptPrinter("Usage: ./learncrypt [options]\n\n Options:", helpInformation.options); + defaultGetoptPrinter("Usage: bayernfahrplan [options]\n\n Options:", helpInformation.options); return; } From 0f95b5777e5997ae8080e77dd3814b160fae665c Mon Sep 17 00:00:00 2001 From: Johannes Loher Date: Tue, 11 Apr 2017 18:57:10 +0200 Subject: [PATCH 17/21] switched api to http://mobile.defas-fgi.de/beg/ and added tests for the corresponding functions --- source/app.d | 40 +++-- source/fahrplanparser.d | 314 +++++++++++++++++++++++----------------- 2 files changed, 210 insertions(+), 144 deletions(-) diff --git a/source/app.d b/source/app.d index 83ec0a1..ba5a3ed 100644 --- a/source/app.d +++ b/source/app.d @@ -6,22 +6,29 @@ import std.getopt : defaultGetoptPrinter, getopt; import std.json : JSONValue; import std.stdio : File, writeln; -import requests : postContent; +import requests : getContent; import fahrplanparser; import substitution; +// find stops: http://mobile.defas-fgi.de/beg/XML_STOPFINDER_REQUEST?outputFormat=XML&stateless=1&locationServerActive=1®ionID_sf=1&type_sf=stop&name_sf=Regensburg%20Universit%C3%A4t + +enum baseURL = "http://mobile.defas-fgi.de/beg/"; +enum departureMonitorRequest = "XML_DM_REQUEST"; +enum stopFinderRequest = "XML_STOPFINDER_REQUEST"; + void main(string[] args) { string fileName; string busStop = "4014080"; string substitutionFileName = "replacement.txt"; - + // dfmt off auto helpInformation = getopt(args, - "file|f", "The file that the data is written to.", &fileName, - "stop|s", "The bus stop for which to fetch data.", &busStop, - "replacement-file|r", "The file that contais the direction name replacement info.", &substitutionFileName); + "file|f", "The file that the data is written to.", &fileName, + "stop|s", "The bus stop for which to fetch data.", &busStop, + "replacement-file|r", "The file that contais the direction name replacement info.", &substitutionFileName); + // dfmt on if (helpInformation.helpWanted) { @@ -29,13 +36,20 @@ void main(string[] args) return; } - auto content = postContent("http://txt.bayern-fahrplan.de/textversion/bcl_abfahrtstafel", - ["limit" : "20", - "useRealtime" : "1", - "name_dm" : busStop, - "mode" : "direct", - "type_dm" : "any", - "itdLPxx_bcl" : "true"]); + // dfmt off + auto content = getContent(baseURL ~ departureMonitorRequest, + ["outputFormat" : "XML", + "language" : "de", + "stateless" : "1", + "type_dm" : "stop", + "name_dm" : busStop, + "useRealtime" : "1", + "mode" : "direct", + "ptOptionActive" : "1", + "mergeDep" : "1", + "limit" : "20", + "deleteAssignedStops_dm" : "1"]); + // dfmt on if (substitutionFileName.exists && substitutionFileName.isFile) { @@ -44,6 +58,7 @@ void main(string[] args) auto currentTime = Clock.currTime; JSONValue j = ["time" : "%02s:%02s".format(currentTime.hour, currentTime.minute)]; + j.object["departures"] = (cast(string) content.data).parsedFahrplan.array.JSONValue; auto output = j.toPrettyString.replace("\\/", "/"); if (fileName !is null) @@ -57,5 +72,4 @@ void main(string[] args) { output.writeln; } - } diff --git a/source/fahrplanparser.d b/source/fahrplanparser.d index 7b0e9d3..e92cb34 100644 --- a/source/fahrplanparser.d +++ b/source/fahrplanparser.d @@ -1,12 +1,10 @@ module fahrplanparser; -import std.algorithm : filter, map; -import std.array : empty, front, replace; +import std.algorithm : map; +import std.array : front; import std.conv : to; -import std.datetime : dur, TimeOfDay; -import std.regex : ctRegex, matchAll; -import std.string : strip; -import std.typecons : tuple, Tuple; +import std.datetime : dur, TimeOfDay, DateTimeException; +import std.string : format; import kxml.xml : readDocument, XmlNode; @@ -14,14 +12,18 @@ import substitution; private: -enum ScheduleHeadings +enum departureNodeName = "dp"; +enum timeNodeName = "t"; +enum realTimeNodeName = "rt"; + +enum departuresXPath = "/efa/dps/" ~ departureNodeName; +template timeXPath(string _timeNodeName = timeNodeName) { - date, - departure, - line, - direction, - platform + enum timeXPath = "/st/" ~ _timeNodeName; } +enum useRealTimeXPath = "/realtime"; +enum lineXPath = "/m/nu"; +enum directionXPath = "/m/des"; public: @@ -29,132 +31,182 @@ auto parsedFahrplan(in string data) { // dfmt off return data.readDocument - .parseXPath(`//table[@id="departureMonitor"]/tbody/tr`)[1 .. $] - .getRowContents - .filter!(row => !row.empty) - .map!(a => ["departure" : a[0].parseTime[0].to!string[0 .. $ - 3], - "delay" : a[0].parseTime[1].total!"minutes".to!string, - "line" : a[1], - "direction" : a[2].substitute]); + .parseXPath(departuresXPath) + .map!(dp => ["departure" : "%02s:%02s".format(dp.departureTime.hour, dp.departureTime.minute), + "delay" : dp.delay.total!"minutes".to!string, + "line": dp.parseXPath(lineXPath).front.getCData, + "direction": dp.parseXPath(directionXPath).front.getCData.substitute]); // dfmt on } -private: - -class BadTimeInputException : Exception -{ - this(string msg) @safe pure nothrow @nogc - { - super(msg); - } - - this() @safe pure nothrow @nogc - { - this(""); - } -} - -auto parseTime(in string input) @safe -{ - auto matches = matchAll(input, ctRegex!(`(?P\d{1,2}):(?P\d{2})`)); - if (matches.empty) - throw new BadTimeInputException(); - auto actualTime = TimeOfDay(matches.front["hours"].to!int, matches.front["minutes"].to!int); - matches.popFront; - if (!matches.empty) - { - auto expectedTime = TimeOfDay(matches.front["hours"].to!int, - matches.front["minutes"].to!int); - auto timeDiff = actualTime - expectedTime; - - if (timeDiff < dur!"minutes"(0)) - timeDiff = dur!"hours"(24) + timeDiff; - - return tuple(expectedTime, timeDiff); - } - return tuple(actualTime, dur!"minutes"(0)); -} - -@safe unittest -{ - import std.exception : assertThrown; - - assertThrown(parseTime("")); - assertThrown(parseTime("lkeqf")); - assertThrown(parseTime(":")); - assertThrown(parseTime("00:0")); - - assert("00:00".parseTime == tuple(TimeOfDay(0, 0), dur!"minutes"(0))); - assert("0:00".parseTime == tuple(TimeOfDay(0, 0), dur!"minutes"(0))); - - assert("00:00 00:00".parseTime == tuple(TimeOfDay(0, 0), dur!"minutes"(0))); - - assert("00:00 00:00 12:00".parseTime == tuple(TimeOfDay(0, 0), dur!"minutes"(0))); - - assert("12:3412:34".parseTime == tuple(TimeOfDay(12, 34), dur!"minutes"(0))); - - assert("ölqjfo12:34oieqf12:31ölqjf".parseTime == tuple(TimeOfDay(12, 31), dur!"minutes"(3))); - - assert("17:53 (planmäßig 17:51 Uhr)".parseTime == tuple(TimeOfDay(17, 51), dur!"minutes"(2))); - - assert("00:00 23:59".parseTime == tuple(TimeOfDay(23, 59), dur!"minutes"(1))); -} - -auto getRowContents(XmlNode[] rows) -{ - return rows.map!(x => getRowContent(x)); -} - -auto getRowContent(XmlNode row) -{ - return row.parseXPath("//td")[ScheduleHeadings.departure .. ScheduleHeadings.direction + 1].map!( - cell => stripLinks(cell)); -} - -auto stripLinks(XmlNode cell) -{ - auto links = cell.parseXPath("//a"); - if (links.empty) - { - return cell.getCData; - } - else - { - return links.front.getCData.replace("...", ""); - } -} - @system unittest { - auto foo = new XmlNode("foo"); - assert(foo.stripLinks == ""); + import std.stdio; + import std.array: array; + auto xml = ""; + assert(xml.parsedFahrplan.array == []); - auto link = new XmlNode("a"); - link.setCData("test"); - foo.addChild(link); - assert(foo.stripLinks == "test"); + xml = ""; + assert(xml.parsedFahrplan.array == []); - link.setCData("test2..."); - assert(foo.stripLinks == "test2"); + xml = "1122412426Wernerwerkstraße"; + assert(xml.parsedFahrplan.array == [["direction":"Wernerwerkstraße", "line":"6", "departure":"12:24", "delay":"18"]]); - auto bar = new XmlNode("bar"); - bar.setCData("test3"); - assert(bar.stripLinks == "test3"); + xml = "012246Wernerwerkstraße"; + assert(xml.parsedFahrplan.array == [["direction":"Wernerwerkstraße", "line":"6", "departure":"12:24", "delay":"0"]]); - bar.addChild(link); - assert(bar.stripLinks == "test2"); - - auto baz = new XmlNode("baz"); - auto subNode = new XmlNode("subNode"); - baz.addChild(subNode); - assert(baz.stripLinks == ""); - - baz.addChild(link); - assert(baz.stripLinks == "test2"); - - baz.addCData("test4"); - assert(baz.stripLinks == "test2"); - - baz.removeChild(link); - assert(baz.stripLinks == "test4"); + xml = "012246Wernerwerkstraße11353135611Burgweinting"; + assert(xml.parsedFahrplan.array == [["direction":"Wernerwerkstraße", "line":"6", "departure":"12:24", "delay":"0"], ["direction":"Burgweinting", "line":"11", "departure":"13:53", "delay":"3"]]); +} + +private: + +class UnexpectedValueException(T) : Exception +{ + this(T t, string node) @safe pure + { + super(`Unexpected value "%s" for node "%s"`.format(t, node)); + } +} + +auto departureTime(string _timeNodeName = timeNodeName)(XmlNode dp) +in +{ + assert(dp.getName == departureNodeName); +} +body +{ + return TimeOfDay.fromISOString(dp.parseXPath(timeXPath!_timeNodeName).front.getCData ~ "00"); +} + +@system unittest +{ + import std.exception : assertThrown; + + auto xml = "0000".readDocument.parseXPath("/dp").front; + assert(xml.departureTime == TimeOfDay(0, 0)); + + xml = "0013".readDocument.parseXPath("/dp").front; + assert(xml.departureTime == TimeOfDay(0, 13)); + + xml = "1100".readDocument.parseXPath("/dp").front; + assert(xml.departureTime == TimeOfDay(11, 00)); + + xml = "1242".readDocument.parseXPath("/dp").front; + assert(xml.departureTime == TimeOfDay(12, 42)); + + xml = "2359".readDocument.parseXPath("/dp").front; + assert(xml.departureTime == TimeOfDay(23, 59)); + + assertThrown!DateTimeException("2400".readDocument.parseXPath("/dp") + .front.departureTime); + assertThrown!DateTimeException("0061".readDocument.parseXPath("/dp") + .front.departureTime); + assertThrown!DateTimeException("2567".readDocument.parseXPath("/dp") + .front.departureTime); + assertThrown!DateTimeException("".readDocument.parseXPath("/dp") + .front.departureTime); + assertThrown!DateTimeException("0".readDocument.parseXPath("/dp") + .front.departureTime); + assertThrown!DateTimeException("00".readDocument.parseXPath("/dp") + .front.departureTime); + assertThrown!DateTimeException("000000".readDocument.parseXPath("/dp") + .front.departureTime); + assertThrown!DateTimeException("00:00".readDocument.parseXPath("/dp") + .front.departureTime); + assertThrown!DateTimeException("abcd".readDocument.parseXPath("/dp") + .front.departureTime); +} + +auto delay(XmlNode dp) +in +{ + assert(dp.getName == departureNodeName); +} +body +{ + auto useRealtimeString = dp.parseXPath(useRealTimeXPath).front.getCData; + if (useRealtimeString == "0") + return dur!"minutes"(0); + else if (useRealtimeString == "1") + { + auto expectedTime = dp.departureTime; + auto realTime = dp.departureTime!realTimeNodeName; + auto timeDiff = realTime - expectedTime; + if (timeDiff < dur!"minutes"(0)) + timeDiff = dur!"hours"(24) + timeDiff; + return timeDiff; + } + else + throw new UnexpectedValueException!string(useRealtimeString, "realtime"); +} + +@system unittest +{ + import std.exception : assertThrown; + import core.exception : AssertError; + + auto xml = "0".readDocument.parseXPath("/dp").front; + assert(xml.delay == dur!"minutes"(0)); + + xml = "".readDocument.parseXPath("/dp").front; + assertThrown!(UnexpectedValueException!string)(xml.delay); + + xml = "2".readDocument.parseXPath("/dp").front; + assertThrown!(UnexpectedValueException!string)(xml.delay); + + xml = "a".readDocument.parseXPath("/dp").front; + assertThrown!(UnexpectedValueException!string)(xml.delay); + + xml = "1".readDocument.parseXPath("/dp").front; + assertThrown!AssertError(xml.delay); + + xml = "1".readDocument.parseXPath("/dp").front; + assertThrown!DateTimeException(xml.delay); + + xml = "1".readDocument.parseXPath("/dp").front; + assertThrown!AssertError(xml.delay); + + xml = "".readDocument.parseXPath("/dp").front; + assertThrown!AssertError(xml.delay); + + xml = "1".readDocument.parseXPath("/dp") + .front; + assertThrown!DateTimeException(xml.delay); + + xml = "10000".readDocument.parseXPath("/dp") + .front; + assertThrown!DateTimeException(xml.delay); + + xml = "10000".readDocument.parseXPath("/dp") + .front; + assertThrown!DateTimeException(xml.delay); + + xml = "100000000" + .readDocument.parseXPath("/dp").front; + assert(xml.delay == dur!"minutes"(0)); + + xml = "100010000" + .readDocument.parseXPath("/dp").front; + assert(xml.delay == dur!"minutes"(1)); + + xml = "117531751" + .readDocument.parseXPath("/dp").front; + assert(xml.delay == dur!"minutes"(2)); + + xml = "110101000" + .readDocument.parseXPath("/dp").front; + assert(xml.delay == dur!"minutes"(10)); + + xml = "113011242" + .readDocument.parseXPath("/dp").front; + assert(xml.delay == dur!"minutes"(19)); + + xml = "100001242" + .readDocument.parseXPath("/dp").front; + assert(xml.delay == dur!"minutes"(678)); + + xml = "100002359" + .readDocument.parseXPath("/dp").front; + assert(xml.delay == dur!"minutes"(1)); } From a6aeaa336e503eedddbddb77aceabcdac253111d Mon Sep 17 00:00:00 2001 From: Johannes Loher Date: Mon, 8 May 2017 22:38:31 +0200 Subject: [PATCH 18/21] use name of bus stop instead of id --- source/app.d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/app.d b/source/app.d index ba5a3ed..a407ff6 100644 --- a/source/app.d +++ b/source/app.d @@ -21,7 +21,7 @@ enum stopFinderRequest = "XML_STOPFINDER_REQUEST"; void main(string[] args) { string fileName; - string busStop = "4014080"; + string busStop = "Regensburg Universität"; string substitutionFileName = "replacement.txt"; // dfmt off auto helpInformation = getopt(args, From c7cf7fedfd321f4866040e910d6ae857739ac18a Mon Sep 17 00:00:00 2001 From: Johannes Loher Date: Sun, 14 May 2017 11:29:19 +0200 Subject: [PATCH 19/21] cleanup --- dub.json | 2 +- source/app.d | 3 --- source/fahrplanparser.d | 21 +++++++++++++-------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/dub.json b/dub.json index d5076dd..23a066c 100644 --- a/dub.json +++ b/dub.json @@ -5,7 +5,7 @@ "Oliver Rümpelein" ], "dependencies": { - "requests": "~>0.3.1", + "requests": "~>0.4.1", "kxml": "~>1.0.1" }, "description": "A minimal D application.", diff --git a/source/app.d b/source/app.d index a407ff6..ce064ab 100644 --- a/source/app.d +++ b/source/app.d @@ -12,11 +12,8 @@ import fahrplanparser; import substitution; -// find stops: http://mobile.defas-fgi.de/beg/XML_STOPFINDER_REQUEST?outputFormat=XML&stateless=1&locationServerActive=1®ionID_sf=1&type_sf=stop&name_sf=Regensburg%20Universit%C3%A4t - enum baseURL = "http://mobile.defas-fgi.de/beg/"; enum departureMonitorRequest = "XML_DM_REQUEST"; -enum stopFinderRequest = "XML_STOPFINDER_REQUEST"; void main(string[] args) { diff --git a/source/fahrplanparser.d b/source/fahrplanparser.d index e92cb34..7860e86 100644 --- a/source/fahrplanparser.d +++ b/source/fahrplanparser.d @@ -21,6 +21,7 @@ template timeXPath(string _timeNodeName = timeNodeName) { enum timeXPath = "/st/" ~ _timeNodeName; } + enum useRealTimeXPath = "/realtime"; enum lineXPath = "/m/nu"; enum directionXPath = "/m/des"; @@ -41,8 +42,8 @@ auto parsedFahrplan(in string data) @system unittest { - import std.stdio; - import std.array: array; + import std.array : array; + auto xml = ""; assert(xml.parsedFahrplan.array == []); @@ -50,13 +51,17 @@ auto parsedFahrplan(in string data) assert(xml.parsedFahrplan.array == []); xml = "1122412426Wernerwerkstraße"; - assert(xml.parsedFahrplan.array == [["direction":"Wernerwerkstraße", "line":"6", "departure":"12:24", "delay":"18"]]); + assert(xml.parsedFahrplan.array == [["direction" : "Wernerwerkstraße", + "line" : "6", "departure" : "12:24", "delay" : "18"]]); xml = "012246Wernerwerkstraße"; - assert(xml.parsedFahrplan.array == [["direction":"Wernerwerkstraße", "line":"6", "departure":"12:24", "delay":"0"]]); + assert(xml.parsedFahrplan.array == [["direction" : "Wernerwerkstraße", + "line" : "6", "departure" : "12:24", "delay" : "0"]]); xml = "012246Wernerwerkstraße11353135611Burgweinting"; - assert(xml.parsedFahrplan.array == [["direction":"Wernerwerkstraße", "line":"6", "departure":"12:24", "delay":"0"], ["direction":"Burgweinting", "line":"11", "departure":"13:53", "delay":"3"]]); + assert(xml.parsedFahrplan.array == [["direction" : "Wernerwerkstraße", "line" : "6", + "departure" : "12:24", "delay" : "0"], ["direction" : "Burgweinting", + "line" : "11", "departure" : "13:53", "delay" : "3"]]); } private: @@ -125,13 +130,13 @@ in } body { - auto useRealtimeString = dp.parseXPath(useRealTimeXPath).front.getCData; + immutable useRealtimeString = dp.parseXPath(useRealTimeXPath).front.getCData; if (useRealtimeString == "0") return dur!"minutes"(0); else if (useRealtimeString == "1") { - auto expectedTime = dp.departureTime; - auto realTime = dp.departureTime!realTimeNodeName; + immutable expectedTime = dp.departureTime; + immutable realTime = dp.departureTime!realTimeNodeName; auto timeDiff = realTime - expectedTime; if (timeDiff < dur!"minutes"(0)) timeDiff = dur!"hours"(24) + timeDiff; From 9d51f28619242f6ea2deb13ed5dec706201c94e4 Mon Sep 17 00:00:00 2001 From: Johannes Loher Date: Tue, 16 May 2017 14:34:42 +0200 Subject: [PATCH 20/21] Added docs for the public stuff in fahrplanparser.d and substitution.d --- source/fahrplanparser.d | 6 ++++++ source/substitution.d | 11 +++++++++++ 2 files changed, 17 insertions(+) diff --git a/source/fahrplanparser.d b/source/fahrplanparser.d index 7860e86..d2961ba 100644 --- a/source/fahrplanparser.d +++ b/source/fahrplanparser.d @@ -28,6 +28,11 @@ enum directionXPath = "/m/des"; public: +/*********************************** +* Parses the departure monitor data and returns it as an associative array. +* data is expected to contain valid XML as returned by queries sent to http://mobile.defas-fgi.de/beg/. +*/ + auto parsedFahrplan(in string data) { // dfmt off @@ -40,6 +45,7 @@ auto parsedFahrplan(in string data) // dfmt on } +/// @system unittest { import std.array : array; diff --git a/source/substitution.d b/source/substitution.d index 66e7b9d..07c239f 100644 --- a/source/substitution.d +++ b/source/substitution.d @@ -6,6 +6,10 @@ import std.traits : Parameters; public: +/*********************************** +* Loads a substitution dictonary from a file. +*/ + void loadSubstitutionFile(alias slurpFun = slurp)(string fileName) if (is(Parameters!(slurpFun!(string, string)) == AliasSeq!(string, const char[]))) { @@ -15,6 +19,7 @@ void loadSubstitutionFile(alias slurpFun = slurp)(string fileName) slurpFun!(string, string)(fileName, `"%s" = "%s"`).each!(pair => map[pair[0]] = pair[1]); } +/// @safe unittest { import std.typecons : Tuple, tuple; @@ -65,11 +70,17 @@ void loadSubstitutionFile(alias slurpFun = slurp)(string fileName) assert(map["Text in"] == "wird durch diesen ersetzt"); } +/*********************************** +* Substitutes a string with its corresponding replacement, if one is available. +* Otherwise just returns the original string. +*/ + auto substitute(string s) @safe nothrow { return s in map ? map[s] : s; } +/// @safe unittest { map[""] = ""; From 4b0102c85d99ad985b067b31ab0f037964f78471 Mon Sep 17 00:00:00 2001 From: Johannes Loher Date: Wed, 12 Jul 2017 15:42:23 +0200 Subject: [PATCH 21/21] fixed crash when rt tag is missing, but realtime is set to 1 --- source/fahrplanparser.d | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/source/fahrplanparser.d b/source/fahrplanparser.d index 7860e86..2f58147 100644 --- a/source/fahrplanparser.d +++ b/source/fahrplanparser.d @@ -1,7 +1,7 @@ module fahrplanparser; import std.algorithm : map; -import std.array : front; +import std.array : empty, front; import std.conv : to; import std.datetime : dur, TimeOfDay, DateTimeException; import std.string : format; @@ -74,6 +74,14 @@ class UnexpectedValueException(T) : Exception } } +class CouldNotFindeNodeException : Exception +{ + this(string node) @safe pure + { + super(`Could not find node "%s"`.format(node)); + } +} + auto departureTime(string _timeNodeName = timeNodeName)(XmlNode dp) in { @@ -81,7 +89,11 @@ in } body { - return TimeOfDay.fromISOString(dp.parseXPath(timeXPath!_timeNodeName).front.getCData ~ "00"); + auto timeNodes = dp.parseXPath(timeXPath!_timeNodeName); + if(timeNodes.empty) + throw new CouldNotFindeNodeException(_timeNodeName); + + return TimeOfDay.fromISOString(timeNodes.front.getCData ~ "00"); } @system unittest @@ -135,12 +147,19 @@ body return dur!"minutes"(0); else if (useRealtimeString == "1") { - immutable expectedTime = dp.departureTime; - immutable realTime = dp.departureTime!realTimeNodeName; - auto timeDiff = realTime - expectedTime; - if (timeDiff < dur!"minutes"(0)) - timeDiff = dur!"hours"(24) + timeDiff; - return timeDiff; + try + { + immutable expectedTime = dp.departureTime; + immutable realTime = dp.departureTime!realTimeNodeName; + auto timeDiff = realTime - expectedTime; + if (timeDiff < dur!"minutes"(0)) + timeDiff = dur!"hours"(24) + timeDiff; + return timeDiff; + } + catch (CouldNotFindeNodeException e) + { + return dur!"minutes"(0); + } } else throw new UnexpectedValueException!string(useRealtimeString, "realtime");