Merge branch 'defas-fgi_api' into 'master'

Defas fgi api

This implement the usage of the defas-fgi API instead of parsing txt.bayernfahrplan.de.

This includes the following changes:
 - Rewrite of app.d and fahrplanparser.d to use the new API
 - Addition of unittests for farhplanparser.d (100% cov, yay!)

See merge request !2

dub.json

@@ -5,7 +5,7 @@
 		"Oliver Rümpelein"
 	],
 	"dependencies": {
-		"requests": "~>0.3.1",
+		"requests": "~>0.4.1",
 		"kxml": "~>1.0.1"
 	},
 	"description": "A minimal D application.",

source/app.d

@@ -6,22 +6,26 @@ import std.getopt : defaultGetoptPrinter, getopt;
 import std.json : JSONValue;
 import std.stdio : File, writeln;
 
-import requests : postContent;
+import requests : getContent;
 
 import fahrplanparser;
 
 import substitution;
 
+enum baseURL = "http://mobile.defas-fgi.de/beg/";
+enum departureMonitorRequest = "XML_DM_REQUEST";
+
 void main(string[] args)
 {
     string fileName;
-    string busStop = "4014080";
+    string busStop = "Regensburg Universität";
     string substitutionFileName = "replacement.txt";
-
+    // dfmt off
     auto helpInformation = getopt(args,
-            "file|f", "The file that the data is written to.", &fileName,
+        "file|f", "The file that the data is written to.", &fileName,
-            "stop|s", "The bus stop for which to fetch data.", &busStop,
+        "stop|s", "The bus stop for which to fetch data.", &busStop,
-            "replacement-file|r", "The file that contais the direction name replacement info.", &substitutionFileName);
+        "replacement-file|r", "The file that contais the direction name replacement info.", &substitutionFileName);
+    // dfmt on
 
     if (helpInformation.helpWanted)
     {
@@ -29,13 +33,20 @@ void main(string[] args)
         return;
     }
 
-    auto content = postContent("http://txt.bayern-fahrplan.de/textversion/bcl_abfahrtstafel",
+    // dfmt off
-            ["limit" : "20",
+    auto content = getContent(baseURL ~ departureMonitorRequest,
-             "useRealtime" : "1",
+        ["outputFormat" : "XML",
-             "name_dm" : busStop,
+         "language" : "de",
-             "mode" : "direct",
+         "stateless" : "1",
-             "type_dm" : "any",
+         "type_dm" : "stop",
-             "itdLPxx_bcl" : "true"]);
+         "name_dm" : busStop,
+         "useRealtime" : "1",
+         "mode" : "direct",
+         "ptOptionActive" : "1",
+         "mergeDep" : "1",
+         "limit" : "20",
+         "deleteAssignedStops_dm" : "1"]);
+    // dfmt on
 
     if (substitutionFileName.exists && substitutionFileName.isFile)
     {
@@ -44,6 +55,7 @@ void main(string[] args)
 
     auto currentTime = Clock.currTime;
     JSONValue j = ["time" : "%02s:%02s".format(currentTime.hour, currentTime.minute)];
+
     j.object["departures"] = (cast(string) content.data).parsedFahrplan.array.JSONValue;
     auto output = j.toPrettyString.replace("\\/", "/");
     if (fileName !is null)
@@ -57,5 +69,4 @@ void main(string[] args)
     {
         output.writeln;
     }
-
 }

source/fahrplanparser.d

@@ -1,12 +1,10 @@
 module fahrplanparser;
 
-import std.algorithm : filter, map;
+import std.algorithm : map;
-import std.array : empty, front, replace;
+import std.array : front;
 import std.conv : to;
-import std.datetime : dur, TimeOfDay;
+import std.datetime : dur, TimeOfDay, DateTimeException;
-import std.regex : ctRegex, matchAll;
+import std.string : format;
-import std.string : strip;
-import std.typecons : tuple, Tuple;
 
 import kxml.xml : readDocument, XmlNode;
 
@@ -14,147 +12,206 @@ import substitution;
 
 private:
 
-enum ScheduleHeadings
+enum departureNodeName = "dp";
+enum timeNodeName = "t";
+enum realTimeNodeName = "rt";
+
+enum departuresXPath = "/efa/dps/" ~ departureNodeName;
+template timeXPath(string _timeNodeName = timeNodeName)
 {
-    date,
+    enum timeXPath = "/st/" ~ _timeNodeName;
-    departure,
-    line,
-    direction,
-    platform
 }
 
+enum useRealTimeXPath = "/realtime";
+enum lineXPath = "/m/nu";
+enum directionXPath = "/m/des";
+
 public:
 
 auto parsedFahrplan(in string data)
 {
     // dfmt off
     return data.readDocument
-        .parseXPath(`//table[@id="departureMonitor"]/tbody/tr`)[1 .. $]
+        .parseXPath(departuresXPath)
-        .getRowContents
+        .map!(dp => ["departure" : "%02s:%02s".format(dp.departureTime.hour, dp.departureTime.minute),
-        .filter!(row => !row.empty)
+                     "delay" : dp.delay.total!"minutes".to!string,
-        .map!(a => ["departure" : a[0].parseTime[0].to!string[0 .. $ - 3],
+                     "line": dp.parseXPath(lineXPath).front.getCData,
-                    "delay" : a[0].parseTime[1].total!"minutes".to!string,
+                     "direction": dp.parseXPath(directionXPath).front.getCData.substitute]);
-                    "line" : a[1],
-                    "direction" : a[2].substitute]);
     // dfmt on
 }
 
-private:
-
-class BadTimeInputException : Exception
-{
-    this(string msg) @safe pure nothrow @nogc
-    {
-        super(msg);
-    }
-
-    this() @safe pure nothrow @nogc
-    {
-        this("");
-    }
-}
-
-auto parseTime(in string input) @safe
-{
-    auto matches = matchAll(input, ctRegex!(`(?P<hours>\d{1,2}):(?P<minutes>\d{2})`));
-    if (matches.empty)
-        throw new BadTimeInputException();
-    auto actualTime = TimeOfDay(matches.front["hours"].to!int, matches.front["minutes"].to!int);
-    matches.popFront;
-    if (!matches.empty)
-    {
-        auto expectedTime = TimeOfDay(matches.front["hours"].to!int,
-                matches.front["minutes"].to!int);
-        auto timeDiff = actualTime - expectedTime;
-
-        if (timeDiff < dur!"minutes"(0))
-            timeDiff = dur!"hours"(24) + timeDiff;
-
-        return tuple(expectedTime, timeDiff);
-    }
-    return tuple(actualTime, dur!"minutes"(0));
-}
-
-@safe unittest
-{
-    import std.exception : assertThrown;
-
-    assertThrown(parseTime(""));
-    assertThrown(parseTime("lkeqf"));
-    assertThrown(parseTime(":"));
-    assertThrown(parseTime("00:0"));
-
-    assert("00:00".parseTime == tuple(TimeOfDay(0, 0), dur!"minutes"(0)));
-    assert("0:00".parseTime == tuple(TimeOfDay(0, 0), dur!"minutes"(0)));
-
-    assert("00:00 00:00".parseTime == tuple(TimeOfDay(0, 0), dur!"minutes"(0)));
-
-    assert("00:00 00:00 12:00".parseTime == tuple(TimeOfDay(0, 0), dur!"minutes"(0)));
-
-    assert("12:3412:34".parseTime == tuple(TimeOfDay(12, 34), dur!"minutes"(0)));
-
-    assert("ölqjfo12:34oieqf12:31ölqjf".parseTime == tuple(TimeOfDay(12, 31), dur!"minutes"(3)));
-
-    assert("17:53 (planmäßig 17:51 Uhr)".parseTime == tuple(TimeOfDay(17, 51), dur!"minutes"(2)));
-
-    assert("00:00 23:59".parseTime == tuple(TimeOfDay(23, 59), dur!"minutes"(1)));
-}
-
-auto getRowContents(XmlNode[] rows)
-{
-    return rows.map!(x => getRowContent(x));
-}
-
-auto getRowContent(XmlNode row)
-{
-    return row.parseXPath("//td")[ScheduleHeadings.departure .. ScheduleHeadings.direction + 1].map!(
-            cell => stripLinks(cell));
-}
-
-auto stripLinks(XmlNode cell)
-{
-    auto links = cell.parseXPath("//a");
-    if (links.empty)
-    {
-        return cell.getCData;
-    }
-    else
-    {
-        return links.front.getCData.replace("...", "");
-    }
-}
-
 @system unittest
 {
-    auto foo = new XmlNode("foo");
+    import std.array : array;
-    assert(foo.stripLinks == "");
 
-    auto link = new XmlNode("a");
+    auto xml = "";
-    link.setCData("test");
+    assert(xml.parsedFahrplan.array == []);
-    foo.addChild(link);
-    assert(foo.stripLinks == "test");
 
-    link.setCData("test2...");
+    xml = "<efa><dps></dps></efa>";
-    assert(foo.stripLinks == "test2");
+    assert(xml.parsedFahrplan.array == []);
 
-    auto bar = new XmlNode("bar");
+    xml = "<efa><dps><dp><realtime>1</realtime><st><t>1224</t><rt>1242</rt></st><m><nu>6</nu><des>Wernerwerkstraße</des></m></dp></dps></efa>";
-    bar.setCData("test3");
+    assert(xml.parsedFahrplan.array == [["direction" : "Wernerwerkstraße",
-    assert(bar.stripLinks == "test3");
+            "line" : "6", "departure" : "12:24", "delay" : "18"]]);
 
-    bar.addChild(link);
+    xml = "<efa><dps><dp><realtime>0</realtime><st><t>1224</t></st><m><nu>6</nu><des>Wernerwerkstraße</des></m></dp></dps></efa>";
-    assert(bar.stripLinks == "test2");
+    assert(xml.parsedFahrplan.array == [["direction" : "Wernerwerkstraße",
+            "line" : "6", "departure" : "12:24", "delay" : "0"]]);
 
-    auto baz = new XmlNode("baz");
+    xml = "<efa><dps><dp><realtime>0</realtime><st><t>1224</t></st><m><nu>6</nu><des>Wernerwerkstraße</des></m></dp><dp><realtime>1</realtime><st><t>1353</t><rt>1356</rt></st><m><nu>11</nu><des>Burgweinting</des></m></dp></dps></efa>";
-    auto subNode = new XmlNode("subNode");
+    assert(xml.parsedFahrplan.array == [["direction" : "Wernerwerkstraße", "line" : "6",
-    baz.addChild(subNode);
+            "departure" : "12:24", "delay" : "0"], ["direction" : "Burgweinting",
-    assert(baz.stripLinks == "");
+            "line" : "11", "departure" : "13:53", "delay" : "3"]]);
-
+}
-    baz.addChild(link);
+
-    assert(baz.stripLinks == "test2");
+private:
-
+
-    baz.addCData("test4");
+class UnexpectedValueException(T) : Exception
-    assert(baz.stripLinks == "test2");
+{
-
+    this(T t, string node) @safe pure
-    baz.removeChild(link);
+    {
-    assert(baz.stripLinks == "test4");
+        super(`Unexpected value "%s" for node "%s"`.format(t, node));
+    }
+}
+
+auto departureTime(string _timeNodeName = timeNodeName)(XmlNode dp)
+in
+{
+    assert(dp.getName == departureNodeName);
+}
+body
+{
+    return TimeOfDay.fromISOString(dp.parseXPath(timeXPath!_timeNodeName).front.getCData ~ "00");
+}
+
+@system unittest
+{
+    import std.exception : assertThrown;
+
+    auto xml = "<dp><st><t>0000</t></st></dp>".readDocument.parseXPath("/dp").front;
+    assert(xml.departureTime == TimeOfDay(0, 0));
+
+    xml = "<dp><st><t>0013</t></st></dp>".readDocument.parseXPath("/dp").front;
+    assert(xml.departureTime == TimeOfDay(0, 13));
+
+    xml = "<dp><st><t>1100</t></st></dp>".readDocument.parseXPath("/dp").front;
+    assert(xml.departureTime == TimeOfDay(11, 00));
+
+    xml = "<dp><st><t>1242</t></st></dp>".readDocument.parseXPath("/dp").front;
+    assert(xml.departureTime == TimeOfDay(12, 42));
+
+    xml = "<dp><st><t>2359</t></st></dp>".readDocument.parseXPath("/dp").front;
+    assert(xml.departureTime == TimeOfDay(23, 59));
+
+    assertThrown!DateTimeException("<dp><st><t>2400</t></st></dp>".readDocument.parseXPath("/dp")
+            .front.departureTime);
+    assertThrown!DateTimeException("<dp><st><t>0061</t></st></dp>".readDocument.parseXPath("/dp")
+            .front.departureTime);
+    assertThrown!DateTimeException("<dp><st><t>2567</t></st></dp>".readDocument.parseXPath("/dp")
+            .front.departureTime);
+    assertThrown!DateTimeException("<dp><st><t></t></st></dp>".readDocument.parseXPath("/dp")
+            .front.departureTime);
+    assertThrown!DateTimeException("<dp><st><t>0</t></st></dp>".readDocument.parseXPath("/dp")
+            .front.departureTime);
+    assertThrown!DateTimeException("<dp><st><t>00</t></st></dp>".readDocument.parseXPath("/dp")
+            .front.departureTime);
+    assertThrown!DateTimeException("<dp><st><t>000000</t></st></dp>".readDocument.parseXPath("/dp")
+            .front.departureTime);
+    assertThrown!DateTimeException("<dp><st><t>00:00</t></st></dp>".readDocument.parseXPath("/dp")
+            .front.departureTime);
+    assertThrown!DateTimeException("<dp><st><t>abcd</t></st></dp>".readDocument.parseXPath("/dp")
+            .front.departureTime);
+}
+
+auto delay(XmlNode dp)
+in
+{
+    assert(dp.getName == departureNodeName);
+}
+body
+{
+    immutable useRealtimeString = dp.parseXPath(useRealTimeXPath).front.getCData;
+    if (useRealtimeString == "0")
+        return dur!"minutes"(0);
+    else if (useRealtimeString == "1")
+    {
+        immutable expectedTime = dp.departureTime;
+        immutable realTime = dp.departureTime!realTimeNodeName;
+        auto timeDiff = realTime - expectedTime;
+        if (timeDiff < dur!"minutes"(0))
+            timeDiff = dur!"hours"(24) + timeDiff;
+        return timeDiff;
+    }
+    else
+        throw new UnexpectedValueException!string(useRealtimeString, "realtime");
+}
+
+@system unittest
+{
+    import std.exception : assertThrown;
+    import core.exception : AssertError;
+
+    auto xml = "<dp><realtime>0</realtime></dp>".readDocument.parseXPath("/dp").front;
+    assert(xml.delay == dur!"minutes"(0));
+
+    xml = "<dp><realtime></realtime></dp>".readDocument.parseXPath("/dp").front;
+    assertThrown!(UnexpectedValueException!string)(xml.delay);
+
+    xml = "<dp><realtime>2</realtime></dp>".readDocument.parseXPath("/dp").front;
+    assertThrown!(UnexpectedValueException!string)(xml.delay);
+
+    xml = "<dp><realtime>a</realtime></dp>".readDocument.parseXPath("/dp").front;
+    assertThrown!(UnexpectedValueException!string)(xml.delay);
+
+    xml = "<dp><realtime>1</realtime></dp>".readDocument.parseXPath("/dp").front;
+    assertThrown!AssertError(xml.delay);
+
+    xml = "<dp><realtime>1</realtime><st><t></t></st></dp>".readDocument.parseXPath("/dp").front;
+    assertThrown!DateTimeException(xml.delay);
+
+    xml = "<dp><realtime>1</realtime><st><rt></rt></st></dp>".readDocument.parseXPath("/dp").front;
+    assertThrown!AssertError(xml.delay);
+
+    xml = "<dp><st><rt></rt><t></t></st></dp>".readDocument.parseXPath("/dp").front;
+    assertThrown!AssertError(xml.delay);
+
+    xml = "<dp><realtime>1</realtime><st><rt></rt><t></t></st></dp>".readDocument.parseXPath("/dp")
+        .front;
+    assertThrown!DateTimeException(xml.delay);
+
+    xml = "<dp><realtime>1</realtime><st><rt>0000</rt><t></t></st></dp>".readDocument.parseXPath("/dp")
+        .front;
+    assertThrown!DateTimeException(xml.delay);
+
+    xml = "<dp><realtime>1</realtime><st><rt></rt><t>0000</t></st></dp>".readDocument.parseXPath("/dp")
+        .front;
+    assertThrown!DateTimeException(xml.delay);
+
+    xml = "<dp><realtime>1</realtime><st><rt>0000</rt><t>0000</t></st></dp>"
+        .readDocument.parseXPath("/dp").front;
+    assert(xml.delay == dur!"minutes"(0));
+
+    xml = "<dp><realtime>1</realtime><st><rt>0001</rt><t>0000</t></st></dp>"
+        .readDocument.parseXPath("/dp").front;
+    assert(xml.delay == dur!"minutes"(1));
+
+    xml = "<dp><realtime>1</realtime><st><rt>1753</rt><t>1751</t></st></dp>"
+        .readDocument.parseXPath("/dp").front;
+    assert(xml.delay == dur!"minutes"(2));
+
+    xml = "<dp><realtime>1</realtime><st><rt>1010</rt><t>1000</t></st></dp>"
+        .readDocument.parseXPath("/dp").front;
+    assert(xml.delay == dur!"minutes"(10));
+
+    xml = "<dp><realtime>1</realtime><st><rt>1301</rt><t>1242</t></st></dp>"
+        .readDocument.parseXPath("/dp").front;
+    assert(xml.delay == dur!"minutes"(19));
+
+    xml = "<dp><realtime>1</realtime><st><rt>0000</rt><t>1242</t></st></dp>"
+        .readDocument.parseXPath("/dp").front;
+    assert(xml.delay == dur!"minutes"(678));
+
+    xml = "<dp><realtime>1</realtime><st><rt>0000</rt><t>2359</t></st></dp>"
+        .readDocument.parseXPath("/dp").front;
+    assert(xml.delay == dur!"minutes"(1));
 }