Use crawling date

src/surykatka/bot.py

@@ -509,7 +509,7 @@ class WebBot:
                 checked_domain_dict[dns_change["domain"]] = {}
             checked_domain_dict[dns_change["domain"]][
                 dns_change["rdtype"]
-            ] = dns_change["response"]
+            ] = dns_change
 
             result_dict["dns_query"].append(
                 {
@@ -524,9 +524,9 @@ class WebBot:
         for domain in domain_list:
             if domain in checked_domain_dict:
                 if "A" in checked_domain_dict[domain]:
-                    for server_ip in checked_domain_dict[domain]["A"].split(
-                        ", "
-                    ):
+                    for server_ip in checked_domain_dict[domain]["A"][
+                        "response"
+                    ].split(", "):
                         if not server_ip:
                             # drop empty response
                             continue
@@ -543,9 +543,9 @@ class WebBot:
                     )
 
                 if "MX" in checked_domain_dict[domain]:
-                    if checked_domain_dict[domain]["MX"]:
-                        for mx_domain in checked_domain_dict[domain][
-                            "MX"
+                    if checked_domain_dict[domain]["MX"]["response"]:
+                        for mx_domain in checked_domain_dict[domain]["MX"][
+                            "response"
                         ].split(", "):
                             if mx_domain not in checked_domain_dict:
                                 result_dict["missing_data"].append(
@@ -568,13 +568,24 @@ class WebBot:
                     )
 
                 if "TXT" in checked_domain_dict[domain]:
-                    if '"v=spf' not in checked_domain_dict[domain]["TXT"]:
+                    if (
+                        '"v=spf'
+                        not in checked_domain_dict[domain]["TXT"]["response"]
+                    ):
                         result_dict["warning"].append(
                             {
                                 "text": "(No spf configured: %s) "
-                                % str(checked_domain_dict[domain]["TXT"])
+                                % str(
+                                    checked_domain_dict[domain]["TXT"][
+                                        "response"
+                                    ]
+                                )
                                 + domain,
-                                "date": result_dict["bot_status"][0]["date"],
+                                "date": rfc822(
+                                    checked_domain_dict[domain]["TXT"][
+                                        "status"
+                                    ]
+                                ),
                             }
                         )
                 else:
@@ -730,9 +741,7 @@ class WebBot:
                                 {
                                     "text": "(No Content-Type header) %s"
                                     % (network_change["url"],),
-                                    "date": result_dict["bot_status"][0][
-                                        "date"
-                                    ],
+                                    "date": rfc822(network_change["status"]),
                                 }
                             )
                     elif network_change["http_header_dict"][
@@ -751,9 +760,7 @@ class WebBot:
                                 {
                                     "text": "(No Content-Security-Policy header) %s"
                                     % (network_change["url"],),
-                                    "date": result_dict["bot_status"][0][
-                                        "date"
-                                    ],
+                                    "date": rfc822(network_change["status"]),
                                 }
                             )