"""Verify a daily-calendar fixture or an explicitly supplied URL using only the standard library.""" import argparse import hashlib import json from pathlib import Path from urllib.request import urlopen from urllib.parse import urlsplit def read_bytes(location): if location.startswith(("https://", "http://")): with urlopen(location, timeout=20) as response: return response.read() return Path(location).read_bytes() def resource_name(location): if location.startswith(("https://", "http://")): return urlsplit(location).path.rsplit("/", 1)[-1] return Path(location).name def verify(data_location, manifest_location): manifest = json.loads(read_bytes(manifest_location)) data_bytes = read_bytes(data_location) name = resource_name(data_location) metadata = manifest.get("files", {}).get(name) if not metadata: raise ValueError("The manifest does not describe " + name) actual_hash = hashlib.sha256(data_bytes).hexdigest() if len(data_bytes) != metadata.get("bytes") or actual_hash != metadata.get("sha256"): raise ValueError("Daily-calendar file does not match its manifest hash and byte count") projection = json.loads(data_bytes) if projection.get("data_version") != manifest.get("data_version"): raise ValueError("Data version does not match the manifest") if projection.get("year") != manifest.get("year"): raise ValueError("Calendar year does not match the manifest") if len(projection.get("rows", [])) != manifest.get("rows_count"): raise ValueError("Row count does not match the manifest") return projection def show_comparison(projection): by_date = {} unknown = 0 for row in projection["rows"]: by_date.setdefault(row["date"], {})[row["market_id"]] = row if row["status"] == "unknown" and row["is_trading_day"] is None: unknown += 1 differences = 0 for date in sorted(by_date): pair = by_date[date] if "SSE" not in pair or "NB" not in pair: continue sse, nb = pair["SSE"], pair["NB"] if (sse["status"], sse["is_trading_day"]) != (nb["status"], nb["is_trading_day"]): print("{}: SSE={} NB={}".format(date, sse["status"], nb["status"])) differences += 1 if differences == 0: print("No SSE/NB differences in this fixture.") print("SSE/NB different dates: {}".format(differences)) print("Unknown rows retained: {}".format(unknown)) print("Data version: {}".format(projection["data_version"])) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--data", required=True, help="Local JSON file or explicit HTTP(S) URL") parser.add_argument("--manifest", required=True, help="Local manifest JSON or explicit HTTP(S) URL") args = parser.parse_args() try: projection = verify(args.data, args.manifest) show_comparison(projection) except (OSError, ValueError, KeyError, json.JSONDecodeError) as error: parser.error(str(error)) if __name__ == "__main__": main()