ingest.py
20def fetch_raw(url: str) -> list[Any]:
23 response = requests.get(url, timeout=REQUEST_TIMEOUT_SECONDS)
26 # Some sources wrap rows in {"data": [...]}, others return a list.
27 records = payload.get("data", payload) if isinstance(payload, dict) else payload
28
34def parse_records(records: list[Any]) -> tuple[list[Posting], int]:
45 parsed.append(Posting.model_validate(record))
46 except ValidationError as exc:
47 rejected += 1
pipeline.py
38SOURCE_NAME = "postings" # team A often "events" — not an .env key
73 source_api_url=required("SOURCE_API_URL"),
78 landing_container=os.getenv("LANDING_CONTAINER", PRODUCTION_CONTAINER),
79 landing_prefix=os.getenv("LANDING_PREFIX", PRODUCTION_PREFIX),
92 records = fetch_raw(config.source_api_url)
93 parsed, rejected = parse_records(records)
108 path = blob_path(SOURCE_NAME, run_date, config.landing_prefix)
120 landed = land_raw_json(…, records=records, container=config.landing_container)