Track new papers across many topics on a schedule

import os, time, requests

API = "https://api.consensus.app/v1/search"
HEADERS = {"x-api-key": os.environ["CONSENSUS_API_KEY"]}

STRATEGY = dict(
    study_types=["rct", "meta-analysis", "systematic review"],
    exclude_preprints=False,   # early signals often appear as preprints first
    page_size=200,
)

def sweep(query, since):
    """Every record for one query since the last run."""
    page, out = 0, []
    while page <= 49:                         # 0-indexed; 49 is the ceiling
        resp = requests.get(API, headers=HEADERS, params={
            "query": query,
            "year_min": since.year,
            "month_min": since.month,
            "page": page,
            **STRATEGY,
        })
        if resp.status_code == 429:           # rate limited — back off and retry
            time.sleep(int(resp.headers.get("retry-after", 30)))
            continue
        resp.raise_for_status()
        body = resp.json()
        out.extend(body["results"])
        if body["is_end"] or body.get("next_page") is None:
            break
        page = body["next_page"]
    return out

def run(entities, template, since, seen_dois):
    new, counts = {}, {}
    for entity in entities:
        results = sweep(template.format(entity=entity), since)
        counts[entity] = len(results)
        for paper in results:
            doi = paper.get("doi")
            if doi and doi not in seen_dois:
                new[doi] = {**paper, "entity": entity}
    return new, counts        # persist counts even when new is empty