krz/omaha-metro-blotter

Archive of police activity and ALPR surveillance across the Omaha metro.

clone: git clone https://gitbay.org/krz/omaha-metro-blotter.git

0bd5b692151fe7668420c14b241b8c8932831abc

verified · cmc

author: Christian Cleberg <hello@cleberg.net> · 2026-08-23T19:14:46Z

Add --import-flock to file a portal download correctly

Every Flock export downloads as public_search_audit.csv with the
agency nowhere in the file, so the one manual step that can go wrong
is renaming it. This validates the columns, reports the window it
covers, files it under the right slug and loads it in one command.

Defaults to council-bluffs-ia-pd, the only metro portal offering an
export, and always prints the agency it used — filing one agency's
searches under another would be silent and wrong.

 README.nfo | 17 +++++++++++++----
 ingest.py  | 40 +++++++++++++++++++++++++++++++++++++++-
 2 files changed, 52 insertions(+), 5 deletions(-)

diff --git a/README.nfo b/README.nfo
index a5fc225..c015cd9 100644
--- a/README.nfo
+++ b/README.nfo
@@ -93,10 +93,19 @@ FLOCK SEARCH AUDIT
 
   cloudflare serves a challenge to every non-browser client, so
   ingest.py cannot fetch it. collecting is manual: open the portal,
-  "download csv", save it as raw_data/flock/<slug>_<date>.csv and
-  commit. loading is not manual -- the flock source runs in the daily
-  job and picks up whatever is committed. search ids are stable
-  uuids, so overlapping exports dedupe.
+  click "download csv", then
+
+      .venv/bin/python ingest.py --import-flock ~/Downloads/public_search_audit.csv
+
+  which checks the columns, files it under the right slug and loads
+  it. commit what it writes. every export is named
+  public_search_audit.csv with the agency nowhere inside, so pass
+  --agency for any portal other than council bluffs.
+
+  loading is not manual -- the flock source runs in the daily job and
+  picks up whatever is committed. search ids are stable uuids, so
+  overlapping exports dedupe and re-importing the same window is a
+  no-op.
 
   the portals keep 30 days. miss a month and that month is gone.
 
diff --git a/ingest.py b/ingest.py
index 4144a20..843c164 100644
--- a/ingest.py
+++ b/ingest.py
@@ -326,6 +326,34 @@ def ingest_opd_csv(conn, _since):
 
 
 FLOCK_COLUMNS = ("id", "userId", "searchDate", "networkCount", "reason")
+FLOCK_DIR = ROOT / "raw_data" / "flock"
+# Council Bluffs is the only metro portal that offers the export at all; Sarpy
+# and Douglas publish counts without one.
+FLOCK_DEFAULT_AGENCY = "council-bluffs-ia-pd"
+
+
+def import_flock(path, agency):
+    """File a portal download under the name ingest_flock expects.
+
+    The portal names every export public_search_audit.csv, with the agency
+    nowhere in the file, so the slug has to be supplied and is worth printing:
+    getting it wrong silently files one agency's searches under another."""
+    src = Path(path).expanduser()
+    with src.open(newline="") as fh:
+        reader = csv.DictReader(fh)
+        if tuple(reader.fieldnames or ()) != FLOCK_COLUMNS:
+            raise SystemExit(f"  {src.name}: not a Flock search audit "
+                             f"(columns {reader.fieldnames})")
+        rows = list(reader)
+    if not rows:
+        raise SystemExit(f"  {src.name}: no rows")
+    span = f"{min(r['searchDate'] for r in rows)[:10]} to " \
+           f"{max(r['searchDate'] for r in rows)[:10]}"
+    FLOCK_DIR.mkdir(parents=True, exist_ok=True)
+    dest = FLOCK_DIR / f"{agency}_{datetime.now(LOCAL):%Y-%m-%d}.csv"
+    dest.write_bytes(src.read_bytes())
+    print(f"  {len(rows)} searches, {span}")
+    print(f"  filed as {dest.relative_to(ROOT)} under agency '{agency}'")
 
 
 def ingest_flock(conn, _since):
@@ -429,8 +457,18 @@ def main():
                         "ignored by alpr, flock and opd_csv")
     p.add_argument("--full", action="store_true",
                    help="pull the complete feed instead of --since-days")
+    p.add_argument("--import-flock", metavar="CSV",
+                   help="file a Flock portal download into raw_data/flock and "
+                        "load it")
+    p.add_argument("--agency", default=FLOCK_DEFAULT_AGENCY,
+                   help=f"portal slug for --import-flock (default "
+                        f"{FLOCK_DEFAULT_AGENCY})")
     args = p.parse_args()
-    sources = args.sources or ["opd", "sarpy", "cbpd", "alpr", "flock"]
+    if args.import_flock:
+        import_flock(args.import_flock, args.agency)
+        sources = ["flock"]
+    else:
+        sources = args.sources or ["opd", "sarpy", "cbpd", "alpr", "flock"]
 
     since = None if args.full else datetime.now(timezone.utc) - timedelta(days=args.since_days)