krz/omaha-metro-blotter
Archive of police activity and ALPR surveillance across the Omaha metro.
clone: git clone https://gitbay.org/krz/omaha-metro-blotter.git
63f759d70a521c6c61c93e92ea76ede18adff096
verified · cmc
author: Christian Cleberg <hello@cleberg.net> · 2026-08-23T07:20:38Z
.github/workflows/daily-pull.yml | 28 ++++++++++ README.nfo | 20 ++++++++ analysis.py | 48 +++++++++++++++--- app.py | 1 + ingest.py | 107 +++++++++++++++++++++++++++++++-------- schema.sql | 58 ++++++++++++++++++++- 6 files changed, 232 insertions(+), 30 deletions(-) @@ -69,6 +69,10 @@ jobs: sqlite3 -noheader -separator ' ' "$DB" \ "SELECT source, COUNT(*) FROM incidents GROUP BY source ORDER BY source" \ > before.txt + # absent until the amendment migration has run against this archive + sqlite3 -noheader -separator ' ' "$DB" \ + "SELECT source || '+amend', COUNT(*) FROM incident_amendments + GROUP BY source ORDER BY source" >> before.txt 2>/dev/null || true fi cat before.txt @@ -85,6 +89,9 @@ jobs: sqlite3 -noheader -separator ' ' "$DB" \ "SELECT source, COUNT(*) FROM incidents GROUP BY source ORDER BY source" \ > after.txt + sqlite3 -noheader -separator ' ' "$DB" \ + "SELECT source || '+amend', COUNT(*) FROM incident_amendments + GROUP BY source ORDER BY source" >> after.txt cat after.txt test -s after.txt || { echo "::error::archive is empty"; exit 1; } # Keyed on FILENAME, not NR == FNR: before.txt is empty on a bootstrap @@ -101,6 +108,22 @@ jobs: exit bad }' before.txt after.txt + # An amendment identical to its original means the digest drifted + # against what SQLite stores, and every run would file the same + # phantom again. Cheap to check, silent and cumulative if it happens. + phantom=$(sqlite3 -noheader "$DB" " + SELECT COUNT(*) FROM incident_amendments a + JOIN incidents o ON o.source = a.source AND o.source_key = a.source_key + WHERE a.agency IS o.agency AND a.case_id IS o.case_id + AND a.occurred_at IS o.occurred_at AND a.category IS o.category + AND a.call_type IS o.call_type AND a.disposition IS o.disposition + AND a.offense_desc IS o.offense_desc AND a.is_stop IS o.is_stop + AND a.address IS o.address AND a.lat IS o.lat AND a.lon IS o.lon") + if [ "$phantom" -ne 0 ]; then + echo "::error::$phantom amendments are identical to their original" + exit 1 + fi + - name: Publish archive run: | sqlite3 "$DB" "VACUUM;" @@ -115,6 +138,11 @@ jobs: echo echo "Updated $(date -u '+%Y-%m-%d %H:%M UTC'). Schema: schema.sql." echo + echo "incidents holds each record as first published; every later" + echo "version the feed served is a row in incident_amendments" + echo "($(sqlite3 -noheader "$DB" 'SELECT COUNT(*) FROM incident_amendments') so far)." + echo "incidents_current is the newest version of each." + echo echo '```' sqlite3 -header -column "$DB" \ "SELECT agency, COUNT(*) AS rows, SUM(is_stop) AS stops, @@ -61,6 +61,26 @@ ARCHIVE 0 6 * * * cd /path/to/omaha-incidents && .venv/bin/python ingest.py +AMENDMENTS + agencies edit records after publishing them: a disposition changes, + a case reopens, a record is withdrawn. nothing in the incidents + table is ever updated, so what an agency published first stays + readable. every later version the feed serves lands in + incident_amendments, and incidents_current is the newest version of + each record. analysis.changed_stop_outcomes() lists stops whose + disposition changed after filing. + + a version is keyed on the hash of its payload, so a record that + reverts to a payload already on file is not recorded again. this + holds the set of distinct states observed, not a strict timeline. + + the hash has to survive a round trip through sqlite. a lon of -96 + arrives from the feed as a json int and comes back out of a REAL + column as -96.0, so lat, lon and is_stop are coerced before + hashing. get this wrong and every affected record is filed as + amended on every run, forever. the workflow fails if any amendment + is byte-identical to its original. + NOTES all three arcgis services return utc epochs; their where-clause literals do not agree (opd and council bluffs utc, sarpy central). @@ -1,4 +1,8 @@ -"""Queries behind the dashboard: agency activity and distance to the nearest ALPR.""" +"""Queries behind the dashboard: agency activity and distance to the nearest ALPR. + +Reads incidents_current, the newest known version of each record. The originals +stay in incidents and every superseded version in incident_amendments, so a +disposition an agency changed after the fact is still recoverable.""" import sqlite3 from pathlib import Path @@ -36,8 +40,8 @@ def load_incidents(conn, agencies=None, start=None, end=None, categories=None): params.append(f"{end}T23:59:59") df = pd.read_sql_query( f"""SELECT source, agency, case_id, occurred_at, category, call_type, - disposition, offense_desc, is_stop, address, lat, lon - FROM incidents WHERE {' AND '.join(where)}""", + disposition, offense_desc, is_stop, address, lat, lon, amended + FROM incidents_current WHERE {' AND '.join(where)}""", conn, params=params) df["occurred_at"] = pd.to_datetime(df["occurred_at"]) return df @@ -126,21 +130,49 @@ def camera_proximity(df, cameras, bin_m=200, max_m=2000): return g +def amendment_history(conn, source, source_key): + """Every version of one record, oldest first.""" + return pd.read_sql_query( + """SELECT 'original' AS version, occurred_at, category, call_type, + disposition, is_stop, address, first_seen AS seen_at + FROM incidents WHERE source = ? AND source_key = ? + UNION ALL + SELECT 'amended', occurred_at, category, call_type, + disposition, is_stop, address, seen_at + FROM incident_amendments WHERE source = ? AND source_key = ? + ORDER BY seen_at""", + conn, params=[source, source_key, source, source_key]) + + +def changed_stop_outcomes(conn): + """Stops whose disposition the agency changed after first publishing it.""" + return pd.read_sql_query( + """SELECT o.agency, o.case_id, o.occurred_at, + o.disposition AS first_published, + a.disposition AS later_published, a.seen_at + FROM incident_amendments a + JOIN incidents o + ON o.source = a.source AND o.source_key = a.source_key + WHERE o.is_stop = 1 + AND IFNULL(a.disposition, '') <> IFNULL(o.disposition, '') + ORDER BY a.seen_at DESC""", conn) + + def agency_options(conn): rows = conn.execute( - "SELECT agency, COUNT(*) FROM incidents GROUP BY agency ORDER BY 2 DESC" - ).fetchall() + "SELECT agency, COUNT(*) FROM incidents_current GROUP BY agency" + " ORDER BY 2 DESC").fetchall() return [a for a, _ in rows] def category_options(conn): rows = conn.execute( - "SELECT category, COUNT(*) FROM incidents WHERE category IS NOT NULL" - " GROUP BY category ORDER BY 2 DESC").fetchall() + "SELECT category, COUNT(*) FROM incidents_current" + " WHERE category IS NOT NULL GROUP BY category ORDER BY 2 DESC").fetchall() return [c for c, _ in rows] def date_bounds(conn): lo, hi = conn.execute( - "SELECT MIN(occurred_at), MAX(occurred_at) FROM incidents").fetchone() + "SELECT MIN(occurred_at), MAX(occurred_at) FROM incidents_current").fetchone() return lo[:10], hi[:10] @@ -131,6 +131,7 @@ def update_kpis(agencies, categories, start, end, _theme): ("Vehicle stops", f"{len(stops):,}"), ("Stops per day", rate), ("Agencies", f"{df['agency'].nunique()}"), + ("Amended since filing", f"{int(df['amended'].sum()):,}"), ] return [html.Div(className="kpi", children=[html.Span(v, className="kpi-value"), html.Span(k, className="kpi-label")]) @@ -18,6 +18,7 @@ carries its own literal timezone and page size. import argparse import csv +import hashlib import json import sqlite3 import ssl @@ -158,20 +159,67 @@ def local_iso(epoch_ms): return dt.astimezone(LOCAL).strftime("%Y-%m-%dT%H:%M:%S") +# The order every row tuple is built in, and the order digest() hashes. +COLUMNS = ("source", "source_key", "agency", "case_id", "occurred_at", "category", + "call_type", "disposition", "offense_desc", "is_stop", "address", + "lat", "lon") +PAYLOAD = COLUMNS[2:] # everything the feed can change; the first two are the key + + +# Columns whose SQLite affinity rewrites what the feed sent: a JSON lon of -96 +# arrives as int and comes back out of a REAL column as -96.0. Hashing the raw +# value would mark such a record amended on every run forever, so coerce to the +# stored representation first. +REAL_FIELDS = {"lat", "lon"} +INT_FIELDS = {"is_stop"} + + +def digest(values): + """Hash of a record's payload. Feeds amend records after publishing them, so + this is what tells an unchanged record from a genuinely new version. Must + give the same answer for a value going into the database and coming back.""" + parts = [] + for name, v in zip(PAYLOAD, values): + if v is None: + parts.append("") + elif name in REAL_FIELDS: + parts.append(repr(float(v))) + elif name in INT_FIELDS: + parts.append(str(int(v))) + else: + parts.append(str(v)) + return hashlib.blake2b("\x1f".join(parts).encode(), digest_size=8).hexdigest() + + def upsert(conn, rows): - conn.executemany( - """INSERT INTO incidents - (source, source_key, agency, case_id, occurred_at, category, - call_type, disposition, offense_desc, is_stop, address, lat, lon) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?) - ON CONFLICT(source, source_key) DO UPDATE SET - agency=excluded.agency, case_id=excluded.case_id, - occurred_at=excluded.occurred_at, category=excluded.category, - call_type=excluded.call_type, disposition=excluded.disposition, - offense_desc=excluded.offense_desc, is_stop=excluded.is_stop, - address=excluded.address, lat=excluded.lat, lon=excluded.lon""", - rows) - return len(rows) + """Insert records not seen before; file a changed record as an amendment. + + Nothing in incidents is ever updated. A record whose payload differs from + the one on file is appended to incident_amendments, so the version the + agency published first stays readable next to what it published later.""" + now = datetime.now(LOCAL).strftime("%Y-%m-%dT%H:%M:%S") + marks = ",".join("?" * (len(COLUMNS) + 1)) + staged = [r + (digest(r[2:]),) for r in rows] + + conn.execute("DROP TABLE IF EXISTS temp.incoming") + conn.execute(f"CREATE TEMP TABLE incoming ({','.join(COLUMNS)}, digest)") + conn.executemany(f"INSERT INTO temp.incoming VALUES ({marks})", staged) + conn.execute("CREATE INDEX temp.incoming_key ON incoming (source, source_key)") + + cols = ",".join(COLUMNS) + conn.execute( + f"""INSERT OR IGNORE INTO incidents ({cols}, digest, first_seen) + SELECT {cols}, digest, ? FROM temp.incoming""", (now,)) + amended = conn.execute( + f"""INSERT OR IGNORE INTO incident_amendments ({cols}, digest, seen_at) + SELECT i.{', i.'.join(COLUMNS)}, i.digest, ? + FROM temp.incoming i + JOIN incidents o + ON o.source = i.source AND o.source_key = i.source_key + WHERE o.digest <> i.digest""", (now,)).rowcount + + conn.execute("DROP TABLE temp.incoming") + return len(rows), amended def ingest_opd(conn, since): @@ -206,10 +254,10 @@ def ingest_sarpy(conn, since): a.get("StatuteDesc"), int(a.get("Category") == SARPY_STOP_CATEGORY), a.get("BlkAddress"), g.get("y"), g.get("x"))) - n = upsert(conn, rows) + result = upsert(conn, rows) if unmapped: print(f" unmapped IncidentId prefixes: {sorted(unmapped)}") - return n + return result def ingest_cbpd(conn, since): @@ -278,20 +326,38 @@ def ingest_alpr(conn, _since): direction=excluded.direction, last_seen=excluded.last_seen, tags=excluded.tags""", rows) - return len(rows) + return len(rows), 0 def migrate(conn): """Add columns introduced after a database was first built. Runs before - schema.sql so its indexes can reference the new columns.""" + schema.sql so its views and indexes can reference the new columns.""" have = {r[1] for r in conn.execute("PRAGMA table_info(incidents)")} - if have and "is_stop" not in have: + if not have: + return + + if "is_stop" not in have: conn.execute("ALTER TABLE incidents ADD COLUMN is_stop INTEGER NOT NULL" " DEFAULT 0") conn.execute("UPDATE incidents SET is_stop = 1 WHERE category = ?", (SARPY_STOP_CATEGORY,)) conn.commit() + if "digest" not in have: + # Backfill from the stored values, using the same function ingest uses, + # so the next run sees the existing rows as unchanged rather than + # amending all of them. + conn.execute("ALTER TABLE incidents ADD COLUMN digest TEXT") + conn.execute("ALTER TABLE incidents ADD COLUMN first_seen TEXT") + payload = ", ".join(PAYLOAD) + rows = conn.execute( + f"SELECT source, source_key, {payload} FROM incidents").fetchall() + conn.executemany( + "UPDATE incidents SET digest = ? WHERE source = ? AND source_key = ?", + [(digest(r[2:]), r[0], r[1]) for r in rows]) + conn.commit() + print(f" migrated: digested {len(rows)} existing rows") + SOURCES = { "opd": ingest_opd, @@ -324,9 +390,10 @@ def main(): for name in sources: start = time.monotonic() print(f" {name}: pulling...") - n = SOURCES[name](conn, since) + seen, amended = SOURCES[name](conn, since) conn.commit() - print(f" {name}: {n} rows in {time.monotonic() - start:.1f}s") + note = f", {amended} amended" if amended else "" + print(f" {name}: {seen} rows{note} in {time.monotonic() - start:.1f}s") conn.close() @@ -1,5 +1,8 @@ +-- Incidents as first observed. Rows here are never updated: when a feed serves a +-- changed version of a record it goes to incident_amendments instead, so the +-- original survives a reclassification, a reopened case or a withdrawn record. +-- occurred_at is local time (America/Chicago); the services return UTC epochs +-- and ingest.py converts on the way in. CREATE TABLE IF NOT EXISTS incidents ( source TEXT NOT NULL, -- opd | sarpy | cbpd | opd_csv source_key TEXT NOT NULL, -- PK (opd) | IncidentId (sarpy) | cfs_number (cbpd) @@ -14,6 +17,8 @@ CREATE TABLE IF NOT EXISTS incidents ( address TEXT, lat REAL, lon REAL, + digest TEXT NOT NULL, -- hash of the payload, for change detection + first_seen TEXT, -- when ingest first saw it; NULL if pre-dating PRIMARY KEY (source, source_key) ); @@ -22,6 +27,55 @@ CREATE INDEX IF NOT EXISTS incidents_agency ON incidents (agency, occurred_at) CREATE INDEX IF NOT EXISTS incidents_category ON incidents (category); CREATE INDEX IF NOT EXISTS incidents_stop ON incidents (is_stop, occurred_at); +-- Every distinct later version of a record, one row per version. Keyed on the +-- payload digest, so a version is stored once no matter how many runs serve it. +-- A record that reverts to a payload already on file is therefore not recorded +-- again: this holds the set of distinct states observed, not a strict timeline. +CREATE TABLE IF NOT EXISTS incident_amendments ( + source TEXT NOT NULL, + source_key TEXT NOT NULL, + agency TEXT NOT NULL, + case_id TEXT, + occurred_at TEXT NOT NULL, + category TEXT, + call_type TEXT, + disposition TEXT, + offense_desc TEXT, + is_stop INTEGER NOT NULL DEFAULT 0, + address TEXT, + lat REAL, + lon REAL, + digest TEXT NOT NULL, + seen_at TEXT NOT NULL, -- when this version was first observed + PRIMARY KEY (source, source_key, digest) +); + +CREATE INDEX IF NOT EXISTS amendments_key ON incident_amendments (source, source_key); +CREATE INDEX IF NOT EXISTS amendments_seen ON incident_amendments (seen_at); + +-- The newest known version of each record: its latest amendment, or the +-- original where a record has never been amended. +CREATE VIEW IF NOT EXISTS incidents_current AS +SELECT source, source_key, agency, case_id, occurred_at, category, call_type, + disposition, offense_desc, is_stop, address, lat, lon, observed_at, + amended +FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY source, source_key + ORDER BY amended DESC, observed_at DESC) AS rn + FROM ( + SELECT source, source_key, agency, case_id, occurred_at, category, + call_type, disposition, offense_desc, is_stop, address, lat, lon, + first_seen AS observed_at, 0 AS amended + FROM incidents + UNION ALL + SELECT source, source_key, agency, case_id, occurred_at, category, + call_type, disposition, offense_desc, is_stop, address, lat, lon, + seen_at AS observed_at, 1 AS amended + FROM incident_amendments + ) +) +WHERE rn = 1; + -- ALPR cameras from OpenStreetMap (ODbL). first_seen/last_seen track when a node -- entered and was last present in the Overpass result, so cameras that appear or -- are removed are visible over time.