.gitbay/ci.yml

 1# Twice-daily archive pull, ported from the GitHub workflow. Sarpy and
 2# Council Bluffs serve a rolling 12-month window; records that age out of
 3# those feeds exist nowhere else. This job is the only thing keeping them,
 4# so it refuses to publish an archive smaller than the one it started with.
 5# The archive lives as metro.db.gz on the gitbay release tagged "archive";
 6# the site deploys to the pages branch.
 7#
 8# The build runs in a container on the instance's runner, which holds no
 9# key of its own inside the container. Publishing goes over SSH as the
10# blotter-ci account (write on this repository): its private key arrives
11# as the BOT_SSH_KEY build secret and is written into the workspace for
12# the build, beside an ssh config every ssh and git call is pointed at
13# with -F. GITBAY_SSH, set by the runner, is the instance as this build
14# reaches it. Nothing is written outside the workspace, so the job runs
15# the same in a container and on a machine that is somebody's own.
16jobs:
17  daily-pull:
18    schedule: "17 11,23 * * *"
19    steps:
20      - python3 -m venv .venv && .venv/bin/pip install -q -r requirements-ingest.txt
21      - |
22        set -e
23        test -n "$BOT_SSH_KEY" || { echo "ERROR: BOT_SSH_KEY secret is not set"; exit 1; }
24        test -n "$GITBAY_SSH" || { echo "ERROR: GITBAY_SSH is not set; the runner is too old"; exit 1; }
25        umask 077
26        printf '%s\n' "$BOT_SSH_KEY" > "$PWD/.bot_key"
27        printf 'IdentityFile %s\nIdentitiesOnly yes\nStrictHostKeyChecking accept-new\nUserKnownHostsFile %s\n' "$PWD/.bot_key" "$PWD/.known_hosts" > "$PWD/.ssh_config"
28        ssh -F "$PWD/.ssh_config" "$GITBAY_SSH" whoami
29      - sh ci/pull-publish.sh
30      - |
31        set -e
32        export PATH="$PWD/.venv/bin:$PATH" DB=raw_data/metro.db GIT_SSH_COMMAND="ssh -F $PWD/.ssh_config"
33        .venv/bin/pip install -q -r requirements.txt
34        python build_site.py
35        origin=ssh://$GITBAY_SSH/krz/omaha-metro-blotter.git
36        cd site && git init -q -b pages
37        git -c user.name=ci -c user.email=ci@gitbay.org add -A
38        git -c user.name=ci -c user.email=ci@gitbay.org commit -q -m "site $(date -u '+%Y-%m-%d %H:%M')"
39        git push -qf "$origin" pages:pages
40        echo "site deployed"
41      - |
42        set -e
43        export DB=raw_data/metro.db
44        # Alarms last, on purpose: a stale feed should fail the build loudly
45        # without having blocked the archive or the site.
46        sqlite3 -noheader "$DB" \
47          "SELECT source || ' ' || MAX(occurred_at) FROM incidents
48           WHERE source IN ('opd', 'sarpy', 'cbpd') GROUP BY source
49           HAVING MAX(occurred_at) < datetime('now', '-7 days')" > stale.txt
50        if [ -s stale.txt ]; then
51          while read -r line; do echo "ERROR: feed is stale: $line"; done < stale.txt
52          exit 1
53        fi
54        echo "all feeds current"
55        age=$(sqlite3 -noheader "$DB" "
56          SELECT CAST(julianday('now') - julianday(MAX(imported_at)) AS INT)
57            FROM alpr_searches" 2>/dev/null || echo "")
58        if [ -z "$age" ]; then echo "no Flock export on file yet"; exit 0; fi
59        echo "newest Flock export imported $age days ago"
60        if [ "$age" -ge 27 ]; then
61          echo "ERROR: Flock search audit is $age days old and the portal only keeps 30."
62          echo "Download it from https://transparency.flocksafety.com/council-bluffs-ia-pd"
63          echo "and commit it to raw_data/flock/ before the window closes."
64          exit 1
65        elif [ "$age" -ge 21 ]; then
66          echo "WARNING: Flock search audit is $age days old; refresh it soon."
67        fi