ci: ssh -F with a config in the workspace !6
1 file changed, +16 −17
Layout: unified · split
.gitbay/ci.yml +16 −17
| @@ -9,8 +9,10 @@ | ||
| 9 | 9 | # key of its own inside the container. Publishing goes over SSH as the |
| 10 | 10 | # blotter-ci account (write on this repository): its private key arrives |
| 11 | 11 | # as the BOT_SSH_KEY build secret and is written into the workspace for |
| 12 | # the build. GITBAY_SSH, set by the runner, is the instance as this build | |
| 13 | # reaches it; ~/.ssh/config points ssh and git at the key for that host. | |
| 12 | # the build, beside an ssh config every ssh and git call is pointed at | |
| 13 | # with -F. GITBAY_SSH, set by the runner, is the instance as this build | |
| 14 | # reaches it. Nothing is written outside the workspace, so the job runs | |
| 15 | # the same in a container and on a machine that is somebody's own. | |
| 14 | 16 | jobs: |
| 15 | 17 | daily-pull: |
| 16 | 18 | schedule: "17 11,23 * * *" |
| @@ -22,21 +24,18 @@ jobs: | ||
| 22 | 24 | test -n "$GITBAY_SSH" || { echo "ERROR: GITBAY_SSH is not set; the runner is too old"; exit 1; } |
| 23 | 25 | umask 077 |
| 24 | 26 | printf '%s\n' "$BOT_SSH_KEY" > "$PWD/.bot_key" |
| 25 | # ssh expands ~ from the passwd entry, not $HOME; in the build's | |
| 26 | # container that is /root while $HOME is the build home. | |
| 27 | sshhome=$(getent passwd "$(id -u)" | cut -d: -f6) | |
| 28 | mkdir -p "$sshhome/.ssh" | |
| 29 | printf 'Host %s\n IdentityFile %s\n IdentitiesOnly yes\n StrictHostKeyChecking accept-new\n' "${GITBAY_SSH#*@}" "$PWD/.bot_key" > "$sshhome/.ssh/config" | |
| 30 | ssh "$GITBAY_SSH" whoami | |
| 27 | printf 'IdentityFile %s\nIdentitiesOnly yes\nStrictHostKeyChecking accept-new\nUserKnownHostsFile %s\n' "$PWD/.bot_key" "$PWD/.known_hosts" > "$PWD/.ssh_config" | |
| 28 | ssh -F "$PWD/.ssh_config" "$GITBAY_SSH" whoami | |
| 31 | 29 | - | |
| 32 | 30 | set -e |
| 33 | 31 | export PATH="$PWD/.venv/bin:$PATH" DB=raw_data/metro.db HOST=$GITBAY_SSH R=krz/omaha-metro-blotter |
| 32 | SSH="ssh -F $PWD/.ssh_config" | |
| 34 | 33 | mkdir -p raw_data |
| 35 | 34 | |
| 36 | 35 | # Restore the published archive; a crashed publish leaves metro.db.new.gz. |
| 37 | if ssh $HOST release asset get $R archive metro.db.gz > metro.db.gz 2>/dev/null && [ -s metro.db.gz ]; then | |
| 36 | if $SSH $HOST release asset get $R archive metro.db.gz > metro.db.gz 2>/dev/null && [ -s metro.db.gz ]; then | |
| 38 | 37 | : |
| 39 | elif ssh $HOST release asset get $R archive metro.db.new.gz > metro.db.gz 2>/dev/null && [ -s metro.db.gz ]; then | |
| 38 | elif $SSH $HOST release asset get $R archive metro.db.new.gz > metro.db.gz 2>/dev/null && [ -s metro.db.gz ]; then | |
| 40 | 39 | echo "recovered from interrupted publish" |
| 41 | 40 | else |
| 42 | 41 | echo "ERROR: no metro.db.gz on release 'archive'; aged-out records cannot be recovered" |
| @@ -99,11 +98,11 @@ jobs: | ||
| 99 | 98 | # point at least one asset holds the full archive. |
| 100 | 99 | sqlite3 "$DB" "VACUUM;" |
| 101 | 100 | gzip -c "$DB" > metro.db.gz |
| 102 | ssh $HOST release asset remove $R archive metro.db.new.gz 2>/dev/null || true | |
| 103 | ssh $HOST release asset add $R archive metro.db.new.gz < metro.db.gz | |
| 104 | ssh $HOST release asset remove $R archive metro.db.gz 2>/dev/null || true | |
| 105 | ssh $HOST release asset add $R archive metro.db.gz < metro.db.gz | |
| 106 | ssh $HOST release asset remove $R archive metro.db.new.gz 2>/dev/null || true | |
| 101 | $SSH $HOST release asset remove $R archive metro.db.new.gz 2>/dev/null || true | |
| 102 | $SSH $HOST release asset add $R archive metro.db.new.gz < metro.db.gz | |
| 103 | $SSH $HOST release asset remove $R archive metro.db.gz 2>/dev/null || true | |
| 104 | $SSH $HOST release asset add $R archive metro.db.gz < metro.db.gz | |
| 105 | $SSH $HOST release asset remove $R archive metro.db.new.gz 2>/dev/null || true | |
| 107 | 106 | rm metro.db.gz |
| 108 | 107 | { |
| 109 | 108 | echo "SQLite archive of Omaha metro police incident feeds, rebuilt twice daily." |
| @@ -123,12 +122,12 @@ jobs: | ||
| 123 | 122 | FROM incidents GROUP BY agency ORDER BY rows DESC" |
| 124 | 123 | echo '```' |
| 125 | 124 | } > notes.md |
| 126 | ssh $HOST "release edit $R archive --title 'Incident archive' --file -" < notes.md | |
| 125 | $SSH $HOST "release edit $R archive --title 'Incident archive' --file -" < notes.md | |
| 127 | 126 | rm notes.md |
| 128 | 127 | echo "archive published" |
| 129 | 128 | - | |
| 130 | 129 | set -e |
| 131 | export PATH="$PWD/.venv/bin:$PATH" DB=raw_data/metro.db | |
| 130 | export PATH="$PWD/.venv/bin:$PATH" DB=raw_data/metro.db GIT_SSH_COMMAND="ssh -F $PWD/.ssh_config" | |
| 132 | 131 | .venv/bin/pip install -q -r requirements.txt |
| 133 | 132 | python build_site.py |
| 134 | 133 | origin=ssh://$GITBAY_SSH/krz/omaha-metro-blotter.git |