b3b5f89518 · krz/omaha-metro-blotter

9

# key of its own inside the container. Publishing goes over SSH as the

9

# key of its own inside the container. Publishing goes over SSH as the

10

# blotter-ci account (write on this repository): its private key arrives

10

# blotter-ci account (write on this repository): its private key arrives

11

# as the BOT_SSH_KEY build secret and is written into the workspace for

11

# as the BOT_SSH_KEY build secret and is written into the workspace for

12

# the build. GITBAY_SSH, set by the runner, is the instance as this build

12

# the build, beside an ssh config every ssh and git call is pointed at

13

# reaches it; ~/.ssh/config points ssh and git at the key for that host.

13

# with -F. GITBAY_SSH, set by the runner, is the instance as this build

14

# reaches it. Nothing is written outside the workspace, so the job runs

15

# the same in a container and on a machine that is somebody's own.

14

jobs:

16

jobs:

15

  daily-pull:

17

  daily-pull:

16

    schedule: "17 11,23 * * *"

18

    schedule: "17 11,23 * * *"

22

        test -n "$GITBAY_SSH" || { echo "ERROR: GITBAY_SSH is not set; the runner is too old"; exit 1; }

24

        test -n "$GITBAY_SSH" || { echo "ERROR: GITBAY_SSH is not set; the runner is too old"; exit 1; }

23

        umask 077

25

        umask 077

24

        printf '%s\n' "$BOT_SSH_KEY" > "$PWD/.bot_key"

26

        printf '%s\n' "$BOT_SSH_KEY" > "$PWD/.bot_key"

25

        # ssh expands ~ from the passwd entry, not $HOME; in the build's

27

        printf 'IdentityFile %s\nIdentitiesOnly yes\nStrictHostKeyChecking accept-new\nUserKnownHostsFile %s\n' "$PWD/.bot_key" "$PWD/.known_hosts" > "$PWD/.ssh_config"

26

        # container that is /root while $HOME is the build home.

28

        ssh -F "$PWD/.ssh_config" "$GITBAY_SSH" whoami

27

        sshhome=$(getent passwd "$(id -u)" | cut -d: -f6)

28

        mkdir -p "$sshhome/.ssh"

29

        printf 'Host %s\n  IdentityFile %s\n  IdentitiesOnly yes\n  StrictHostKeyChecking accept-new\n' "${GITBAY_SSH#*@}" "$PWD/.bot_key" > "$sshhome/.ssh/config"

30

        ssh "$GITBAY_SSH" whoami

31

- |

29

- |

32

set -e

30

set -e

33

        export PATH="$PWD/.venv/bin:$PATH" DB=raw_data/metro.db HOST=$GITBAY_SSH R=krz/omaha-metro-blotter

31

        export PATH="$PWD/.venv/bin:$PATH" DB=raw_data/metro.db HOST=$GITBAY_SSH R=krz/omaha-metro-blotter

32

SSH="ssh -F $PWD/.ssh_config"

34

mkdir -p raw_data

33

mkdir -p raw_data

35

34

36

# Restore the published archive; a crashed publish leaves metro.db.new.gz.

35

# Restore the published archive; a crashed publish leaves metro.db.new.gz.

37

        if ssh $HOST release asset get $R archive metro.db.gz > metro.db.gz 2>/dev/null && [ -s metro.db.gz ]; then

36

        if $SSH $HOST release asset get $R archive metro.db.gz > metro.db.gz 2>/dev/null && [ -s metro.db.gz ]; then

38

:

37

:

39

        elif ssh $HOST release asset get $R archive metro.db.new.gz > metro.db.gz 2>/dev/null && [ -s metro.db.gz ]; then

38

        elif $SSH $HOST release asset get $R archive metro.db.new.gz > metro.db.gz 2>/dev/null && [ -s metro.db.gz ]; then

40

echo "recovered from interrupted publish"

39

echo "recovered from interrupted publish"

41

else

40

else

42

          echo "ERROR: no metro.db.gz on release 'archive'; aged-out records cannot be recovered"

41

          echo "ERROR: no metro.db.gz on release 'archive'; aged-out records cannot be recovered"

99

        # point at least one asset holds the full archive.

98

        # point at least one asset holds the full archive.

100

        sqlite3 "$DB" "VACUUM;"

99

        sqlite3 "$DB" "VACUUM;"

101

        gzip -c "$DB" > metro.db.gz

100

        gzip -c "$DB" > metro.db.gz

102

        ssh $HOST release asset remove $R archive metro.db.new.gz 2>/dev/null || true

101

        $SSH $HOST release asset remove $R archive metro.db.new.gz 2>/dev/null || true

103

        ssh $HOST release asset add $R archive metro.db.new.gz < metro.db.gz

102

        $SSH $HOST release asset add $R archive metro.db.new.gz < metro.db.gz

104

        ssh $HOST release asset remove $R archive metro.db.gz 2>/dev/null || true

103

        $SSH $HOST release asset remove $R archive metro.db.gz 2>/dev/null || true

105

        ssh $HOST release asset add $R archive metro.db.gz < metro.db.gz

104

        $SSH $HOST release asset add $R archive metro.db.gz < metro.db.gz

106

        ssh $HOST release asset remove $R archive metro.db.new.gz 2>/dev/null || true

105

        $SSH $HOST release asset remove $R archive metro.db.new.gz 2>/dev/null || true

107

        rm metro.db.gz

106

        rm metro.db.gz

108

{

107

{

109

          echo "SQLite archive of Omaha metro police incident feeds, rebuilt twice daily."

108

          echo "SQLite archive of Omaha metro police incident feeds, rebuilt twice daily."

123

             FROM incidents GROUP BY agency ORDER BY rows DESC"

122

             FROM incidents GROUP BY agency ORDER BY rows DESC"

124

          echo '```'

123

          echo '```'

125

        } > notes.md

124

        } > notes.md

126

        ssh $HOST "release edit $R archive --title 'Incident archive' --file -" < notes.md

125

        $SSH $HOST "release edit $R archive --title 'Incident archive' --file -" < notes.md

127

        rm notes.md

126

        rm notes.md

128

        echo "archive published"

127

        echo "archive published"

129

- |

128

- |

130

set -e

129

set -e

131

export PATH="$PWD/.venv/bin:$PATH" DB=raw_data/metro.db

130

        export PATH="$PWD/.venv/bin:$PATH" DB=raw_data/metro.db GIT_SSH_COMMAND="ssh -F $PWD/.ssh_config"

132

.venv/bin/pip install -q -r requirements.txt

131

.venv/bin/pip install -q -r requirements.txt

133

python build_site.py

132

python build_site.py

134

        origin=ssh://$GITBAY_SSH/krz/omaha-metro-blotter.git

133

        origin=ssh://$GITBAY_SSH/krz/omaha-metro-blotter.git

Commit b3b5f89518

Commit `b3b5f89518`