Commit acbff854f2
Verified · cmc
Layout: unified · split
.env.example added +14
| @@ -0,0 +1,14 @@ | ||
| 1 | API_KEY=replace-me | |
| 2 | ENABLE_SCHEDULER=false | |
| 3 | SRHT_TOKEN=replace-me | |
| 4 | TODO_SRHT_ENDPOINT=https://todo.sr.ht/query | |
| 5 | GIT_SRHT_ENDPOINT=https://git.sr.ht/query | |
| 6 | DATABASE_URL=sqlite:///./srht_contrib.db | |
| 7 | DEFAULT_ACTOR=~ccleberg | |
| 8 | POLL_INTERVAL_SECONDS=900 | |
| 9 | # Optional JSON object. Example: | |
| 10 | # {"~ccleberg":["cmc@example.com","Chris Cleberg"]} | |
| 11 | ACTOR_ALIASES_JSON={} | |
| 12 | # Optional JSON array. Example: | |
| 13 | # ["Hutch","~ccleberg/cleberg.net"] | |
| 14 | GIT_TRACKED_REPOSITORIES=[] | |
.gitignore added +31
| @@ -0,0 +1,31 @@ | ||
| 1 | .env | |
| 2 | .env.* | |
| 3 | !.env.example | |
| 4 | ||
| 5 | .venv/ | |
| 6 | venv/ | |
| 7 | ||
| 8 | __pycache__/ | |
| 9 | *.py[cod] | |
| 10 | *.so | |
| 11 | ||
| 12 | .pytest_cache/ | |
| 13 | .mypy_cache/ | |
| 14 | .ruff_cache/ | |
| 15 | .hypothesis/ | |
| 16 | .coverage | |
| 17 | .coverage.* | |
| 18 | htmlcov/ | |
| 19 | ||
| 20 | build/ | |
| 21 | dist/ | |
| 22 | *.egg-info/ | |
| 23 | .eggs/ | |
| 24 | ||
| 25 | *.db | |
| 26 | *.sqlite | |
| 27 | *.sqlite3 | |
| 28 | ||
| 29 | .DS_Store | |
| 30 | .idea/ | |
| 31 | .vscode/ | |
README.md added +325
| @@ -0,0 +1,325 @@ | ||
| 1 | # srht-contrib | |
| 2 | ||
| 3 | `srht-contrib` is a small Python service that polls SourceHut activity, normalizes it into one internal event model, stores it in SQLite, and exposes a contribution-calendar JSON API that an iOS app can render directly. | |
| 4 | ||
| 5 | The current V1 is intentionally narrow and production-oriented: | |
| 6 | ||
| 7 | - FastAPI JSON API only | |
| 8 | - SQLite-backed persistence | |
| 9 | - polling-based ingestion | |
| 10 | - complete `todo.sr.ht` ingestion path | |
| 11 | - practical `git.sr.ht` commit ingestion for tracked repositories | |
| 12 | - API-key protection for `/api/*` | |
| 13 | - Alembic-managed schema migrations | |
| 14 | ||
| 15 | ## What It Does | |
| 16 | ||
| 17 | The service collects SourceHut activity from one or more sr.ht GraphQL services, turns those records into a canonical event shape, aggregates activity by day, and returns zero-filled calendar ranges so the client never has to patch missing dates. | |
| 18 | ||
| 19 | Example use cases: | |
| 20 | ||
| 21 | - render a GitHub-style contribution grid in an iOS app | |
| 22 | - show total score and streak stats for a SourceHut user | |
| 23 | - poll recent activity on a schedule or trigger polling manually | |
| 24 | ||
| 25 | ## Architecture Overview | |
| 26 | ||
| 27 | The code is split into small, testable layers: | |
| 28 | ||
| 29 | - `src/srht_contrib/config.py`: environment-driven settings and event weights | |
| 30 | - `src/srht_contrib/db.py`: SQLAlchemy engine/session setup and app-scoped DB access | |
| 31 | - `src/srht_contrib/models.py`: ORM models for normalized events, sync state, aliases, and tracked repos | |
| 32 | - `src/srht_contrib/services/srht_client.py`: generic SourceHut GraphQL client with error handling and simple retries | |
| 33 | - `src/srht_contrib/services/todo.py`: `todo.sr.ht` ingestion and normalization | |
| 34 | - `src/srht_contrib/services/git.py`: `git.sr.ht` tracked-repository commit ingestion | |
| 35 | - `src/srht_contrib/services/aggregator.py`: per-day aggregation and streak/stat calculations | |
| 36 | - `src/srht_contrib/jobs/poller.py`: repeated-safe polling and idempotent persistence | |
| 37 | - `src/srht_contrib/api/`: FastAPI routes, auth dependencies, and repository management | |
| 38 | - `alembic/`: schema migration environment and versioned migrations | |
| 39 | ||
| 40 | ## Supported sr.ht Services | |
| 41 | ||
| 42 | ### Implemented | |
| 43 | ||
| 44 | - `todo.sr.ht` | |
| 45 | - `git.sr.ht` | |
| 46 | ||
| 47 | Current normalized event types: | |
| 48 | ||
| 49 | - `ticket_created` | |
| 50 | - `ticket_comment` | |
| 51 | - `ticket_closed` | |
| 52 | - `commit` | |
| 53 | ||
| 54 | `todo.sr.ht` uses a feed-first strategy and falls back to crawling the authenticated user’s trackers, tickets, and ticket events when the top-level activity feed is empty. `git.sr.ht` polls tracked repositories for recent commits on the default branch. | |
| 55 | ||
| 56 | ## Canonical Event Model | |
| 57 | ||
| 58 | All ingestion services normalize external activity into this shape: | |
| 59 | ||
| 60 | - `service` | |
| 61 | - `event_type` | |
| 62 | - `actor` | |
| 63 | - `repo_name` | |
| 64 | - `resource_id` | |
| 65 | - `external_uid` | |
| 66 | - `occurred_at` | |
| 67 | - `weight` | |
| 68 | - `raw_payload_json` | |
| 69 | ||
| 70 | The database enforces uniqueness on `(service, external_uid)` so polling is safe to repeat. | |
| 71 | ||
| 72 | Tracked git repositories are persisted in the `tracked_repositories` table and stored in canonical `~owner/repo` form. The poller seeds that table from `GIT_TRACKED_REPOSITORIES`, and repositories can also be created, updated, and deleted through the API. | |
| 73 | ||
| 74 | ## Configuration | |
| 75 | ||
| 76 | Environment variables: | |
| 77 | ||
| 78 | - `API_KEY`: required header token for all `/api/*` routes via `X-API-Key` | |
| 79 | - `ENABLE_SCHEDULER`: defaults to `false`; enables in-process polling when set to `true` | |
| 80 | - `SRHT_TOKEN`: bearer token for SourceHut GraphQL | |
| 81 | - `TODO_SRHT_ENDPOINT`: defaults to `https://todo.sr.ht/query` | |
| 82 | - `GIT_SRHT_ENDPOINT`: defaults to `https://git.sr.ht/query` | |
| 83 | - `DATABASE_URL`: defaults to `sqlite:///./srht_contrib.db` | |
| 84 | - `DEFAULT_ACTOR`: actor used by the scheduled poll job | |
| 85 | - `POLL_INTERVAL_SECONDS`: scheduler interval in seconds | |
| 86 | - `ACTOR_ALIASES_JSON`: optional JSON object for actor/email/display-name alias mapping | |
| 87 | - `GIT_TRACKED_REPOSITORIES`: optional JSON array of repository names or `owner/repo` strings for git polling | |
| 88 | ||
| 89 | Example `.env`: | |
| 90 | ||
| 91 | ```env | |
| 92 | API_KEY=replace-me | |
| 93 | ENABLE_SCHEDULER=false | |
| 94 | SRHT_TOKEN=replace-me | |
| 95 | TODO_SRHT_ENDPOINT=https://todo.sr.ht/query | |
| 96 | GIT_SRHT_ENDPOINT=https://git.sr.ht/query | |
| 97 | DATABASE_URL=sqlite:///./srht_contrib.db | |
| 98 | DEFAULT_ACTOR=~ccleberg | |
| 99 | POLL_INTERVAL_SECONDS=900 | |
| 100 | ACTOR_ALIASES_JSON={"~ccleberg":["cmc@example.com","Chris Cleberg"]} | |
| 101 | GIT_TRACKED_REPOSITORIES=["Hutch","~ccleberg/cleberg.net"] | |
| 102 | ``` | |
| 103 | ||
| 104 | ## Local Run Instructions | |
| 105 | ||
| 106 | ### 1. Create a virtual environment and install dependencies | |
| 107 | ||
| 108 | Using `uv`: | |
| 109 | ||
| 110 | ```bash | |
| 111 | uv venv | |
| 112 | source .venv/bin/activate | |
| 113 | uv pip install -e ".[dev]" | |
| 114 | ``` | |
| 115 | ||
| 116 | Using `pip`: | |
| 117 | ||
| 118 | ```bash | |
| 119 | python3.12 -m venv .venv | |
| 120 | source .venv/bin/activate | |
| 121 | pip install -e ".[dev]" | |
| 122 | ``` | |
| 123 | ||
| 124 | ### 2. Configure environment | |
| 125 | ||
| 126 | ```bash | |
| 127 | cp .env.example .env | |
| 128 | ``` | |
| 129 | ||
| 130 | Set at least: | |
| 131 | ||
| 132 | - `API_KEY` | |
| 133 | - `SRHT_TOKEN` | |
| 134 | - `DEFAULT_ACTOR` | |
| 135 | - `GIT_TRACKED_REPOSITORIES` if you want git commit ingestion | |
| 136 | ||
| 137 | ### 3. Run database migrations | |
| 138 | ||
| 139 | ```bash | |
| 140 | alembic upgrade head | |
| 141 | ``` | |
| 142 | ||
| 143 | ### 4. Run the API | |
| 144 | ||
| 145 | ```bash | |
| 146 | uvicorn srht_contrib.main:app --reload | |
| 147 | ``` | |
| 148 | ||
| 149 | ## Manual Polling | |
| 150 | ||
| 151 | Manual polling is exposed as an API endpoint: | |
| 152 | ||
| 153 | ```bash | |
| 154 | curl -X POST "http://127.0.0.1:8000/api/contributions/poll?actor=~ccleberg" \ | |
| 155 | -H "X-API-Key: replace-me" | |
| 156 | ``` | |
| 157 | ||
| 158 | Example response: | |
| 159 | ||
| 160 | ```json | |
| 161 | { | |
| 162 | "actor": "~ccleberg", | |
| 163 | "inserted_events": 3, | |
| 164 | "services": ["todo", "git"] | |
| 165 | } | |
| 166 | ``` | |
| 167 | ||
| 168 | Scheduled polling only runs when `ENABLE_SCHEDULER=true` and uses `DEFAULT_ACTOR`. | |
| 169 | ||
| 170 | For `git.sr.ht`, tracked repositories are configured via `GIT_TRACKED_REPOSITORIES`. Entries may be either: | |
| 171 | ||
| 172 | - `"Hutch"` for a repository owned by `DEFAULT_ACTOR` | |
| 173 | - `"~ccleberg/cleberg.net"` for an explicit owner/repository pair | |
| 174 | ||
| 175 | ## API Endpoints | |
| 176 | ||
| 177 | ### Health | |
| 178 | ||
| 179 | ```bash | |
| 180 | curl "http://127.0.0.1:8000/health" | |
| 181 | ``` | |
| 182 | ||
| 183 | Response: | |
| 184 | ||
| 185 | ```json | |
| 186 | {"status":"ok"} | |
| 187 | ``` | |
| 188 | ||
| 189 | ### Contribution Calendar by Year | |
| 190 | ||
| 191 | ```bash | |
| 192 | curl "http://127.0.0.1:8000/api/contributions/~ccleberg?year=2026" \ | |
| 193 | -H "X-API-Key: replace-me" | |
| 194 | ``` | |
| 195 | ||
| 196 | ### Contribution Calendar by Date Range | |
| 197 | ||
| 198 | ```bash | |
| 199 | curl "http://127.0.0.1:8000/api/contributions/~ccleberg?from=2026-01-01&to=2026-03-30" \ | |
| 200 | -H "X-API-Key: replace-me" | |
| 201 | ``` | |
| 202 | ||
| 203 | Example response: | |
| 204 | ||
| 205 | ```json | |
| 206 | { | |
| 207 | "actor": "~ccleberg", | |
| 208 | "from": "2026-01-01", | |
| 209 | "to": "2026-03-30", | |
| 210 | "days": [ | |
| 211 | {"date": "2026-03-28", "count": 3, "score": 3.5}, | |
| 212 | {"date": "2026-03-29", "count": 0, "score": 0.0}, | |
| 213 | {"date": "2026-03-30", "count": 7, "score": 8.25} | |
| 214 | ] | |
| 215 | } | |
| 216 | ``` | |
| 217 | ||
| 218 | ### Contribution Stats | |
| 219 | ||
| 220 | ```bash | |
| 221 | curl "http://127.0.0.1:8000/api/contributions/~ccleberg/stats?year=2026" \ | |
| 222 | -H "X-API-Key: replace-me" | |
| 223 | ``` | |
| 224 | ||
| 225 | Example response: | |
| 226 | ||
| 227 | ```json | |
| 228 | { | |
| 229 | "actor": "~ccleberg", | |
| 230 | "from": "2026-01-01", | |
| 231 | "to": "2026-12-31", | |
| 232 | "total_events": 42, | |
| 233 | "total_score": 37.5, | |
| 234 | "active_days": 18, | |
| 235 | "longest_streak": 5, | |
| 236 | "current_streak": 2 | |
| 237 | } | |
| 238 | ``` | |
| 239 | ||
| 240 | ### Tracked Repositories | |
| 241 | ||
| 242 | List tracked repositories: | |
| 243 | ||
| 244 | ```bash | |
| 245 | curl "http://127.0.0.1:8000/api/repositories?actor=~ccleberg" \ | |
| 246 | -H "X-API-Key: replace-me" | |
| 247 | ``` | |
| 248 | ||
| 249 | Create a tracked repository: | |
| 250 | ||
| 251 | ```bash | |
| 252 | curl -X POST "http://127.0.0.1:8000/api/repositories" \ | |
| 253 | -H "X-API-Key: replace-me" \ | |
| 254 | -H "Content-Type: application/json" \ | |
| 255 | -d '{"actor":"~ccleberg","repo_name":"Hutch"}' | |
| 256 | ``` | |
| 257 | ||
| 258 | Get, update, and delete a tracked repository: | |
| 259 | ||
| 260 | ```bash | |
| 261 | curl "http://127.0.0.1:8000/api/repositories/1" \ | |
| 262 | -H "X-API-Key: replace-me" | |
| 263 | ||
| 264 | curl -X PATCH "http://127.0.0.1:8000/api/repositories/1" \ | |
| 265 | -H "X-API-Key: replace-me" \ | |
| 266 | -H "Content-Type: application/json" \ | |
| 267 | -d '{"repo_name":"~ccleberg/cleberg.net"}' | |
| 268 | ||
| 269 | curl -X DELETE "http://127.0.0.1:8000/api/repositories/1" \ | |
| 270 | -H "X-API-Key: replace-me" | |
| 271 | ``` | |
| 272 | ||
| 273 | ## Event Weighting | |
| 274 | ||
| 275 | Weights live in `src/srht_contrib/config.py` so they are easy to tune without touching aggregation code: | |
| 276 | ||
| 277 | - `commit`: `1.0` | |
| 278 | - `ticket_created`: `1.0` | |
| 279 | - `ticket_comment`: `0.5` | |
| 280 | - `ticket_closed`: `0.75` | |
| 281 | - `build_started`: `0.25` | |
| 282 | - `build_passed`: `0.25` | |
| 283 | ||
| 284 | ## Testing | |
| 285 | ||
| 286 | Run the test suite with: | |
| 287 | ||
| 288 | ```bash | |
| 289 | pytest | |
| 290 | ``` | |
| 291 | ||
| 292 | Covered areas: | |
| 293 | ||
| 294 | - health endpoint | |
| 295 | - API key enforcement | |
| 296 | - calendar aggregation | |
| 297 | - zero-filled ranges | |
| 298 | - stats calculations | |
| 299 | - invalid date handling | |
| 300 | - idempotent ingestion | |
| 301 | - todo feed fallback traversal | |
| 302 | - repository CRUD and normalization | |
| 303 | - SourceHut error mapping | |
| 304 | - git commit alias normalization | |
| 305 | - Alembic upgrade path | |
| 306 | ||
| 307 | ## SourceHut Schema Assumptions | |
| 308 | ||
| 309 | The SourceHut-specific assumptions are isolated to the service modules: | |
| 310 | ||
| 311 | - `src/srht_contrib/services/todo.py` uses the authenticated `events(cursor)` feed first, then falls back to tracker/ticket event traversal for reliable contribution discovery. | |
| 312 | - `src/srht_contrib/services/git.py` uses the documented repository `log(cursor)` query against tracked repositories and attributes commits through the configured alias map. | |
| 313 | ||
| 314 | ## Known Limitations | |
| 315 | ||
| 316 | - `git.sr.ht` polling is limited to repositories listed in `GIT_TRACKED_REPOSITORIES` | |
| 317 | - scheduled polling runs in-process, so it is not a distributed scheduler | |
| 318 | - alias management is config-driven; there is no alias CRUD API yet | |
| 319 | - current deployment model is trusted-operator V1, not a public multi-tenant service | |
| 320 | ||
| 321 | ## Recommended Next Steps | |
| 322 | ||
| 323 | 1. Add alias-management APIs or seed files for stronger actor identity mapping. | |
| 324 | 2. Add more SourceHut services such as `builds.sr.ht` and `lists.sr.ht`. | |
| 325 | 3. Move scheduled polling into an external worker if the deployment grows past a single process. | |
alembic.ini added +37
| @@ -0,0 +1,37 @@ | ||
| 1 | [alembic] | |
| 2 | script_location = alembic | |
| 3 | prepend_sys_path = . | |
| 4 | path_separator = os | |
| 5 | sqlalchemy.url = sqlite:///./srht_contrib.db | |
| 6 | ||
| 7 | [loggers] | |
| 8 | keys = root,sqlalchemy,alembic | |
| 9 | ||
| 10 | [handlers] | |
| 11 | keys = console | |
| 12 | ||
| 13 | [formatters] | |
| 14 | keys = generic | |
| 15 | ||
| 16 | [logger_root] | |
| 17 | level = WARN | |
| 18 | handlers = console | |
| 19 | ||
| 20 | [logger_sqlalchemy] | |
| 21 | level = WARN | |
| 22 | handlers = | |
| 23 | qualname = sqlalchemy.engine | |
| 24 | ||
| 25 | [logger_alembic] | |
| 26 | level = INFO | |
| 27 | handlers = console | |
| 28 | qualname = alembic | |
| 29 | ||
| 30 | [handler_console] | |
| 31 | class = StreamHandler | |
| 32 | args = (sys.stderr,) | |
| 33 | level = NOTSET | |
| 34 | formatter = generic | |
| 35 | ||
| 36 | [formatter_generic] | |
| 37 | format = %(levelname)-5.5s [%(name)s] %(message)s | |
alembic/env.py added +58
| @@ -0,0 +1,58 @@ | ||
| 1 | from __future__ import annotations | |
| 2 | ||
| 3 | from logging.config import fileConfig | |
| 4 | ||
| 5 | from alembic import context | |
| 6 | from sqlalchemy import engine_from_config, pool | |
| 7 | ||
| 8 | from srht_contrib.config import Settings | |
| 9 | from srht_contrib.db import Base | |
| 10 | from srht_contrib.models import ActorAlias, ContributionEvent, SyncState, TrackedRepository # noqa: F401 | |
| 11 | ||
| 12 | config = context.config | |
| 13 | ||
| 14 | if config.config_file_name is not None: | |
| 15 | fileConfig(config.config_file_name) | |
| 16 | ||
| 17 | target_metadata = Base.metadata | |
| 18 | ||
| 19 | ||
| 20 | def get_database_url() -> str: | |
| 21 | configured_url = config.get_main_option("sqlalchemy.url") | |
| 22 | if configured_url: | |
| 23 | return configured_url | |
| 24 | return Settings().database_url | |
| 25 | ||
| 26 | ||
| 27 | def run_migrations_offline() -> None: | |
| 28 | context.configure( | |
| 29 | url=get_database_url(), | |
| 30 | target_metadata=target_metadata, | |
| 31 | literal_binds=True, | |
| 32 | dialect_opts={"paramstyle": "named"}, | |
| 33 | ) | |
| 34 | ||
| 35 | with context.begin_transaction(): | |
| 36 | context.run_migrations() | |
| 37 | ||
| 38 | ||
| 39 | def run_migrations_online() -> None: | |
| 40 | section = config.get_section(config.config_ini_section) or {} | |
| 41 | section["sqlalchemy.url"] = get_database_url() | |
| 42 | connectable = engine_from_config( | |
| 43 | section, | |
| 44 | prefix="sqlalchemy.", | |
| 45 | poolclass=pool.NullPool, | |
| 46 | ) | |
| 47 | ||
| 48 | with connectable.connect() as connection: | |
| 49 | context.configure(connection=connection, target_metadata=target_metadata) | |
| 50 | ||
| 51 | with context.begin_transaction(): | |
| 52 | context.run_migrations() | |
| 53 | ||
| 54 | ||
| 55 | if context.is_offline_mode(): | |
| 56 | run_migrations_offline() | |
| 57 | else: | |
| 58 | run_migrations_online() | |
alembic/script.py.mako added +18
| @@ -0,0 +1,18 @@ | ||
| 1 | """${message}""" | |
| 2 | ||
| 3 | revision = ${repr(up_revision)} | |
| 4 | down_revision = ${repr(down_revision)} | |
| 5 | branch_labels = ${repr(branch_labels)} | |
| 6 | depends_on = ${repr(depends_on)} | |
| 7 | ||
| 8 | from alembic import op | |
| 9 | import sqlalchemy as sa | |
| 10 | ${imports if imports else ""} | |
| 11 | ||
| 12 | ||
| 13 | def upgrade() -> None: | |
| 14 | ${upgrades if upgrades else "pass"} | |
| 15 | ||
| 16 | ||
| 17 | def downgrade() -> None: | |
| 18 | ${downgrades if downgrades else "pass"} | |
alembic/versions/20260409_0001_initial_schema.py added +148
| @@ -0,0 +1,148 @@ | ||
| 1 | """initial schema""" | |
| 2 | ||
| 3 | from __future__ import annotations | |
| 4 | ||
| 5 | from alembic import op | |
| 6 | import sqlalchemy as sa | |
| 7 | from sqlalchemy import inspect | |
| 8 | ||
| 9 | from srht_contrib.config import Settings | |
| 10 | ||
| 11 | ||
| 12 | revision = "20260409_0001" | |
| 13 | down_revision = None | |
| 14 | branch_labels = None | |
| 15 | depends_on = None | |
| 16 | ||
| 17 | ||
| 18 | def _table_names() -> set[str]: | |
| 19 | return set(inspect(op.get_bind()).get_table_names()) | |
| 20 | ||
| 21 | ||
| 22 | def _index_names(table_name: str) -> set[str]: | |
| 23 | return {index["name"] for index in inspect(op.get_bind()).get_indexes(table_name)} | |
| 24 | ||
| 25 | ||
| 26 | def _tracked_repositories_needs_upgrade() -> bool: | |
| 27 | inspector = inspect(op.get_bind()) | |
| 28 | columns = {column["name"]: column for column in inspector.get_columns("tracked_repositories")} | |
| 29 | actor_column = columns.get("actor") | |
| 30 | if actor_column is None or actor_column.get("nullable", True): | |
| 31 | return True | |
| 32 | ||
| 33 | unique_constraints = {constraint["name"] for constraint in inspector.get_unique_constraints("tracked_repositories")} | |
| 34 | return "uq_tracked_repository_service_actor_name" not in unique_constraints | |
| 35 | ||
| 36 | ||
| 37 | def _upgrade_tracked_repositories() -> None: | |
| 38 | default_actor = Settings().default_actor | |
| 39 | op.execute( | |
| 40 | sa.text( | |
| 41 | """ | |
| 42 | CREATE TABLE tracked_repositories__alembic_new ( | |
| 43 | id INTEGER NOT NULL PRIMARY KEY, | |
| 44 | service VARCHAR(32) NOT NULL, | |
| 45 | repo_name VARCHAR(255) NOT NULL, | |
| 46 | actor VARCHAR(255) NOT NULL, | |
| 47 | CONSTRAINT uq_tracked_repository_service_actor_name UNIQUE (service, actor, repo_name) | |
| 48 | ) | |
| 49 | """ | |
| 50 | ) | |
| 51 | ) | |
| 52 | op.execute( | |
| 53 | sa.text( | |
| 54 | """ | |
| 55 | INSERT INTO tracked_repositories__alembic_new (id, service, repo_name, actor) | |
| 56 | SELECT id, service, repo_name, COALESCE(actor, :default_actor) | |
| 57 | FROM tracked_repositories | |
| 58 | """ | |
| 59 | ).bindparams(default_actor=default_actor) | |
| 60 | ) | |
| 61 | op.execute(sa.text("DROP TABLE tracked_repositories")) | |
| 62 | op.execute(sa.text("ALTER TABLE tracked_repositories__alembic_new RENAME TO tracked_repositories")) | |
| 63 | ||
| 64 | ||
| 65 | def upgrade() -> None: | |
| 66 | table_names = _table_names() | |
| 67 | ||
| 68 | if "contribution_events" not in table_names: | |
| 69 | op.create_table( | |
| 70 | "contribution_events", | |
| 71 | sa.Column("id", sa.Integer(), primary_key=True), | |
| 72 | sa.Column("service", sa.String(length=32), nullable=False), | |
| 73 | sa.Column("event_type", sa.String(length=64), nullable=False), | |
| 74 | sa.Column("actor", sa.String(length=255), nullable=False), | |
| 75 | sa.Column("repo_name", sa.String(length=255), nullable=True), | |
| 76 | sa.Column("resource_id", sa.String(length=255), nullable=False), | |
| 77 | sa.Column("external_uid", sa.String(length=255), nullable=False), | |
| 78 | sa.Column("occurred_at", sa.DateTime(timezone=True), nullable=False), | |
| 79 | sa.Column("weight", sa.Float(), nullable=False), | |
| 80 | sa.Column("raw_payload_json", sa.JSON(), nullable=True), | |
| 81 | sa.UniqueConstraint("service", "external_uid", name="uq_contribution_event_service_uid"), | |
| 82 | ) | |
| 83 | ||
| 84 | contribution_event_indexes = _index_names("contribution_events") | |
| 85 | if "ix_contribution_events_actor_occurred_at" not in contribution_event_indexes: | |
| 86 | op.create_index( | |
| 87 | "ix_contribution_events_actor_occurred_at", | |
| 88 | "contribution_events", | |
| 89 | ["actor", "occurred_at"], | |
| 90 | unique=False, | |
| 91 | ) | |
| 92 | if "ix_contribution_events_service_occurred_at" not in contribution_event_indexes: | |
| 93 | op.create_index( | |
| 94 | "ix_contribution_events_service_occurred_at", | |
| 95 | "contribution_events", | |
| 96 | ["service", "occurred_at"], | |
| 97 | unique=False, | |
| 98 | ) | |
| 99 | ||
| 100 | if "sync_states" not in table_names: | |
| 101 | op.create_table( | |
| 102 | "sync_states", | |
| 103 | sa.Column("id", sa.Integer(), primary_key=True), | |
| 104 | sa.Column("service", sa.String(length=32), nullable=False), | |
| 105 | sa.Column("actor", sa.String(length=255), nullable=False), | |
| 106 | sa.Column("cursor_value", sa.Text(), nullable=True), | |
| 107 | sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), | |
| 108 | sa.UniqueConstraint("service", "actor", name="uq_sync_state_service_actor"), | |
| 109 | ) | |
| 110 | ||
| 111 | if "tracked_repositories" not in table_names: | |
| 112 | op.create_table( | |
| 113 | "tracked_repositories", | |
| 114 | sa.Column("id", sa.Integer(), primary_key=True), | |
| 115 | sa.Column("service", sa.String(length=32), nullable=False), | |
| 116 | sa.Column("repo_name", sa.String(length=255), nullable=False), | |
| 117 | sa.Column("actor", sa.String(length=255), nullable=False), | |
| 118 | sa.UniqueConstraint("service", "actor", "repo_name", name="uq_tracked_repository_service_actor_name"), | |
| 119 | ) | |
| 120 | elif _tracked_repositories_needs_upgrade(): | |
| 121 | _upgrade_tracked_repositories() | |
| 122 | ||
| 123 | if "actor_aliases" not in table_names: | |
| 124 | op.create_table( | |
| 125 | "actor_aliases", | |
| 126 | sa.Column("id", sa.Integer(), primary_key=True), | |
| 127 | sa.Column("canonical_actor", sa.String(length=255), nullable=False), | |
| 128 | sa.Column("alias", sa.String(length=255), nullable=False), | |
| 129 | sa.UniqueConstraint("alias", name="uq_actor_alias_alias"), | |
| 130 | ) | |
| 131 | ||
| 132 | ||
| 133 | def downgrade() -> None: | |
| 134 | table_names = _table_names() | |
| 135 | ||
| 136 | if "actor_aliases" in table_names: | |
| 137 | op.drop_table("actor_aliases") | |
| 138 | if "tracked_repositories" in table_names: | |
| 139 | op.drop_table("tracked_repositories") | |
| 140 | if "sync_states" in table_names: | |
| 141 | op.drop_table("sync_states") | |
| 142 | if "contribution_events" in table_names: | |
| 143 | contribution_event_indexes = _index_names("contribution_events") | |
| 144 | if "ix_contribution_events_service_occurred_at" in contribution_event_indexes: | |
| 145 | op.drop_index("ix_contribution_events_service_occurred_at", table_name="contribution_events") | |
| 146 | if "ix_contribution_events_actor_occurred_at" in contribution_event_indexes: | |
| 147 | op.drop_index("ix_contribution_events_actor_occurred_at", table_name="contribution_events") | |
| 148 | op.drop_table("contribution_events") | |
pyproject.toml added +35
| @@ -0,0 +1,35 @@ | ||
| 1 | [build-system] | |
| 2 | requires = ["setuptools>=69", "wheel"] | |
| 3 | build-backend = "setuptools.build_meta" | |
| 4 | ||
| 5 | [project] | |
| 6 | name = "srht-contrib" | |
| 7 | version = "0.1.0" | |
| 8 | description = "SourceHut contribution calendar API built with FastAPI" | |
| 9 | readme = "README.md" | |
| 10 | requires-python = ">=3.12" | |
| 11 | dependencies = [ | |
| 12 | "fastapi>=0.115,<1.0", | |
| 13 | "uvicorn[standard]>=0.30,<1.0", | |
| 14 | "httpx>=0.27,<1.0", | |
| 15 | "sqlalchemy>=2.0,<3.0", | |
| 16 | "alembic>=1.13,<2.0", | |
| 17 | "pydantic>=2.8,<3.0", | |
| 18 | "pydantic-settings>=2.3,<3.0", | |
| 19 | "apscheduler>=3.10,<4.0", | |
| 20 | ] | |
| 21 | ||
| 22 | [project.optional-dependencies] | |
| 23 | dev = [ | |
| 24 | "pytest>=8.2,<9.0", | |
| 25 | ] | |
| 26 | ||
| 27 | [tool.setuptools] | |
| 28 | package-dir = {"" = "src"} | |
| 29 | ||
| 30 | [tool.setuptools.packages.find] | |
| 31 | where = ["src"] | |
| 32 | ||
| 33 | [tool.pytest.ini_options] | |
| 34 | pythonpath = ["src"] | |
| 35 | testpaths = ["tests"] | |
src/srht_contrib/__init__.py added +5
| @@ -0,0 +1,5 @@ | ||
| 1 | """srht_contrib package.""" | |
| 2 | ||
| 3 | __all__ = ["__version__"] | |
| 4 | ||
| 5 | __version__ = "0.1.0" | |
src/srht_contrib/api/__init__.py added +1
| @@ -0,0 +1 @@ | ||
| 1 | """API routes.""" | |
src/srht_contrib/api/dependencies.py added +45
| @@ -0,0 +1,45 @@ | ||
| 1 | from __future__ import annotations | |
| 2 | ||
| 3 | from fastapi import Depends, Header, HTTPException, Request, status | |
| 4 | from sqlalchemy.orm import Session | |
| 5 | ||
| 6 | from srht_contrib.config import Settings | |
| 7 | from srht_contrib.db import get_db_session | |
| 8 | from srht_contrib.jobs.poller import PollerService | |
| 9 | from srht_contrib.utils.identity import ActorIdentityResolver | |
| 10 | ||
| 11 | ||
| 12 | def get_db(request: Request): | |
| 13 | yield from get_db_session(request) | |
| 14 | ||
| 15 | ||
| 16 | def get_settings(request: Request) -> Settings: | |
| 17 | settings = getattr(request.app.state, "settings", None) | |
| 18 | if settings is None: | |
| 19 | raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Settings not configured.") | |
| 20 | return settings | |
| 21 | ||
| 22 | ||
| 23 | def get_actor_identity_resolver(request: Request) -> ActorIdentityResolver: | |
| 24 | resolver = getattr(request.app.state, "actor_identity_resolver", None) | |
| 25 | if resolver is None: | |
| 26 | raise HTTPException( | |
| 27 | status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, | |
| 28 | detail="Actor identity resolver not configured.", | |
| 29 | ) | |
| 30 | return resolver | |
| 31 | ||
| 32 | ||
| 33 | def get_poller(request: Request) -> PollerService: | |
| 34 | poller = getattr(request.app.state, "poller", None) | |
| 35 | if poller is None: | |
| 36 | raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Poller not configured.") | |
| 37 | return poller | |
| 38 | ||
| 39 | ||
| 40 | def require_api_key( | |
| 41 | x_api_key: str | None = Header(default=None, alias="X-API-Key"), | |
| 42 | settings: Settings = Depends(get_settings), | |
| 43 | ) -> None: | |
| 44 | if not settings.api_key or x_api_key != settings.api_key: | |
| 45 | raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid API key.") | |
src/srht_contrib/api/routes_contributions.py added +78
| @@ -0,0 +1,78 @@ | ||
| 1 | from __future__ import annotations | |
| 2 | ||
| 3 | from datetime import date | |
| 4 | ||
| 5 | from fastapi import APIRouter, Depends, HTTPException, Query | |
| 6 | from sqlalchemy.orm import Session | |
| 7 | ||
| 8 | from srht_contrib.api.dependencies import get_actor_identity_resolver, get_db, get_poller, require_api_key | |
| 9 | from srht_contrib.jobs.poller import PollerService | |
| 10 | from srht_contrib.schemas import ContributionCalendarResponse, ContributionStatsResponse, PollResponse | |
| 11 | from srht_contrib.services.aggregator import ContributionAggregator | |
| 12 | from srht_contrib.services.srht_client import SourceHutClientError | |
| 13 | from srht_contrib.utils.dates import parse_date, year_bounds | |
| 14 | from srht_contrib.utils.identity import ActorIdentityResolver | |
| 15 | ||
| 16 | router = APIRouter(prefix="/api/contributions", tags=["contributions"], dependencies=[Depends(require_api_key)]) | |
| 17 | ||
| 18 | ||
| 19 | def _resolve_range(year: int | None, from_date: str | None, to_date: str | None) -> tuple[date, date]: | |
| 20 | if year is not None and (from_date or to_date): | |
| 21 | raise HTTPException(status_code=400, detail="Use either `year` or `from`/`to`, not both.") | |
| 22 | ||
| 23 | try: | |
| 24 | if year is not None: | |
| 25 | return year_bounds(year) | |
| 26 | if from_date and to_date: | |
| 27 | start = parse_date(from_date) | |
| 28 | end = parse_date(to_date) | |
| 29 | if start > end: | |
| 30 | raise HTTPException(status_code=400, detail="`from` must be on or before `to`.") | |
| 31 | return start, end | |
| 32 | except ValueError as exc: | |
| 33 | raise HTTPException(status_code=400, detail="Invalid date format. Use YYYY-MM-DD.") from exc | |
| 34 | ||
| 35 | raise HTTPException(status_code=400, detail="Provide `year` or both `from` and `to`.") | |
| 36 | ||
| 37 | ||
| 38 | @router.get("/{actor}", response_model=ContributionCalendarResponse) | |
| 39 | def get_contributions( | |
| 40 | actor: str, | |
| 41 | year: int | None = Query(default=None, ge=1970, le=3000), | |
| 42 | from_date: str | None = Query(default=None, alias="from"), | |
| 43 | to_date: str | None = Query(default=None, alias="to"), | |
| 44 | db: Session = Depends(get_db), | |
| 45 | actor_identity_resolver: ActorIdentityResolver = Depends(get_actor_identity_resolver), | |
| 46 | ) -> ContributionCalendarResponse: | |
| 47 | start, end = _resolve_range(year, from_date, to_date) | |
| 48 | canonical_actor = actor_identity_resolver.canonicalize(actor, db=db) | |
| 49 | return ContributionAggregator().build_calendar(db, canonical_actor, start, end) | |
| 50 | ||
| 51 | ||
| 52 | @router.get("/{actor}/stats", response_model=ContributionStatsResponse) | |
| 53 | def get_contribution_stats( | |
| 54 | actor: str, | |
| 55 | year: int | None = Query(default=None, ge=1970, le=3000), | |
| 56 | from_date: str | None = Query(default=None, alias="from"), | |
| 57 | to_date: str | None = Query(default=None, alias="to"), | |
| 58 | db: Session = Depends(get_db), | |
| 59 | actor_identity_resolver: ActorIdentityResolver = Depends(get_actor_identity_resolver), | |
| 60 | ) -> ContributionStatsResponse: | |
| 61 | start, end = _resolve_range(year, from_date, to_date) | |
| 62 | canonical_actor = actor_identity_resolver.canonicalize(actor, db=db) | |
| 63 | return ContributionAggregator().build_stats(db, canonical_actor, start, end) | |
| 64 | ||
| 65 | ||
| 66 | @router.post("/poll", response_model=PollResponse) | |
| 67 | def trigger_manual_poll( | |
| 68 | actor: str, | |
| 69 | poller: PollerService = Depends(get_poller), | |
| 70 | db: Session = Depends(get_db), | |
| 71 | actor_identity_resolver: ActorIdentityResolver = Depends(get_actor_identity_resolver), | |
| 72 | ) -> PollResponse: | |
| 73 | canonical_actor = actor_identity_resolver.canonicalize(actor, db=db) | |
| 74 | try: | |
| 75 | inserted = poller.poll_all(db, canonical_actor) | |
| 76 | except SourceHutClientError as exc: | |
| 77 | raise HTTPException(status_code=502, detail=f"SourceHut polling failed: {exc}") from exc | |
| 78 | return PollResponse(actor=canonical_actor, inserted_events=inserted, services=["todo", "git"]) | |
src/srht_contrib/api/routes_health.py added +12
| @@ -0,0 +1,12 @@ | ||
| 1 | from __future__ import annotations | |
| 2 | ||
| 3 | from fastapi import APIRouter | |
| 4 | ||
| 5 | from srht_contrib.schemas import HealthResponse | |
| 6 | ||
| 7 | router = APIRouter(tags=["health"]) | |
| 8 | ||
| 9 | ||
| 10 | @router.get("/health", response_model=HealthResponse) | |
| 11 | def health() -> HealthResponse: | |
| 12 | return HealthResponse(status="ok") | |
src/srht_contrib/api/routes_repositories.py added +117
| @@ -0,0 +1,117 @@ | ||
| 1 | from __future__ import annotations | |
| 2 | ||
| 3 | from fastapi import APIRouter, Depends, HTTPException, Query, Response, status | |
| 4 | from sqlalchemy import select | |
| 5 | from sqlalchemy.exc import IntegrityError | |
| 6 | from sqlalchemy.orm import Session | |
| 7 | ||
| 8 | from srht_contrib.api.dependencies import get_actor_identity_resolver, get_db, require_api_key | |
| 9 | from srht_contrib.models import TrackedRepository | |
| 10 | from srht_contrib.schemas import ( | |
| 11 | TrackedRepositoryCreateRequest, | |
| 12 | TrackedRepositoryResponse, | |
| 13 | TrackedRepositoryUpdateRequest, | |
| 14 | ) | |
| 15 | from srht_contrib.utils.identity import ActorIdentityResolver | |
| 16 | from srht_contrib.utils.repositories import canonicalize_repository_name | |
| 17 | ||
| 18 | router = APIRouter(prefix="/api/repositories", tags=["repositories"], dependencies=[Depends(require_api_key)]) | |
| 19 | ||
| 20 | ||
| 21 | def _to_response(repository: TrackedRepository) -> TrackedRepositoryResponse: | |
| 22 | return TrackedRepositoryResponse( | |
| 23 | id=repository.id, | |
| 24 | service=repository.service, | |
| 25 | actor=repository.actor, | |
| 26 | repo_name=repository.repo_name, | |
| 27 | ) | |
| 28 | ||
| 29 | ||
| 30 | def _get_repository_or_404(db: Session, repository_id: int) -> TrackedRepository: | |
| 31 | repository = db.scalar(select(TrackedRepository).where(TrackedRepository.id == repository_id)) | |
| 32 | if repository is None: | |
| 33 | raise HTTPException(status_code=404, detail="Tracked repository not found.") | |
| 34 | return repository | |
| 35 | ||
| 36 | ||
| 37 | @router.get("", response_model=list[TrackedRepositoryResponse]) | |
| 38 | def list_tracked_repositories( | |
| 39 | actor: str | None = Query(default=None), | |
| 40 | db: Session = Depends(get_db), | |
| 41 | actor_identity_resolver: ActorIdentityResolver = Depends(get_actor_identity_resolver), | |
| 42 | ) -> list[TrackedRepositoryResponse]: | |
| 43 | stmt = select(TrackedRepository).where(TrackedRepository.service == "git").order_by(TrackedRepository.repo_name) | |
| 44 | if actor: | |
| 45 | canonical_actor = actor_identity_resolver.canonicalize(actor, db=db) | |
| 46 | stmt = stmt.where(TrackedRepository.actor == canonical_actor) | |
| 47 | ||
| 48 | repositories = db.scalars(stmt).all() | |
| 49 | return [_to_response(repository) for repository in repositories] | |
| 50 | ||
| 51 | ||
| 52 | @router.get("/{repository_id}", response_model=TrackedRepositoryResponse) | |
| 53 | def get_tracked_repository( | |
| 54 | repository_id: int, | |
| 55 | db: Session = Depends(get_db), | |
| 56 | ) -> TrackedRepositoryResponse: | |
| 57 | return _to_response(_get_repository_or_404(db, repository_id)) | |
| 58 | ||
| 59 | ||
| 60 | @router.post("", response_model=TrackedRepositoryResponse, status_code=status.HTTP_201_CREATED) | |
| 61 | def create_tracked_repository( | |
| 62 | payload: TrackedRepositoryCreateRequest, | |
| 63 | db: Session = Depends(get_db), | |
| 64 | actor_identity_resolver: ActorIdentityResolver = Depends(get_actor_identity_resolver), | |
| 65 | ) -> TrackedRepositoryResponse: | |
| 66 | canonical_actor = actor_identity_resolver.canonicalize(payload.actor, db=db) | |
| 67 | repository = TrackedRepository( | |
| 68 | service="git", | |
| 69 | actor=canonical_actor, | |
| 70 | repo_name=canonicalize_repository_name(canonical_actor, payload.repo_name), | |
| 71 | ) | |
| 72 | db.add(repository) | |
| 73 | try: | |
| 74 | db.commit() | |
| 75 | except IntegrityError as exc: | |
| 76 | db.rollback() | |
| 77 | raise HTTPException(status_code=409, detail="Tracked repository already exists.") from exc | |
| 78 | ||
| 79 | db.refresh(repository) | |
| 80 | return _to_response(repository) | |
| 81 | ||
| 82 | ||
| 83 | @router.patch("/{repository_id}", response_model=TrackedRepositoryResponse) | |
| 84 | def update_tracked_repository( | |
| 85 | repository_id: int, | |
| 86 | payload: TrackedRepositoryUpdateRequest, | |
| 87 | db: Session = Depends(get_db), | |
| 88 | actor_identity_resolver: ActorIdentityResolver = Depends(get_actor_identity_resolver), | |
| 89 | ) -> TrackedRepositoryResponse: | |
| 90 | repository = _get_repository_or_404(db, repository_id) | |
| 91 | ||
| 92 | if payload.actor is not None: | |
| 93 | repository.actor = actor_identity_resolver.canonicalize(payload.actor, db=db) | |
| 94 | ||
| 95 | if payload.repo_name is not None: | |
| 96 | repository.repo_name = canonicalize_repository_name(repository.actor, payload.repo_name) | |
| 97 | ||
| 98 | db.add(repository) | |
| 99 | try: | |
| 100 | db.commit() | |
| 101 | except IntegrityError as exc: | |
| 102 | db.rollback() | |
| 103 | raise HTTPException(status_code=409, detail="Tracked repository already exists.") from exc | |
| 104 | ||
| 105 | db.refresh(repository) | |
| 106 | return _to_response(repository) | |
| 107 | ||
| 108 | ||
| 109 | @router.delete("/{repository_id}", status_code=status.HTTP_204_NO_CONTENT) | |
| 110 | def delete_tracked_repository( | |
| 111 | repository_id: int, | |
| 112 | db: Session = Depends(get_db), | |
| 113 | ) -> Response: | |
| 114 | repository = _get_repository_or_404(db, repository_id) | |
| 115 | db.delete(repository) | |
| 116 | db.commit() | |
| 117 | return Response(status_code=status.HTTP_204_NO_CONTENT) | |
src/srht_contrib/config.py added +59
| @@ -0,0 +1,59 @@ | ||
| 1 | from __future__ import annotations | |
| 2 | ||
| 3 | from functools import lru_cache | |
| 4 | ||
| 5 | from pydantic import Field | |
| 6 | from pydantic_settings import BaseSettings, SettingsConfigDict | |
| 7 | ||
| 8 | ||
| 9 | class Settings(BaseSettings): | |
| 10 | model_config = SettingsConfigDict( | |
| 11 | env_file=".env", | |
| 12 | env_file_encoding="utf-8", | |
| 13 | case_sensitive=True, | |
| 14 | ) | |
| 15 | ||
| 16 | app_name: str = "srht-contrib" | |
| 17 | app_env: str = "development" | |
| 18 | api_host: str = "127.0.0.1" | |
| 19 | api_port: int = 8000 | |
| 20 | enable_scheduler: bool = Field(default=False, alias="ENABLE_SCHEDULER") | |
| 21 | api_key: str = Field(default="", alias="API_KEY") | |
| 22 | ||
| 23 | srht_token: str = Field(default="", alias="SRHT_TOKEN") | |
| 24 | todo_srht_endpoint: str = Field( | |
| 25 | default="https://todo.sr.ht/query", | |
| 26 | alias="TODO_SRHT_ENDPOINT", | |
| 27 | ) | |
| 28 | git_srht_endpoint: str = Field( | |
| 29 | default="https://git.sr.ht/query", | |
| 30 | alias="GIT_SRHT_ENDPOINT", | |
| 31 | ) | |
| 32 | database_url: str = Field( | |
| 33 | default="sqlite:///./srht_contrib.db", | |
| 34 | alias="DATABASE_URL", | |
| 35 | ) | |
| 36 | default_actor: str = Field(default="~unknown", alias="DEFAULT_ACTOR") | |
| 37 | poll_interval_seconds: int = Field(default=900, alias="POLL_INTERVAL_SECONDS") | |
| 38 | actor_aliases_json: dict[str, list[str]] = Field( | |
| 39 | default_factory=dict, | |
| 40 | alias="ACTOR_ALIASES_JSON", | |
| 41 | ) | |
| 42 | git_tracked_repositories: list[str] = Field( | |
| 43 | default_factory=list, | |
| 44 | alias="GIT_TRACKED_REPOSITORIES", | |
| 45 | ) | |
| 46 | ||
| 47 | event_weights: dict[str, float] = { | |
| 48 | "commit": 1.0, | |
| 49 | "ticket_created": 1.0, | |
| 50 | "ticket_comment": 0.5, | |
| 51 | "ticket_closed": 0.75, | |
| 52 | "build_started": 0.25, | |
| 53 | "build_passed": 0.25, | |
| 54 | } | |
| 55 | ||
| 56 | ||
| 57 | @lru_cache(maxsize=1) | |
| 58 | def get_settings() -> Settings: | |
| 59 | return Settings() | |
src/srht_contrib/db.py added +50
| @@ -0,0 +1,50 @@ | ||
| 1 | from __future__ import annotations | |
| 2 | ||
| 3 | from collections.abc import Generator | |
| 4 | ||
| 5 | from fastapi import HTTPException, Request, status | |
| 6 | from sqlalchemy import Engine, create_engine, text | |
| 7 | from sqlalchemy.pool import StaticPool | |
| 8 | from sqlalchemy.orm import Session, declarative_base, sessionmaker | |
| 9 | ||
| 10 | from srht_contrib.config import Settings | |
| 11 | ||
| 12 | Base = declarative_base() | |
| 13 | ||
| 14 | ||
| 15 | def make_engine(settings: Settings) -> Engine: | |
| 16 | connect_args = {"check_same_thread": False} if settings.database_url.startswith("sqlite") else {} | |
| 17 | engine_kwargs = {"future": True, "connect_args": connect_args} | |
| 18 | if settings.database_url in {"sqlite://", "sqlite:///:memory:"}: | |
| 19 | engine_kwargs["poolclass"] = StaticPool | |
| 20 | return create_engine(settings.database_url, **engine_kwargs) | |
| 21 | ||
| 22 | ||
| 23 | def make_session_factory(settings: Settings) -> sessionmaker[Session]: | |
| 24 | engine = make_engine(settings) | |
| 25 | return sessionmaker(bind=engine, autoflush=False, autocommit=False, expire_on_commit=False) | |
| 26 | ||
| 27 | ||
| 28 | def validate_db(bind: Engine) -> None: | |
| 29 | with bind.connect() as connection: | |
| 30 | connection.execute(text("SELECT 1")) | |
| 31 | ||
| 32 | ||
| 33 | def get_db() -> Generator[Session, None, None]: | |
| 34 | raise RuntimeError("Use get_db(request) dependency injection with a Request parameter.") | |
| 35 | ||
| 36 | ||
| 37 | def get_session_factory(request: Request) -> sessionmaker[Session]: | |
| 38 | session_factory = getattr(request.app.state, "session_factory", None) | |
| 39 | if session_factory is None: | |
| 40 | raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Database not configured.") | |
| 41 | return session_factory | |
| 42 | ||
| 43 | ||
| 44 | def get_db_session(request: Request) -> Generator[Session, None, None]: | |
| 45 | session_factory = get_session_factory(request) | |
| 46 | db = session_factory() | |
| 47 | try: | |
| 48 | yield db | |
| 49 | finally: | |
| 50 | db.close() | |
src/srht_contrib/jobs/__init__.py added +1
| @@ -0,0 +1 @@ | ||
| 1 | """Background jobs.""" | |
src/srht_contrib/jobs/poller.py added +119
| @@ -0,0 +1,119 @@ | ||
| 1 | from __future__ import annotations | |
| 2 | ||
| 3 | import logging | |
| 4 | from datetime import UTC, datetime, timedelta | |
| 5 | ||
| 6 | from sqlalchemy import select | |
| 7 | from sqlalchemy.exc import IntegrityError | |
| 8 | from sqlalchemy.orm import Session | |
| 9 | ||
| 10 | from srht_contrib.models import ContributionEvent, SyncState, TrackedRepository | |
| 11 | from srht_contrib.schemas import NormalizedEvent | |
| 12 | from srht_contrib.services.git import GitIngestionService | |
| 13 | from srht_contrib.services.todo import TodoIngestionService | |
| 14 | from srht_contrib.utils.repositories import canonicalize_repository_name | |
| 15 | ||
| 16 | ||
| 17 | logger = logging.getLogger(__name__) | |
| 18 | SYNC_OVERLAP = timedelta(hours=24) | |
| 19 | ||
| 20 | ||
| 21 | class PollerService: | |
| 22 | def __init__(self, todo_service: TodoIngestionService, git_service: GitIngestionService) -> None: | |
| 23 | self.todo_service = todo_service | |
| 24 | self.git_service = git_service | |
| 25 | ||
| 26 | def poll_all(self, db: Session, actor: str) -> int: | |
| 27 | inserted = 0 | |
| 28 | inserted += self._poll_service(db, actor, self.todo_service.service_name, self.todo_service.fetch_recent_events) | |
| 29 | self._sync_tracked_repositories(db, actor) | |
| 30 | git_repositories = self._tracked_repositories_for_actor(db, actor) | |
| 31 | inserted += self._poll_service( | |
| 32 | db, | |
| 33 | actor, | |
| 34 | self.git_service.service_name, | |
| 35 | lambda actor, since: self.git_service.fetch_recent_events( | |
| 36 | actor=actor, | |
| 37 | since=since, | |
| 38 | repositories=git_repositories, | |
| 39 | ), | |
| 40 | ) | |
| 41 | db.commit() | |
| 42 | return inserted | |
| 43 | ||
| 44 | def _poll_service(self, db: Session, actor: str, service_name: str, fetcher) -> int: | |
| 45 | state = db.scalar( | |
| 46 | select(SyncState).where(SyncState.service == service_name).where(SyncState.actor == actor) | |
| 47 | ) | |
| 48 | since = datetime.now(tz=UTC) - timedelta(days=30) | |
| 49 | if state and state.cursor_value: | |
| 50 | since = datetime.fromisoformat(state.cursor_value.replace("Z", "+00:00")).astimezone(UTC) - SYNC_OVERLAP | |
| 51 | logger.info( | |
| 52 | "Using sync cursor for %s actor=%s with overlap; since=%s", | |
| 53 | service_name, | |
| 54 | actor, | |
| 55 | since.isoformat(), | |
| 56 | ) | |
| 57 | ||
| 58 | result = fetcher(actor=actor, since=since) | |
| 59 | inserted = self._insert_events(db, result.events) | |
| 60 | self._upsert_sync_state(db, service_name, actor, result.cursor) | |
| 61 | logger.info("Polled %s for %s: inserted=%s", service_name, actor, inserted) | |
| 62 | return inserted | |
| 63 | ||
| 64 | @staticmethod | |
| 65 | def _insert_events(db: Session, events: list[NormalizedEvent]) -> int: | |
| 66 | inserted = 0 | |
| 67 | for event in events: | |
| 68 | try: | |
| 69 | with db.begin_nested(): | |
| 70 | model = ContributionEvent(**event.model_dump()) | |
| 71 | db.add(model) | |
| 72 | db.flush() | |
| 73 | inserted += 1 | |
| 74 | except IntegrityError: | |
| 75 | logger.info("Skipping duplicate event %s for service %s", event.external_uid, event.service) | |
| 76 | return inserted | |
| 77 | ||
| 78 | @staticmethod | |
| 79 | def _upsert_sync_state(db: Session, service: str, actor: str, cursor_value: str) -> None: | |
| 80 | state = db.scalar(select(SyncState).where(SyncState.service == service).where(SyncState.actor == actor)) | |
| 81 | now = datetime.now(tz=UTC) | |
| 82 | if state is None: | |
| 83 | db.add(SyncState(service=service, actor=actor, cursor_value=cursor_value, updated_at=now)) | |
| 84 | db.flush() | |
| 85 | return | |
| 86 | ||
| 87 | state.cursor_value = cursor_value | |
| 88 | state.updated_at = now | |
| 89 | db.add(state) | |
| 90 | db.flush() | |
| 91 | ||
| 92 | def _sync_tracked_repositories(self, db: Session, actor: str) -> None: | |
| 93 | configured = self.git_service.settings.git_tracked_repositories | |
| 94 | for repo_name in configured: | |
| 95 | canonical_repo_name = canonicalize_repository_name(actor, repo_name) | |
| 96 | existing = db.scalar( | |
| 97 | select(TrackedRepository) | |
| 98 | .where(TrackedRepository.service == self.git_service.service_name) | |
| 99 | .where(TrackedRepository.actor == actor) | |
| 100 | .where(TrackedRepository.repo_name == canonical_repo_name) | |
| 101 | ) | |
| 102 | if existing is None: | |
| 103 | db.add( | |
| 104 | TrackedRepository( | |
| 105 | service=self.git_service.service_name, | |
| 106 | repo_name=canonical_repo_name, | |
| 107 | actor=actor, | |
| 108 | ) | |
| 109 | ) | |
| 110 | db.flush() | |
| 111 | ||
| 112 | def _tracked_repositories_for_actor(self, db: Session, actor: str) -> list[str]: | |
| 113 | rows = db.scalars( | |
| 114 | select(TrackedRepository.repo_name) | |
| 115 | .where(TrackedRepository.service == self.git_service.service_name) | |
| 116 | .where(TrackedRepository.actor == actor) | |
| 117 | .order_by(TrackedRepository.repo_name) | |
| 118 | ).all() | |
| 119 | return list(rows) | |
src/srht_contrib/logging.py added +10
| @@ -0,0 +1,10 @@ | ||
| 1 | from __future__ import annotations | |
| 2 | ||
| 3 | import logging | |
| 4 | ||
| 5 | ||
| 6 | def configure_logging() -> None: | |
| 7 | logging.basicConfig( | |
| 8 | level=logging.INFO, | |
| 9 | format="%(asctime)s %(levelname)s [%(name)s] %(message)s", | |
| 10 | ) | |
src/srht_contrib/main.py added +99
| @@ -0,0 +1,99 @@ | ||
| 1 | from __future__ import annotations | |
| 2 | ||
| 3 | from collections.abc import AsyncIterator | |
| 4 | from contextlib import asynccontextmanager | |
| 5 | ||
| 6 | from apscheduler.schedulers.background import BackgroundScheduler | |
| 7 | from fastapi import FastAPI | |
| 8 | from sqlalchemy import Engine | |
| 9 | from sqlalchemy.orm import Session, sessionmaker | |
| 10 | ||
| 11 | from srht_contrib.api.routes_contributions import router as contributions_router | |
| 12 | from srht_contrib.api.routes_health import router as health_router | |
| 13 | from srht_contrib.api.routes_repositories import router as repositories_router | |
| 14 | from srht_contrib.config import Settings, get_settings | |
| 15 | from srht_contrib.db import make_engine, validate_db | |
| 16 | from srht_contrib.jobs.poller import PollerService | |
| 17 | from srht_contrib.logging import configure_logging | |
| 18 | from srht_contrib.services.git import GitIngestionService | |
| 19 | from srht_contrib.services.srht_client import SourceHutGraphQLClient | |
| 20 | from srht_contrib.services.todo import TodoIngestionService | |
| 21 | from srht_contrib.utils.identity import ActorIdentityResolver | |
| 22 | ||
| 23 | ||
| 24 | def build_poller(settings: Settings) -> PollerService: | |
| 25 | todo_client = SourceHutGraphQLClient(settings.todo_srht_endpoint, settings.srht_token) | |
| 26 | git_client = SourceHutGraphQLClient(settings.git_srht_endpoint, settings.srht_token) | |
| 27 | todo_service = TodoIngestionService(todo_client, settings) | |
| 28 | git_service = GitIngestionService(git_client, settings) | |
| 29 | return PollerService(todo_service=todo_service, git_service=git_service) | |
| 30 | ||
| 31 | ||
| 32 | def create_app( | |
| 33 | settings: Settings | None = None, | |
| 34 | *, | |
| 35 | engine: Engine | None = None, | |
| 36 | session_factory: sessionmaker[Session] | None = None, | |
| 37 | poller: PollerService | None = None, | |
| 38 | ) -> FastAPI: | |
| 39 | app_settings = settings or get_settings() | |
| 40 | configure_logging() | |
| 41 | app_engine = engine or make_engine(app_settings) | |
| 42 | app_session_factory = session_factory or sessionmaker( | |
| 43 | bind=app_engine, | |
| 44 | autoflush=False, | |
| 45 | autocommit=False, | |
| 46 | expire_on_commit=False, | |
| 47 | ) | |
| 48 | actor_identity_resolver = ActorIdentityResolver(app_settings.actor_aliases_json) | |
| 49 | ||
| 50 | @asynccontextmanager | |
| 51 | async def lifespan(app: FastAPI) -> AsyncIterator[None]: | |
| 52 | validate_db(app_engine) | |
| 53 | app_poller = poller or build_poller(app_settings) | |
| 54 | app.state.poller = app_poller | |
| 55 | app.state.settings = app_settings | |
| 56 | app.state.engine = app_engine | |
| 57 | app.state.session_factory = app_session_factory | |
| 58 | app.state.actor_identity_resolver = actor_identity_resolver | |
| 59 | scheduler: BackgroundScheduler | None = None | |
| 60 | if app_settings.enable_scheduler: | |
| 61 | scheduler = BackgroundScheduler() | |
| 62 | scheduler.add_job( | |
| 63 | _scheduled_poll, | |
| 64 | "interval", | |
| 65 | seconds=app_settings.poll_interval_seconds, | |
| 66 | args=[app], | |
| 67 | id="srht-poller", | |
| 68 | replace_existing=True, | |
| 69 | ) | |
| 70 | scheduler.start() | |
| 71 | app.state.scheduler = scheduler | |
| 72 | try: | |
| 73 | yield | |
| 74 | finally: | |
| 75 | if scheduler is not None: | |
| 76 | scheduler.shutdown(wait=False) | |
| 77 | app_poller.todo_service.client.close() | |
| 78 | app_poller.git_service.client.close() | |
| 79 | ||
| 80 | app = FastAPI(title=app_settings.app_name, lifespan=lifespan) | |
| 81 | ||
| 82 | app.include_router(health_router) | |
| 83 | app.include_router(contributions_router) | |
| 84 | app.include_router(repositories_router) | |
| 85 | return app | |
| 86 | ||
| 87 | ||
| 88 | def _scheduled_poll(app: FastAPI) -> None: | |
| 89 | settings: Settings = app.state.settings | |
| 90 | poller: PollerService = app.state.poller | |
| 91 | session_factory: sessionmaker[Session] = app.state.session_factory | |
| 92 | db = session_factory() | |
| 93 | try: | |
| 94 | poller.poll_all(db, settings.default_actor) | |
| 95 | finally: | |
| 96 | db.close() | |
| 97 | ||
| 98 | ||
| 99 | app = create_app() | |
src/srht_contrib/models.py added +60
| @@ -0,0 +1,60 @@ | ||
| 1 | from __future__ import annotations | |
| 2 | ||
| 3 | from datetime import datetime | |
| 4 | ||
| 5 | from sqlalchemy import JSON, DateTime, Float, Index, Integer, String, Text, UniqueConstraint | |
| 6 | from sqlalchemy.orm import Mapped, mapped_column | |
| 7 | ||
| 8 | from srht_contrib.db import Base | |
| 9 | ||
| 10 | ||
| 11 | class ContributionEvent(Base): | |
| 12 | __tablename__ = "contribution_events" | |
| 13 | __table_args__ = ( | |
| 14 | UniqueConstraint("service", "external_uid", name="uq_contribution_event_service_uid"), | |
| 15 | Index("ix_contribution_events_actor_occurred_at", "actor", "occurred_at"), | |
| 16 | Index("ix_contribution_events_service_occurred_at", "service", "occurred_at"), | |
| 17 | ) | |
| 18 | ||
| 19 | id: Mapped[int] = mapped_column(Integer, primary_key=True) | |
| 20 | service: Mapped[str] = mapped_column(String(32), nullable=False) | |
| 21 | event_type: Mapped[str] = mapped_column(String(64), nullable=False) | |
| 22 | actor: Mapped[str] = mapped_column(String(255), nullable=False) | |
| 23 | repo_name: Mapped[str | None] = mapped_column(String(255), nullable=True) | |
| 24 | resource_id: Mapped[str] = mapped_column(String(255), nullable=False) | |
| 25 | external_uid: Mapped[str] = mapped_column(String(255), nullable=False) | |
| 26 | occurred_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) | |
| 27 | weight: Mapped[float] = mapped_column(Float, nullable=False) | |
| 28 | raw_payload_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) | |
| 29 | ||
| 30 | ||
| 31 | class SyncState(Base): | |
| 32 | __tablename__ = "sync_states" | |
| 33 | __table_args__ = (UniqueConstraint("service", "actor", name="uq_sync_state_service_actor"),) | |
| 34 | ||
| 35 | id: Mapped[int] = mapped_column(Integer, primary_key=True) | |
| 36 | service: Mapped[str] = mapped_column(String(32), nullable=False) | |
| 37 | actor: Mapped[str] = mapped_column(String(255), nullable=False) | |
| 38 | cursor_value: Mapped[str | None] = mapped_column(Text, nullable=True) | |
| 39 | updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) | |
| 40 | ||
| 41 | ||
| 42 | class TrackedRepository(Base): | |
| 43 | __tablename__ = "tracked_repositories" | |
| 44 | __table_args__ = ( | |
| 45 | UniqueConstraint("service", "actor", "repo_name", name="uq_tracked_repository_service_actor_name"), | |
| 46 | ) | |
| 47 | ||
| 48 | id: Mapped[int] = mapped_column(Integer, primary_key=True) | |
| 49 | service: Mapped[str] = mapped_column(String(32), nullable=False) | |
| 50 | repo_name: Mapped[str] = mapped_column(String(255), nullable=False) | |
| 51 | actor: Mapped[str] = mapped_column(String(255), nullable=False) | |
| 52 | ||
| 53 | ||
| 54 | class ActorAlias(Base): | |
| 55 | __tablename__ = "actor_aliases" | |
| 56 | __table_args__ = (UniqueConstraint("alias", name="uq_actor_alias_alias"),) | |
| 57 | ||
| 58 | id: Mapped[int] = mapped_column(Integer, primary_key=True) | |
| 59 | canonical_actor: Mapped[str] = mapped_column(String(255), nullable=False) | |
| 60 | alias: Mapped[str] = mapped_column(String(255), nullable=False) | |
src/srht_contrib/schemas.py added +78
| @@ -0,0 +1,78 @@ | ||
| 1 | from __future__ import annotations | |
| 2 | ||
| 3 | from datetime import date, datetime | |
| 4 | ||
| 5 | from pydantic import BaseModel, Field, model_validator | |
| 6 | ||
| 7 | ||
| 8 | class NormalizedEvent(BaseModel): | |
| 9 | service: str | |
| 10 | event_type: str | |
| 11 | actor: str | |
| 12 | repo_name: str | None = None | |
| 13 | resource_id: str | |
| 14 | external_uid: str | |
| 15 | occurred_at: datetime | |
| 16 | weight: float | |
| 17 | raw_payload_json: dict | None = None | |
| 18 | ||
| 19 | ||
| 20 | class ContributionDay(BaseModel): | |
| 21 | date: date | |
| 22 | count: int | |
| 23 | score: float | |
| 24 | ||
| 25 | ||
| 26 | class ContributionCalendarResponse(BaseModel): | |
| 27 | actor: str | |
| 28 | from_date: date = Field(alias="from") | |
| 29 | to_date: date = Field(alias="to") | |
| 30 | days: list[ContributionDay] | |
| 31 | ||
| 32 | model_config = {"populate_by_name": True} | |
| 33 | ||
| 34 | ||
| 35 | class ContributionStatsResponse(BaseModel): | |
| 36 | actor: str | |
| 37 | from_date: date = Field(alias="from") | |
| 38 | to_date: date = Field(alias="to") | |
| 39 | total_events: int | |
| 40 | total_score: float | |
| 41 | active_days: int | |
| 42 | longest_streak: int | |
| 43 | current_streak: int | |
| 44 | ||
| 45 | model_config = {"populate_by_name": True} | |
| 46 | ||
| 47 | ||
| 48 | class HealthResponse(BaseModel): | |
| 49 | status: str | |
| 50 | ||
| 51 | ||
| 52 | class PollResponse(BaseModel): | |
| 53 | actor: str | |
| 54 | inserted_events: int | |
| 55 | services: list[str] | |
| 56 | ||
| 57 | ||
| 58 | class TrackedRepositoryCreateRequest(BaseModel): | |
| 59 | actor: str | |
| 60 | repo_name: str | |
| 61 | ||
| 62 | ||
| 63 | class TrackedRepositoryUpdateRequest(BaseModel): | |
| 64 | actor: str | None = None | |
| 65 | repo_name: str | None = None | |
| 66 | ||
| 67 | @model_validator(mode="after") | |
| 68 | def validate_any_field_present(self) -> "TrackedRepositoryUpdateRequest": | |
| 69 | if self.actor is None and self.repo_name is None: | |
| 70 | raise ValueError("Provide `actor`, `repo_name`, or both.") | |
| 71 | return self | |
| 72 | ||
| 73 | ||
| 74 | class TrackedRepositoryResponse(BaseModel): | |
| 75 | id: int | |
| 76 | service: str | |
| 77 | actor: str | |
| 78 | repo_name: str | |
src/srht_contrib/services/__init__.py added +1
| @@ -0,0 +1 @@ | ||
| 1 | """Service layer.""" | |
src/srht_contrib/services/aggregator.py added +100
| @@ -0,0 +1,100 @@ | ||
| 1 | from __future__ import annotations | |
| 2 | ||
| 3 | from dataclasses import dataclass | |
| 4 | from datetime import date | |
| 5 | ||
| 6 | from sqlalchemy import func, select | |
| 7 | from sqlalchemy.orm import Session | |
| 8 | ||
| 9 | from srht_contrib.models import ContributionEvent | |
| 10 | from srht_contrib.schemas import ContributionCalendarResponse, ContributionDay, ContributionStatsResponse | |
| 11 | from srht_contrib.utils.dates import date_range, date_to_utc_bounds | |
| 12 | ||
| 13 | ||
| 14 | @dataclass(slots=True) | |
| 15 | class DailyAggregate: | |
| 16 | date: date | |
| 17 | count: int | |
| 18 | score: float | |
| 19 | ||
| 20 | ||
| 21 | class ContributionAggregator: | |
| 22 | def build_calendar(self, db: Session, actor: str, start: date, end: date) -> ContributionCalendarResponse: | |
| 23 | aggregates = self._query_daily_aggregates(db, actor, start, end) | |
| 24 | by_day = {row.date: row for row in aggregates} | |
| 25 | days = [ | |
| 26 | ContributionDay( | |
| 27 | date=day, | |
| 28 | count=by_day.get(day, DailyAggregate(date=day, count=0, score=0.0)).count, | |
| 29 | score=by_day.get(day, DailyAggregate(date=day, count=0, score=0.0)).score, | |
| 30 | ) | |
| 31 | for day in date_range(start, end) | |
| 32 | ] | |
| 33 | return ContributionCalendarResponse(actor=actor, from_date=start, to_date=end, days=days) | |
| 34 | ||
| 35 | def build_stats(self, db: Session, actor: str, start: date, end: date) -> ContributionStatsResponse: | |
| 36 | calendar = self.build_calendar(db, actor, start, end) | |
| 37 | active_days = [day for day in calendar.days if day.count > 0] | |
| 38 | streaks = self._streak_lengths(calendar.days) | |
| 39 | current_streak = self._current_streak(calendar.days) | |
| 40 | ||
| 41 | return ContributionStatsResponse( | |
| 42 | actor=actor, | |
| 43 | from_date=start, | |
| 44 | to_date=end, | |
| 45 | total_events=sum(day.count for day in calendar.days), | |
| 46 | total_score=round(sum(day.score for day in calendar.days), 2), | |
| 47 | active_days=len(active_days), | |
| 48 | longest_streak=max(streaks, default=0), | |
| 49 | current_streak=current_streak, | |
| 50 | ) | |
| 51 | ||
| 52 | def _query_daily_aggregates(self, db: Session, actor: str, start: date, end: date) -> list[DailyAggregate]: | |
| 53 | start_dt, _ = date_to_utc_bounds(start) | |
| 54 | _, end_dt = date_to_utc_bounds(end) | |
| 55 | ||
| 56 | stmt = ( | |
| 57 | select( | |
| 58 | func.date(ContributionEvent.occurred_at).label("day"), | |
| 59 | func.count(ContributionEvent.id).label("count"), | |
| 60 | func.coalesce(func.sum(ContributionEvent.weight), 0.0).label("score"), | |
| 61 | ) | |
| 62 | .where(ContributionEvent.actor == actor) | |
| 63 | .where(ContributionEvent.occurred_at >= start_dt) | |
| 64 | .where(ContributionEvent.occurred_at <= end_dt) | |
| 65 | .group_by(func.date(ContributionEvent.occurred_at)) | |
| 66 | .order_by(func.date(ContributionEvent.occurred_at)) | |
| 67 | ) | |
| 68 | rows = db.execute(stmt).all() | |
| 69 | return [ | |
| 70 | DailyAggregate( | |
| 71 | date=date.fromisoformat(str(row.day)), | |
| 72 | count=int(row.count), | |
| 73 | score=round(float(row.score), 2), | |
| 74 | ) | |
| 75 | for row in rows | |
| 76 | ] | |
| 77 | ||
| 78 | @staticmethod | |
| 79 | def _streak_lengths(days: list[ContributionDay]) -> list[int]: | |
| 80 | streaks: list[int] = [] | |
| 81 | current = 0 | |
| 82 | for day in days: | |
| 83 | if day.count > 0: | |
| 84 | current += 1 | |
| 85 | elif current > 0: | |
| 86 | streaks.append(current) | |
| 87 | current = 0 | |
| 88 | if current > 0: | |
| 89 | streaks.append(current) | |
| 90 | return streaks | |
| 91 | ||
| 92 | @staticmethod | |
| 93 | def _current_streak(days: list[ContributionDay]) -> int: | |
| 94 | streak = 0 | |
| 95 | for day in reversed(days): | |
| 96 | if day.count > 0: | |
| 97 | streak += 1 | |
| 98 | else: | |
| 99 | break | |
| 100 | return streak | |
src/srht_contrib/services/git.py added +202
| @@ -0,0 +1,202 @@ | ||
| 1 | from __future__ import annotations | |
| 2 | ||
| 3 | from dataclasses import dataclass | |
| 4 | from datetime import UTC, datetime, timedelta | |
| 5 | import logging | |
| 6 | from typing import Any | |
| 7 | ||
| 8 | from srht_contrib.config import Settings | |
| 9 | from srht_contrib.schemas import NormalizedEvent | |
| 10 | from srht_contrib.services.srht_client import SourceHutGraphQLClient | |
| 11 | from srht_contrib.utils.dates import ensure_utc, parse_datetime | |
| 12 | from srht_contrib.utils.identity import ActorIdentityResolver | |
| 13 | ||
| 14 | ||
| 15 | logger = logging.getLogger(__name__) | |
| 16 | ||
| 17 | ||
| 18 | REPOSITORY_LOG_QUERY = """ | |
| 19 | query RepositoryLog($username: String!, $repoName: String!, $cursor: Cursor) { | |
| 20 | user(username: $username) { | |
| 21 | repository(name: $repoName) { | |
| 22 | name | |
| 23 | owner { | |
| 24 | canonicalName | |
| 25 | } | |
| 26 | log(cursor: $cursor) { | |
| 27 | results { | |
| 28 | id | |
| 29 | shortId | |
| 30 | author { | |
| 31 | name | |
| 32 | ||
| 33 | time | |
| 34 | } | |
| 35 | committer { | |
| 36 | name | |
| 37 | ||
| 38 | time | |
| 39 | } | |
| 40 | message | |
| 41 | } | |
| 42 | cursor | |
| 43 | } | |
| 44 | } | |
| 45 | } | |
| 46 | } | |
| 47 | """.strip() | |
| 48 | ||
| 49 | ||
| 50 | @dataclass(slots=True) | |
| 51 | class GitPollResult: | |
| 52 | events: list[NormalizedEvent] | |
| 53 | cursor: str | |
| 54 | ||
| 55 | ||
| 56 | class GitIngestionService: | |
| 57 | """Polls tracked git.sr.ht repositories and normalizes commits for one actor.""" | |
| 58 | ||
| 59 | service_name = "git" | |
| 60 | ||
| 61 | def __init__(self, client: SourceHutGraphQLClient, settings: Settings) -> None: | |
| 62 | self.client = client | |
| 63 | self.settings = settings | |
| 64 | self.identity_resolver = ActorIdentityResolver(settings.actor_aliases_json) | |
| 65 | ||
| 66 | def fetch_recent_events( | |
| 67 | self, | |
| 68 | actor: str, | |
| 69 | since: datetime | None = None, | |
| 70 | repositories: list[str] | None = None, | |
| 71 | ) -> GitPollResult: | |
| 72 | since_dt = ensure_utc(since or (datetime.now(tz=UTC) - timedelta(days=30))) | |
| 73 | tracked_repositories = repositories or self._tracked_repositories(actor) | |
| 74 | if not tracked_repositories: | |
| 75 | logger.info("git poll skipped for actor=%s because no tracked repositories are configured", actor) | |
| 76 | return GitPollResult(events=[], cursor=datetime.now(tz=UTC).isoformat()) | |
| 77 | ||
| 78 | events: list[NormalizedEvent] = [] | |
| 79 | for repository in tracked_repositories: | |
| 80 | owner, repo_name = self._split_repository(actor, repository) | |
| 81 | repo_events = self._fetch_repository_commits(actor=actor, owner=owner, repo_name=repo_name, since=since_dt) | |
| 82 | events.extend(repo_events) | |
| 83 | ||
| 84 | logger.info("git poll complete for actor=%s normalized_events=%s", actor, len(events)) | |
| 85 | return GitPollResult(events=events, cursor=datetime.now(tz=UTC).isoformat()) | |
| 86 | ||
| 87 | def _tracked_repositories(self, actor: str) -> list[str]: | |
| 88 | repositories = self.settings.git_tracked_repositories | |
| 89 | return repositories | |
| 90 | ||
| 91 | @staticmethod | |
| 92 | def _split_repository(default_actor: str, repository: str) -> tuple[str, str]: | |
| 93 | if "/" in repository: | |
| 94 | owner, repo_name = repository.split("/", 1) | |
| 95 | canonical_owner = owner if owner.startswith("~") else f"~{owner}" | |
| 96 | return canonical_owner.lstrip("~"), repo_name | |
| 97 | return default_actor.lstrip("~"), repository | |
| 98 | ||
| 99 | def _fetch_repository_commits( | |
| 100 | self, | |
| 101 | *, | |
| 102 | actor: str, | |
| 103 | owner: str, | |
| 104 | repo_name: str, | |
| 105 | since: datetime, | |
| 106 | ) -> list[NormalizedEvent]: | |
| 107 | events: list[NormalizedEvent] = [] | |
| 108 | cursor: str | None = None | |
| 109 | ||
| 110 | for _ in range(50): | |
| 111 | data = self.client.execute( | |
| 112 | REPOSITORY_LOG_QUERY, | |
| 113 | {"username": owner, "repoName": repo_name, "cursor": cursor}, | |
| 114 | ) | |
| 115 | user = data.get("user") or {} | |
| 116 | repository = user.get("repository") or {} | |
| 117 | log_page = repository.get("log") or {} | |
| 118 | commits = log_page.get("results") or [] | |
| 119 | cursor = log_page.get("cursor") | |
| 120 | logger.info( | |
| 121 | "git repository=%s/%s commit page count=%s next_cursor=%s", | |
| 122 | owner, | |
| 123 | repo_name, | |
| 124 | len(commits), | |
| 125 | bool(cursor), | |
| 126 | ) | |
| 127 | ||
| 128 | stop_paging = False | |
| 129 | for commit in commits: | |
| 130 | if not isinstance(commit, dict): | |
| 131 | continue | |
| 132 | commit_time = parse_datetime((commit.get("author") or {}).get("time")) | |
| 133 | if commit_time < since: | |
| 134 | stop_paging = True | |
| 135 | logger.info( | |
| 136 | "git commit %s skipped because commit_time=%s is before since=%s", | |
| 137 | commit.get("shortId") or commit.get("id"), | |
| 138 | commit_time.isoformat(), | |
| 139 | since.isoformat(), | |
| 140 | ) | |
| 141 | continue | |
| 142 | ||
| 143 | normalized = self._normalize_commit(actor=actor, repo_name=repo_name, commit=commit) | |
| 144 | if normalized is not None: | |
| 145 | logger.info( | |
| 146 | "git commit accepted repo=%s shortId=%s author=%s email=%s", | |
| 147 | repo_name, | |
| 148 | commit.get("shortId"), | |
| 149 | (commit.get("author") or {}).get("name"), | |
| 150 | (commit.get("author") or {}).get("email"), | |
| 151 | ) | |
| 152 | events.append(normalized) | |
| 153 | else: | |
| 154 | logger.info( | |
| 155 | "git commit skipped repo=%s shortId=%s author=%s email=%s", | |
| 156 | repo_name, | |
| 157 | commit.get("shortId"), | |
| 158 | (commit.get("author") or {}).get("name"), | |
| 159 | (commit.get("author") or {}).get("email"), | |
| 160 | ) | |
| 161 | ||
| 162 | if stop_paging or not cursor: | |
| 163 | break | |
| 164 | ||
| 165 | return events | |
| 166 | ||
| 167 | def _normalize_commit( | |
| 168 | self, | |
| 169 | *, | |
| 170 | actor: str, | |
| 171 | repo_name: str, | |
| 172 | commit: dict[str, Any], | |
| 173 | ) -> NormalizedEvent | None: | |
| 174 | author = commit.get("author") or {} | |
| 175 | candidate_aliases = [ | |
| 176 | actor, | |
| 177 | author.get("email", ""), | |
| 178 | author.get("name", ""), | |
| 179 | ] | |
| 180 | matched_actor = None | |
| 181 | for candidate in candidate_aliases: | |
| 182 | canonical = self.identity_resolver.canonicalize(candidate) | |
| 183 | if canonical == actor: | |
| 184 | matched_actor = canonical | |
| 185 | break | |
| 186 | ||
| 187 | if matched_actor is None: | |
| 188 | return None | |
| 189 | ||
| 190 | commit_id = str(commit["id"]) | |
| 191 | commit_time = parse_datetime(author["time"]) | |
| 192 | return NormalizedEvent( | |
| 193 | service=self.service_name, | |
| 194 | event_type="commit", | |
| 195 | actor=matched_actor, | |
| 196 | repo_name=repo_name, | |
| 197 | resource_id=commit_id, | |
| 198 | external_uid=f"git:{repo_name}:{commit_id}", | |
| 199 | occurred_at=commit_time, | |
| 200 | weight=self.settings.event_weights["commit"], | |
| 201 | raw_payload_json=commit, | |
| 202 | ) | |
src/srht_contrib/services/srht_client.py added +77
| @@ -0,0 +1,77 @@ | ||
| 1 | from __future__ import annotations | |
| 2 | ||
| 3 | import logging | |
| 4 | from typing import Any | |
| 5 | ||
| 6 | import httpx | |
| 7 | ||
| 8 | ||
| 9 | logger = logging.getLogger(__name__) | |
| 10 | ||
| 11 | ||
| 12 | class SourceHutClientError(RuntimeError): | |
| 13 | """Raised when a SourceHut GraphQL request fails.""" | |
| 14 | ||
| 15 | ||
| 16 | class SourceHutGraphQLClient: | |
| 17 | def __init__( | |
| 18 | self, | |
| 19 | endpoint: str, | |
| 20 | token: str, | |
| 21 | *, | |
| 22 | timeout: float = 15.0, | |
| 23 | max_retries: int = 2, | |
| 24 | transport: httpx.BaseTransport | None = None, | |
| 25 | ) -> None: | |
| 26 | self.endpoint = endpoint | |
| 27 | self.timeout = timeout | |
| 28 | self.max_retries = max_retries | |
| 29 | headers = { | |
| 30 | "Authorization": f"Bearer {token}", | |
| 31 | "Content-Type": "application/json", | |
| 32 | } | |
| 33 | self._client = httpx.Client(headers=headers, timeout=timeout, transport=transport) | |
| 34 | ||
| 35 | def execute(self, query: str, variables: dict[str, Any] | None = None) -> dict[str, Any]: | |
| 36 | payload = {"query": query, "variables": variables or {}} | |
| 37 | attempts = self.max_retries + 1 | |
| 38 | ||
| 39 | for attempt in range(1, attempts + 1): | |
| 40 | try: | |
| 41 | response = self._client.post(self.endpoint, json=payload) | |
| 42 | response.raise_for_status() | |
| 43 | body = response.json() | |
| 44 | except httpx.HTTPStatusError as exc: | |
| 45 | response_text = exc.response.text[:500] | |
| 46 | logger.warning( | |
| 47 | "SourceHut HTTP failure from %s on attempt %s/%s: %s %s", | |
| 48 | self.endpoint, | |
| 49 | attempt, | |
| 50 | attempts, | |
| 51 | exc.response.status_code, | |
| 52 | response_text, | |
| 53 | ) | |
| 54 | if exc.response.status_code >= 500 and attempt < attempts: | |
| 55 | continue | |
| 56 | raise SourceHutClientError( | |
| 57 | f"HTTP error from SourceHut: {exc.response.status_code} {response_text}".strip() | |
| 58 | ) from exc | |
| 59 | except httpx.HTTPError as exc: | |
| 60 | logger.warning( | |
| 61 | "SourceHut network failure from %s on attempt %s/%s", | |
| 62 | self.endpoint, | |
| 63 | attempt, | |
| 64 | attempts, | |
| 65 | ) | |
| 66 | if attempt < attempts: | |
| 67 | continue | |
| 68 | raise SourceHutClientError("Network error while contacting SourceHut") from exc | |
| 69 | ||
| 70 | if "errors" in body: | |
| 71 | raise SourceHutClientError(f"GraphQL errors returned by SourceHut: {body['errors']}") | |
| 72 | return body.get("data", {}) | |
| 73 | ||
| 74 | raise SourceHutClientError("SourceHut request exhausted retries") | |
| 75 | ||
| 76 | def close(self) -> None: | |
| 77 | self._client.close() | |
src/srht_contrib/services/todo.py added +559
| @@ -0,0 +1,559 @@ | ||
| 1 | from __future__ import annotations | |
| 2 | ||
| 3 | from dataclasses import dataclass | |
| 4 | from datetime import UTC, datetime, timedelta | |
| 5 | import logging | |
| 6 | from typing import Any | |
| 7 | ||
| 8 | from srht_contrib.config import Settings | |
| 9 | from srht_contrib.schemas import NormalizedEvent | |
| 10 | from srht_contrib.services.srht_client import SourceHutGraphQLClient | |
| 11 | from srht_contrib.utils.dates import ensure_utc, parse_datetime | |
| 12 | ||
| 13 | ||
| 14 | logger = logging.getLogger(__name__) | |
| 15 | ||
| 16 | ||
| 17 | TODO_ACTIVITY_QUERY = """ | |
| 18 | query TodoActivity($cursor: Cursor) { | |
| 19 | me { | |
| 20 | canonicalName | |
| 21 | } | |
| 22 | events(cursor: $cursor) { | |
| 23 | results { | |
| 24 | id | |
| 25 | created | |
| 26 | ticket { | |
| 27 | id | |
| 28 | ref | |
| 29 | status | |
| 30 | resolution | |
| 31 | tracker { | |
| 32 | name | |
| 33 | } | |
| 34 | } | |
| 35 | changes { | |
| 36 | __typename | |
| 37 | eventType | |
| 38 | ticket { | |
| 39 | id | |
| 40 | } | |
| 41 | ... on Created { | |
| 42 | author { | |
| 43 | canonicalName | |
| 44 | } | |
| 45 | } | |
| 46 | ... on Comment { | |
| 47 | author { | |
| 48 | canonicalName | |
| 49 | } | |
| 50 | } | |
| 51 | ... on StatusChange { | |
| 52 | editor { | |
| 53 | canonicalName | |
| 54 | } | |
| 55 | oldStatus | |
| 56 | newStatus | |
| 57 | oldResolution | |
| 58 | newResolution | |
| 59 | } | |
| 60 | } | |
| 61 | } | |
| 62 | cursor | |
| 63 | } | |
| 64 | } | |
| 65 | """.strip() | |
| 66 | ||
| 67 | TODO_TRACKERS_QUERY = """ | |
| 68 | query TodoTrackers($cursor: Cursor) { | |
| 69 | me { | |
| 70 | canonicalName | |
| 71 | trackers(cursor: $cursor) { | |
| 72 | results { | |
| 73 | id | |
| 74 | rid | |
| 75 | name | |
| 76 | } | |
| 77 | cursor | |
| 78 | } | |
| 79 | } | |
| 80 | } | |
| 81 | """.strip() | |
| 82 | ||
| 83 | TODO_TRACKER_TICKETS_QUERY = """ | |
| 84 | query TodoTrackerTickets($trackerRid: ID!, $cursor: Cursor) { | |
| 85 | tracker(rid: $trackerRid) { | |
| 86 | id | |
| 87 | name | |
| 88 | tickets(cursor: $cursor) { | |
| 89 | results { | |
| 90 | id | |
| 91 | ref | |
| 92 | created | |
| 93 | updated | |
| 94 | status | |
| 95 | resolution | |
| 96 | submitter { | |
| 97 | canonicalName | |
| 98 | } | |
| 99 | } | |
| 100 | cursor | |
| 101 | } | |
| 102 | } | |
| 103 | } | |
| 104 | """.strip() | |
| 105 | ||
| 106 | TODO_TICKET_EVENTS_QUERY = """ | |
| 107 | query TodoTicketEvents($trackerRid: ID!, $ticketId: Int!, $cursor: Cursor) { | |
| 108 | tracker(rid: $trackerRid) { | |
| 109 | id | |
| 110 | name | |
| 111 | ticket(id: $ticketId) { | |
| 112 | id | |
| 113 | ref | |
| 114 | status | |
| 115 | resolution | |
| 116 | events(cursor: $cursor) { | |
| 117 | results { | |
| 118 | id | |
| 119 | created | |
| 120 | changes { | |
| 121 | __typename | |
| 122 | eventType | |
| 123 | ticket { | |
| 124 | id | |
| 125 | } | |
| 126 | ... on Created { | |
| 127 | author { | |
| 128 | canonicalName | |
| 129 | } | |
| 130 | } | |
| 131 | ... on Comment { | |
| 132 | author { | |
| 133 | canonicalName | |
| 134 | } | |
| 135 | } | |
| 136 | ... on StatusChange { | |
| 137 | editor { | |
| 138 | canonicalName | |
| 139 | } | |
| 140 | oldStatus | |
| 141 | newStatus | |
| 142 | oldResolution | |
| 143 | newResolution | |
| 144 | } | |
| 145 | } | |
| 146 | } | |
| 147 | cursor | |
| 148 | } | |
| 149 | } | |
| 150 | } | |
| 151 | } | |
| 152 | """.strip() | |
| 153 | ||
| 154 | ||
| 155 | TICKET_CLOSED_STATUSES = {"RESOLVED"} | |
| 156 | TICKET_CLOSED_RESOLUTIONS = { | |
| 157 | "CLOSED", | |
| 158 | "FIXED", | |
| 159 | "IMPLEMENTED", | |
| 160 | "WONT_FIX", | |
| 161 | "BY_DESIGN", | |
| 162 | "INVALID", | |
| 163 | "DUPLICATE", | |
| 164 | "NOT_OUR_BUG", | |
| 165 | } | |
| 166 | ||
| 167 | ||
| 168 | class TodoSchemaError(RuntimeError): | |
| 169 | """Raised when SourceHut returns an unexpected todo event shape.""" | |
| 170 | ||
| 171 | ||
| 172 | def _safe_nested_name(entity: dict[str, Any] | None) -> str | None: | |
| 173 | if not entity: | |
| 174 | return None | |
| 175 | return entity.get("canonicalName") or entity.get("name") | |
| 176 | ||
| 177 | ||
| 178 | def _repo_name_from_event(event: dict[str, Any]) -> str | None: | |
| 179 | ticket = event.get("ticket") or {} | |
| 180 | tracker = ticket.get("tracker") or {} | |
| 181 | return tracker.get("name") | |
| 182 | ||
| 183 | ||
| 184 | def _resource_id_from_event(event: dict[str, Any]) -> str: | |
| 185 | ticket = event.get("ticket") or {} | |
| 186 | return str(ticket.get("ref") or ticket.get("id") or event["id"]) | |
| 187 | ||
| 188 | ||
| 189 | def _change_ticket_id(change: dict[str, Any], event: dict[str, Any]) -> str: | |
| 190 | ticket = change.get("ticket") or event.get("ticket") or {} | |
| 191 | return str(ticket.get("id") or event["id"]) | |
| 192 | ||
| 193 | ||
| 194 | def _normalize_event_change( | |
| 195 | *, | |
| 196 | settings: Settings, | |
| 197 | actor: str, | |
| 198 | event: dict[str, Any], | |
| 199 | change: dict[str, Any], | |
| 200 | occurred_at: datetime, | |
| 201 | ) -> NormalizedEvent | None: | |
| 202 | change_type = change.get("__typename") | |
| 203 | event_id = str(event["id"]) | |
| 204 | resource_id = _resource_id_from_event(event) | |
| 205 | repo_name = _repo_name_from_event(event) | |
| 206 | ticket_id = _change_ticket_id(change, event) | |
| 207 | ||
| 208 | if change_type == "Created" and _safe_nested_name(change.get("author")) == actor: | |
| 209 | return NormalizedEvent( | |
| 210 | service="todo", | |
| 211 | event_type="ticket_created", | |
| 212 | actor=actor, | |
| 213 | repo_name=repo_name, | |
| 214 | resource_id=resource_id, | |
| 215 | external_uid=f"todo:event:{event_id}:created:{ticket_id}", | |
| 216 | occurred_at=occurred_at, | |
| 217 | weight=settings.event_weights["ticket_created"], | |
| 218 | raw_payload_json={"event": event, "change": change}, | |
| 219 | ) | |
| 220 | ||
| 221 | if change_type == "Comment" and _safe_nested_name(change.get("author")) == actor: | |
| 222 | return NormalizedEvent( | |
| 223 | service="todo", | |
| 224 | event_type="ticket_comment", | |
| 225 | actor=actor, | |
| 226 | repo_name=repo_name, | |
| 227 | resource_id=resource_id, | |
| 228 | external_uid=f"todo:event:{event_id}:comment:{ticket_id}", | |
| 229 | occurred_at=occurred_at, | |
| 230 | weight=settings.event_weights["ticket_comment"], | |
| 231 | raw_payload_json={"event": event, "change": change}, | |
| 232 | ) | |
| 233 | ||
| 234 | if change_type == "StatusChange" and _safe_nested_name(change.get("editor")) == actor: | |
| 235 | new_status = change.get("newStatus") | |
| 236 | new_resolution = change.get("newResolution") | |
| 237 | if new_status in TICKET_CLOSED_STATUSES or new_resolution in TICKET_CLOSED_RESOLUTIONS: | |
| 238 | return NormalizedEvent( | |
| 239 | service="todo", | |
| 240 | event_type="ticket_closed", | |
| 241 | actor=actor, | |
| 242 | repo_name=repo_name, | |
| 243 | resource_id=resource_id, | |
| 244 | external_uid=f"todo:event:{event_id}:closed:{ticket_id}", | |
| 245 | occurred_at=occurred_at, | |
| 246 | weight=settings.event_weights["ticket_closed"], | |
| 247 | raw_payload_json={"event": event, "change": change}, | |
| 248 | ) | |
| 249 | ||
| 250 | return None | |
| 251 | ||
| 252 | ||
| 253 | def _extract_event_cursor_page(data: dict[str, Any]) -> tuple[str | None, list[dict[str, Any]], str]: | |
| 254 | me = data.get("me") or {} | |
| 255 | canonical_actor = me.get("canonicalName") | |
| 256 | if not canonical_actor: | |
| 257 | raise TodoSchemaError("todo.sr.ht response did not include me.canonicalName") | |
| 258 | ||
| 259 | events = data.get("events") or {} | |
| 260 | results = events.get("results") or [] | |
| 261 | if not isinstance(results, list): | |
| 262 | raise TodoSchemaError("todo.sr.ht response did not include events.results") | |
| 263 | ||
| 264 | return events.get("cursor"), [event for event in results if isinstance(event, dict)], canonical_actor | |
| 265 | ||
| 266 | ||
| 267 | @dataclass(slots=True) | |
| 268 | class TodoPollResult: | |
| 269 | events: list[NormalizedEvent] | |
| 270 | cursor: str | |
| 271 | ||
| 272 | ||
| 273 | class TodoIngestionService: | |
| 274 | """Fetches todo.sr.ht activity from the authenticated event feed and normalizes it.""" | |
| 275 | ||
| 276 | service_name = "todo" | |
| 277 | ||
| 278 | def __init__(self, client: SourceHutGraphQLClient, settings: Settings) -> None: | |
| 279 | self.client = client | |
| 280 | self.settings = settings | |
| 281 | ||
| 282 | def fetch_recent_events(self, actor: str, since: datetime | None = None) -> TodoPollResult: | |
| 283 | since_dt = ensure_utc(since or (datetime.now(tz=UTC) - timedelta(days=30))) | |
| 284 | cursor_time = datetime.now(tz=UTC).isoformat() | |
| 285 | feed_result = self._fetch_from_activity_feed(actor=actor, since=since_dt) | |
| 286 | if feed_result.events: | |
| 287 | return TodoPollResult(events=feed_result.events, cursor=cursor_time) | |
| 288 | ||
| 289 | logger.info( | |
| 290 | "todo activity feed returned no normalized events for actor=%s; falling back to tracker crawl", | |
| 291 | actor, | |
| 292 | ) | |
| 293 | tracker_events = self._fetch_from_trackers(actor=actor, since=since_dt) | |
| 294 | return TodoPollResult(events=tracker_events, cursor=cursor_time) | |
| 295 | ||
| 296 | def _fetch_from_activity_feed(self, actor: str, since: datetime) -> TodoPollResult: | |
| 297 | events: list[NormalizedEvent] = [] | |
| 298 | cursor: str | None = None | |
| 299 | effective_actor = actor | |
| 300 | ||
| 301 | for _ in range(10): | |
| 302 | data = self.client.execute(TODO_ACTIVITY_QUERY, {"cursor": cursor}) | |
| 303 | cursor, page_events, canonical_actor = _extract_event_cursor_page(data) | |
| 304 | effective_actor = actor or canonical_actor | |
| 305 | logger.info( | |
| 306 | "todo page fetched for actor=%s canonical_actor=%s events=%s next_cursor=%s since=%s", | |
| 307 | actor, | |
| 308 | canonical_actor, | |
| 309 | len(page_events), | |
| 310 | bool(cursor), | |
| 311 | since.isoformat(), | |
| 312 | ) | |
| 313 | ||
| 314 | stop_paging = False | |
| 315 | for event in page_events: | |
| 316 | occurred_at = parse_datetime(event["created"]) | |
| 317 | event_id = str(event.get("id")) | |
| 318 | resource_id = _resource_id_from_event(event) | |
| 319 | repo_name = _repo_name_from_event(event) | |
| 320 | change_list = event.get("changes") or [] | |
| 321 | logger.info( | |
| 322 | "todo event id=%s resource=%s repo=%s occurred_at=%s changes=%s", | |
| 323 | event_id, | |
| 324 | resource_id, | |
| 325 | repo_name, | |
| 326 | occurred_at.isoformat(), | |
| 327 | len(change_list) if isinstance(change_list, list) else "unknown", | |
| 328 | ) | |
| 329 | if occurred_at < since: | |
| 330 | stop_paging = True | |
| 331 | logger.info( | |
| 332 | "todo event id=%s skipped because occurred_at=%s is before since=%s", | |
| 333 | event_id, | |
| 334 | occurred_at.isoformat(), | |
| 335 | since.isoformat(), | |
| 336 | ) | |
| 337 | continue | |
| 338 | ||
| 339 | for change in change_list: | |
| 340 | if not isinstance(change, dict): | |
| 341 | logger.info("todo event id=%s skipped non-dict change payload", event_id) | |
| 342 | continue | |
| 343 | change_type = change.get("__typename") | |
| 344 | author = _safe_nested_name(change.get("author")) | |
| 345 | editor = _safe_nested_name(change.get("editor")) | |
| 346 | logger.info( | |
| 347 | "todo change event_id=%s type=%s eventType=%s author=%s editor=%s newStatus=%s newResolution=%s", | |
| 348 | event_id, | |
| 349 | change_type, | |
| 350 | change.get("eventType"), | |
| 351 | author, | |
| 352 | editor, | |
| 353 | change.get("newStatus"), | |
| 354 | change.get("newResolution"), | |
| 355 | ) | |
| 356 | normalized = _normalize_event_change( | |
| 357 | settings=self.settings, | |
| 358 | actor=effective_actor, | |
| 359 | event=event, | |
| 360 | change=change, | |
| 361 | occurred_at=occurred_at, | |
| 362 | ) | |
| 363 | if normalized is not None: | |
| 364 | logger.info( | |
| 365 | "todo change accepted event_id=%s normalized_type=%s external_uid=%s", | |
| 366 | event_id, | |
| 367 | normalized.event_type, | |
| 368 | normalized.external_uid, | |
| 369 | ) | |
| 370 | events.append(normalized) | |
| 371 | else: | |
| 372 | logger.info( | |
| 373 | "todo change skipped event_id=%s for actor=%s", | |
| 374 | event_id, | |
| 375 | effective_actor, | |
| 376 | ) | |
| 377 | ||
| 378 | if stop_paging or not cursor: | |
| 379 | break | |
| 380 | ||
| 381 | logger.info("todo poll complete for actor=%s normalized_events=%s", effective_actor, len(events)) | |
| 382 | return TodoPollResult(events=events, cursor=datetime.now(tz=UTC).isoformat()) | |
| 383 | ||
| 384 | def _fetch_from_trackers(self, actor: str, since: datetime) -> list[NormalizedEvent]: | |
| 385 | data = self.client.execute(TODO_TRACKERS_QUERY, {"cursor": None}) | |
| 386 | me = data.get("me") or {} | |
| 387 | canonical_actor = me.get("canonicalName") or actor | |
| 388 | trackers = ((me.get("trackers") or {}).get("results")) or [] | |
| 389 | logger.info("todo tracker crawl actor=%s trackers=%s", canonical_actor, len(trackers)) | |
| 390 | ||
| 391 | events: list[NormalizedEvent] = [] | |
| 392 | for tracker in trackers: | |
| 393 | if not isinstance(tracker, dict): | |
| 394 | continue | |
| 395 | tracker_id = tracker.get("id") | |
| 396 | tracker_rid = tracker.get("rid") | |
| 397 | tracker_name = tracker.get("name") | |
| 398 | tracker_events = self._fetch_tracker_tickets( | |
| 399 | actor=canonical_actor, | |
| 400 | tracker_id=str(tracker_id), | |
| 401 | tracker_rid=str(tracker_rid), | |
| 402 | tracker_name=tracker_name, | |
| 403 | since=since, | |
| 404 | ) | |
| 405 | events.extend(tracker_events) | |
| 406 | ||
| 407 | logger.info("todo tracker crawl complete actor=%s normalized_events=%s", canonical_actor, len(events)) | |
| 408 | return events | |
| 409 | ||
| 410 | def _fetch_tracker_tickets( | |
| 411 | self, | |
| 412 | *, | |
| 413 | actor: str, | |
| 414 | tracker_id: str, | |
| 415 | tracker_rid: str, | |
| 416 | tracker_name: str | None, | |
| 417 | since: datetime, | |
| 418 | ) -> list[NormalizedEvent]: | |
| 419 | events: list[NormalizedEvent] = [] | |
| 420 | cursor: str | None = None | |
| 421 | ||
| 422 | for _ in range(50): | |
| 423 | data = self.client.execute( | |
| 424 | TODO_TRACKER_TICKETS_QUERY, | |
| 425 | {"trackerRid": tracker_rid, "cursor": cursor}, | |
| 426 | ) | |
| 427 | tracker = data.get("tracker") or {} | |
| 428 | tickets_page = tracker.get("tickets") or {} | |
| 429 | tickets = tickets_page.get("results") or [] | |
| 430 | cursor = tickets_page.get("cursor") | |
| 431 | logger.info( | |
| 432 | "todo tracker=%s ticket page count=%s next_cursor=%s", | |
| 433 | tracker_name or tracker_id, | |
| 434 | len(tickets), | |
| 435 | bool(cursor), | |
| 436 | ) | |
| 437 | ||
| 438 | stop_paging = False | |
| 439 | for ticket in tickets: | |
| 440 | if not isinstance(ticket, dict): | |
| 441 | continue | |
| 442 | updated_at = parse_datetime(ticket["updated"]) | |
| 443 | if updated_at < since: | |
| 444 | stop_paging = True | |
| 445 | logger.info( | |
| 446 | "todo ticket ref=%s skipped because updated_at=%s is before since=%s", | |
| 447 | ticket.get("ref"), | |
| 448 | updated_at.isoformat(), | |
| 449 | since.isoformat(), | |
| 450 | ) | |
| 451 | continue | |
| 452 | events.extend( | |
| 453 | self._fetch_ticket_events( | |
| 454 | actor=actor, | |
| 455 | tracker_id=tracker_id, | |
| 456 | tracker_rid=tracker_rid, | |
| 457 | tracker_name=tracker_name, | |
| 458 | ticket=ticket, | |
| 459 | since=since, | |
| 460 | ) | |
| 461 | ) | |
| 462 | ||
| 463 | if stop_paging or not cursor: | |
| 464 | break | |
| 465 | ||
| 466 | return events | |
| 467 | ||
| 468 | def _fetch_ticket_events( | |
| 469 | self, | |
| 470 | *, | |
| 471 | actor: str, | |
| 472 | tracker_id: str, | |
| 473 | tracker_rid: str, | |
| 474 | tracker_name: str | None, | |
| 475 | ticket: dict[str, Any], | |
| 476 | since: datetime, | |
| 477 | ) -> list[NormalizedEvent]: | |
| 478 | events: list[NormalizedEvent] = [] | |
| 479 | cursor: str | None = None | |
| 480 | ticket_id = int(ticket["id"]) | |
| 481 | ticket_ref = str(ticket.get("ref") or ticket_id) | |
| 482 | ||
| 483 | for _ in range(50): | |
| 484 | data = self.client.execute( | |
| 485 | TODO_TICKET_EVENTS_QUERY, | |
| 486 | {"trackerRid": tracker_rid, "ticketId": ticket_id, "cursor": cursor}, | |
| 487 | ) | |
| 488 | tracker = data.get("tracker") or {} | |
| 489 | ticket_payload = (tracker.get("ticket") or {}) if isinstance(tracker, dict) else {} | |
| 490 | event_page = ticket_payload.get("events") or {} | |
| 491 | page_events = event_page.get("results") or [] | |
| 492 | cursor = event_page.get("cursor") | |
| 493 | logger.info( | |
| 494 | "todo ticket events ref=%s tracker=%s count=%s next_cursor=%s", | |
| 495 | ticket_ref, | |
| 496 | tracker_name or tracker_id, | |
| 497 | len(page_events), | |
| 498 | bool(cursor), | |
| 499 | ) | |
| 500 | ||
| 501 | stop_paging = False | |
| 502 | for event in page_events: | |
| 503 | if not isinstance(event, dict): | |
| 504 | continue | |
| 505 | event["ticket"] = { | |
| 506 | "id": ticket_payload.get("id", ticket.get("id")), | |
| 507 | "ref": ticket_payload.get("ref", ticket_ref), | |
| 508 | "status": ticket_payload.get("status", ticket.get("status")), | |
| 509 | "resolution": ticket_payload.get("resolution", ticket.get("resolution")), | |
| 510 | "tracker": {"name": tracker_name}, | |
| 511 | } | |
| 512 | occurred_at = parse_datetime(event["created"]) | |
| 513 | event_id = str(event.get("id")) | |
| 514 | change_list = event.get("changes") or [] | |
| 515 | logger.info( | |
| 516 | "todo ticket event ref=%s id=%s occurred_at=%s changes=%s", | |
| 517 | ticket_ref, | |
| 518 | event_id, | |
| 519 | occurred_at.isoformat(), | |
| 520 | len(change_list) if isinstance(change_list, list) else "unknown", | |
| 521 | ) | |
| 522 | if occurred_at < since: | |
| 523 | stop_paging = True | |
| 524 | continue | |
| 525 | ||
| 526 | for change in change_list: | |
| 527 | if not isinstance(change, dict): | |
| 528 | continue | |
| 529 | logger.info( | |
| 530 | "todo ticket change ref=%s event_id=%s type=%s eventType=%s author=%s editor=%s newStatus=%s newResolution=%s", | |
| 531 | ticket_ref, | |
| 532 | event_id, | |
| 533 | change.get("__typename"), | |
| 534 | change.get("eventType"), | |
| 535 | _safe_nested_name(change.get("author")), | |
| 536 | _safe_nested_name(change.get("editor")), | |
| 537 | change.get("newStatus"), | |
| 538 | change.get("newResolution"), | |
| 539 | ) | |
| 540 | normalized = _normalize_event_change( | |
| 541 | settings=self.settings, | |
| 542 | actor=actor, | |
| 543 | event=event, | |
| 544 | change=change, | |
| 545 | occurred_at=occurred_at, | |
| 546 | ) | |
| 547 | if normalized is not None: | |
| 548 | logger.info( | |
| 549 | "todo ticket change accepted ref=%s event_id=%s normalized_type=%s", | |
| 550 | ticket_ref, | |
| 551 | event_id, | |
| 552 | normalized.event_type, | |
| 553 | ) | |
| 554 | events.append(normalized) | |
| 555 | ||
| 556 | if stop_paging or not cursor: | |
| 557 | break | |
| 558 | ||
| 559 | return events | |
src/srht_contrib/utils/__init__.py added +1
| @@ -0,0 +1 @@ | ||
| 1 | """Utility helpers.""" | |
src/srht_contrib/utils/dates.py added +37
| @@ -0,0 +1,37 @@ | ||
| 1 | from __future__ import annotations | |
| 2 | ||
| 3 | from datetime import UTC, date, datetime, time, timedelta | |
| 4 | ||
| 5 | ||
| 6 | def ensure_utc(dt: datetime) -> datetime: | |
| 7 | if dt.tzinfo is None: | |
| 8 | return dt.replace(tzinfo=UTC) | |
| 9 | return dt.astimezone(UTC) | |
| 10 | ||
| 11 | ||
| 12 | def parse_date(value: str) -> date: | |
| 13 | return date.fromisoformat(value) | |
| 14 | ||
| 15 | ||
| 16 | def parse_datetime(value: str) -> datetime: | |
| 17 | normalized = value.replace("Z", "+00:00") | |
| 18 | return ensure_utc(datetime.fromisoformat(normalized)) | |
| 19 | ||
| 20 | ||
| 21 | def date_range(start: date, end: date) -> list[date]: | |
| 22 | days: list[date] = [] | |
| 23 | current = start | |
| 24 | while current <= end: | |
| 25 | days.append(current) | |
| 26 | current += timedelta(days=1) | |
| 27 | return days | |
| 28 | ||
| 29 | ||
| 30 | def year_bounds(year: int) -> tuple[date, date]: | |
| 31 | return date(year, 1, 1), date(year, 12, 31) | |
| 32 | ||
| 33 | ||
| 34 | def date_to_utc_bounds(value: date) -> tuple[datetime, datetime]: | |
| 35 | start = datetime.combine(value, time.min, tzinfo=UTC) | |
| 36 | end = datetime.combine(value, time.max, tzinfo=UTC) | |
| 37 | return start, end | |
src/srht_contrib/utils/identity.py added +27
| @@ -0,0 +1,27 @@ | ||
| 1 | from __future__ import annotations | |
| 2 | ||
| 3 | from sqlalchemy import select | |
| 4 | from sqlalchemy.orm import Session | |
| 5 | ||
| 6 | from srht_contrib.models import ActorAlias | |
| 7 | ||
| 8 | ||
| 9 | class ActorIdentityResolver: | |
| 10 | def __init__(self, configured_aliases: dict[str, list[str]] | None = None) -> None: | |
| 11 | self.configured_aliases = configured_aliases or {} | |
| 12 | ||
| 13 | def canonicalize(self, actor: str, db: Session | None = None) -> str: | |
| 14 | normalized = actor.strip() | |
| 15 | if not normalized: | |
| 16 | return normalized | |
| 17 | ||
| 18 | for canonical, aliases in self.configured_aliases.items(): | |
| 19 | if normalized == canonical or normalized in aliases: | |
| 20 | return canonical | |
| 21 | ||
| 22 | if db is not None: | |
| 23 | alias = db.scalar(select(ActorAlias).where(ActorAlias.alias == normalized)) | |
| 24 | if alias is not None: | |
| 25 | return alias.canonical_actor | |
| 26 | ||
| 27 | return normalized | |
src/srht_contrib/utils/repositories.py added +31
| @@ -0,0 +1,31 @@ | ||
| 1 | from __future__ import annotations | |
| 2 | ||
| 3 | from fastapi import HTTPException, status | |
| 4 | ||
| 5 | ||
| 6 | def canonicalize_repository_name(actor: str, repo_name: str) -> str: | |
| 7 | normalized_actor = actor.strip() | |
| 8 | normalized_repo_name = repo_name.strip() | |
| 9 | ||
| 10 | if not normalized_actor: | |
| 11 | raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail="Actor must not be blank.") | |
| 12 | if not normalized_repo_name: | |
| 13 | raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail="Repository name must not be blank.") | |
| 14 | ||
| 15 | if "/" in normalized_repo_name: | |
| 16 | owner, name = normalized_repo_name.split("/", 1) | |
| 17 | owner = owner.strip() | |
| 18 | name = name.strip() | |
| 19 | if not owner or not name or "/" in name: | |
| 20 | raise HTTPException( | |
| 21 | status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, | |
| 22 | detail="Repository name must be `name` or `~owner/name`.", | |
| 23 | ) | |
| 24 | canonical_owner = owner if owner.startswith("~") else f"~{owner}" | |
| 25 | return f"{canonical_owner}/{name}" | |
| 26 | ||
| 27 | if "/" in normalized_actor: | |
| 28 | raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail="Actor must be a canonical sr.ht user.") | |
| 29 | ||
| 30 | canonical_actor = normalized_actor if normalized_actor.startswith("~") else f"~{normalized_actor}" | |
| 31 | return f"{canonical_actor}/{normalized_repo_name}" | |
tests/conftest.py added +58
| @@ -0,0 +1,58 @@ | ||
| 1 | from __future__ import annotations | |
| 2 | ||
| 3 | from collections.abc import Generator | |
| 4 | ||
| 5 | import pytest | |
| 6 | from fastapi.testclient import TestClient | |
| 7 | from sqlalchemy.orm import Session, sessionmaker | |
| 8 | ||
| 9 | from srht_contrib.config import Settings | |
| 10 | from srht_contrib.db import Base, make_engine | |
| 11 | from srht_contrib.main import create_app | |
| 12 | ||
| 13 | ||
| 14 | @pytest.fixture() | |
| 15 | def settings() -> Settings: | |
| 16 | return Settings( | |
| 17 | API_KEY="test-api-key", | |
| 18 | ENABLE_SCHEDULER=False, | |
| 19 | SRHT_TOKEN="test-token", | |
| 20 | DATABASE_URL="sqlite://", | |
| 21 | TODO_SRHT_ENDPOINT="https://todo.sr.ht/query", | |
| 22 | GIT_SRHT_ENDPOINT="https://git.sr.ht/query", | |
| 23 | DEFAULT_ACTOR="~ccleberg", | |
| 24 | POLL_INTERVAL_SECONDS=3600, | |
| 25 | GIT_TRACKED_REPOSITORIES=[], | |
| 26 | ) | |
| 27 | ||
| 28 | ||
| 29 | @pytest.fixture() | |
| 30 | def db_engine(settings: Settings): | |
| 31 | engine = make_engine(settings) | |
| 32 | Base.metadata.create_all(bind=engine) | |
| 33 | try: | |
| 34 | yield engine | |
| 35 | finally: | |
| 36 | Base.metadata.drop_all(bind=engine) | |
| 37 | ||
| 38 | ||
| 39 | @pytest.fixture() | |
| 40 | def session_factory(db_engine) -> sessionmaker[Session]: | |
| 41 | return sessionmaker(bind=db_engine, autoflush=False, autocommit=False, expire_on_commit=False) | |
| 42 | ||
| 43 | ||
| 44 | @pytest.fixture() | |
| 45 | def db_session(session_factory: sessionmaker[Session]) -> Generator[Session, None, None]: | |
| 46 | session = session_factory() | |
| 47 | try: | |
| 48 | yield session | |
| 49 | finally: | |
| 50 | session.close() | |
| 51 | ||
| 52 | ||
| 53 | @pytest.fixture() | |
| 54 | def client(settings: Settings, db_engine, session_factory: sessionmaker[Session]) -> Generator[TestClient, None, None]: | |
| 55 | app = create_app(settings, engine=db_engine, session_factory=session_factory) | |
| 56 | with TestClient(app) as test_client: | |
| 57 | test_client.headers.update({"X-API-Key": settings.api_key}) | |
| 58 | yield test_client | |
tests/test_aggregator.py added +79
| @@ -0,0 +1,79 @@ | ||
| 1 | from datetime import UTC, datetime, date | |
| 2 | ||
| 3 | from sqlalchemy.orm import Session | |
| 4 | ||
| 5 | from srht_contrib.models import ContributionEvent | |
| 6 | from srht_contrib.services.aggregator import ContributionAggregator | |
| 7 | ||
| 8 | ||
| 9 | def test_aggregator_zero_fills_days(db_session: Session) -> None: | |
| 10 | db_session.add( | |
| 11 | ContributionEvent( | |
| 12 | service="todo", | |
| 13 | event_type="ticket_created", | |
| 14 | actor="~ccleberg", | |
| 15 | repo_name=None, | |
| 16 | resource_id="1", | |
| 17 | external_uid="todo:1:created", | |
| 18 | occurred_at=datetime(2026, 3, 28, 12, 0, tzinfo=UTC), | |
| 19 | weight=1.0, | |
| 20 | raw_payload_json=None, | |
| 21 | ) | |
| 22 | ) | |
| 23 | db_session.commit() | |
| 24 | ||
| 25 | calendar = ContributionAggregator().build_calendar( | |
| 26 | db_session, | |
| 27 | "~ccleberg", | |
| 28 | date(2026, 3, 28), | |
| 29 | date(2026, 3, 30), | |
| 30 | ) | |
| 31 | ||
| 32 | assert [day.model_dump() for day in calendar.days] == [ | |
| 33 | {"date": date(2026, 3, 28), "count": 1, "score": 1.0}, | |
| 34 | {"date": date(2026, 3, 29), "count": 0, "score": 0.0}, | |
| 35 | {"date": date(2026, 3, 30), "count": 0, "score": 0.0}, | |
| 36 | ] | |
| 37 | ||
| 38 | ||
| 39 | def test_stats_calculation(db_session: Session) -> None: | |
| 40 | db_session.add_all( | |
| 41 | [ | |
| 42 | ContributionEvent( | |
| 43 | service="todo", | |
| 44 | event_type="ticket_created", | |
| 45 | actor="~ccleberg", | |
| 46 | repo_name=None, | |
| 47 | resource_id="1", | |
| 48 | external_uid="todo:1:created", | |
| 49 | occurred_at=datetime(2026, 3, 28, 12, 0, tzinfo=UTC), | |
| 50 | weight=1.0, | |
| 51 | raw_payload_json=None, | |
| 52 | ), | |
| 53 | ContributionEvent( | |
| 54 | service="todo", | |
| 55 | event_type="ticket_comment", | |
| 56 | actor="~ccleberg", | |
| 57 | repo_name=None, | |
| 58 | resource_id="1", | |
| 59 | external_uid="todo:1:comment:1", | |
| 60 | occurred_at=datetime(2026, 3, 29, 12, 0, tzinfo=UTC), | |
| 61 | weight=0.5, | |
| 62 | raw_payload_json=None, | |
| 63 | ), | |
| 64 | ] | |
| 65 | ) | |
| 66 | db_session.commit() | |
| 67 | ||
| 68 | stats = ContributionAggregator().build_stats( | |
| 69 | db_session, | |
| 70 | "~ccleberg", | |
| 71 | date(2026, 3, 28), | |
| 72 | date(2026, 3, 30), | |
| 73 | ) | |
| 74 | ||
| 75 | assert stats.total_events == 2 | |
| 76 | assert stats.total_score == 1.5 | |
| 77 | assert stats.active_days == 2 | |
| 78 | assert stats.longest_streak == 2 | |
| 79 | assert stats.current_streak == 0 | |
tests/test_contributions_api.py added +120
| @@ -0,0 +1,120 @@ | ||
| 1 | from datetime import UTC, datetime | |
| 2 | ||
| 3 | from fastapi.testclient import TestClient | |
| 4 | ||
| 5 | from srht_contrib.main import create_app | |
| 6 | from srht_contrib.models import ContributionEvent | |
| 7 | ||
| 8 | ||
| 9 | def test_api_routes_require_api_key(settings, db_engine, session_factory) -> None: | |
| 10 | app = create_app(settings, engine=db_engine, session_factory=session_factory) | |
| 11 | with TestClient(app) as open_client: | |
| 12 | response = open_client.get("/health") | |
| 13 | assert response.status_code == 200 | |
| 14 | ||
| 15 | with TestClient(app) as unauthorized: | |
| 16 | unauthorized_response = unauthorized.get("/api/contributions/~ccleberg?from=2026-03-28&to=2026-03-30") | |
| 17 | assert unauthorized_response.status_code == 401 | |
| 18 | ||
| 19 | with TestClient(app) as invalid: | |
| 20 | invalid.headers.update({"X-API-Key": "wrong-key"}) | |
| 21 | invalid_response = invalid.get("/api/repositories") | |
| 22 | assert invalid_response.status_code == 401 | |
| 23 | ||
| 24 | ||
| 25 | def test_contributions_api_returns_zero_filled_range(client: TestClient, db_session) -> None: | |
| 26 | db_session.add( | |
| 27 | ContributionEvent( | |
| 28 | service="todo", | |
| 29 | event_type="ticket_created", | |
| 30 | actor="~ccleberg", | |
| 31 | repo_name=None, | |
| 32 | resource_id="1", | |
| 33 | external_uid="todo:ticket:1:created", | |
| 34 | occurred_at=datetime(2026, 3, 30, 10, 0, tzinfo=UTC), | |
| 35 | weight=1.0, | |
| 36 | raw_payload_json=None, | |
| 37 | ) | |
| 38 | ) | |
| 39 | db_session.commit() | |
| 40 | ||
| 41 | response = client.get("/api/contributions/~ccleberg?from=2026-03-28&to=2026-03-30") | |
| 42 | ||
| 43 | assert response.status_code == 200 | |
| 44 | assert response.json()["days"] == [ | |
| 45 | {"date": "2026-03-28", "count": 0, "score": 0.0}, | |
| 46 | {"date": "2026-03-29", "count": 0, "score": 0.0}, | |
| 47 | {"date": "2026-03-30", "count": 1, "score": 1.0}, | |
| 48 | ] | |
| 49 | ||
| 50 | ||
| 51 | def test_contribution_stats_api(client: TestClient, db_session) -> None: | |
| 52 | db_session.add_all( | |
| 53 | [ | |
| 54 | ContributionEvent( | |
| 55 | service="todo", | |
| 56 | event_type="ticket_created", | |
| 57 | actor="~ccleberg", | |
| 58 | repo_name=None, | |
| 59 | resource_id="1", | |
| 60 | external_uid="todo:ticket:1:created", | |
| 61 | occurred_at=datetime(2026, 3, 29, 10, 0, tzinfo=UTC), | |
| 62 | weight=1.0, | |
| 63 | raw_payload_json=None, | |
| 64 | ), | |
| 65 | ContributionEvent( | |
| 66 | service="todo", | |
| 67 | event_type="ticket_comment", | |
| 68 | actor="~ccleberg", | |
| 69 | repo_name=None, | |
| 70 | resource_id="1", | |
| 71 | external_uid="todo:ticket:1:comment:2", | |
| 72 | occurred_at=datetime(2026, 3, 30, 11, 0, tzinfo=UTC), | |
| 73 | weight=0.5, | |
| 74 | raw_payload_json=None, | |
| 75 | ), | |
| 76 | ] | |
| 77 | ) | |
| 78 | db_session.commit() | |
| 79 | ||
| 80 | response = client.get("/api/contributions/~ccleberg/stats?from=2026-03-28&to=2026-03-30") | |
| 81 | ||
| 82 | assert response.status_code == 200 | |
| 83 | assert response.json()["total_events"] == 2 | |
| 84 | assert response.json()["total_score"] == 1.5 | |
| 85 | assert response.json()["longest_streak"] == 2 | |
| 86 | assert response.json()["current_streak"] == 2 | |
| 87 | ||
| 88 | ||
| 89 | def test_invalid_date_input_returns_400(client: TestClient) -> None: | |
| 90 | response = client.get("/api/contributions/~ccleberg?from=2026-13-01&to=2026-03-30") | |
| 91 | ||
| 92 | assert response.status_code == 400 | |
| 93 | assert "Invalid date format" in response.json()["detail"] | |
| 94 | ||
| 95 | ||
| 96 | def test_contribution_routes_use_settings_backed_alias_resolution(settings, db_engine, session_factory) -> None: | |
| 97 | alias_settings = settings.model_copy(update={"actor_aliases_json": {"~ccleberg": ["cmc@example.com"]}}) | |
| 98 | app = create_app(alias_settings, engine=db_engine, session_factory=session_factory) | |
| 99 | with session_factory() as session: | |
| 100 | session.add( | |
| 101 | ContributionEvent( | |
| 102 | service="todo", | |
| 103 | event_type="ticket_created", | |
| 104 | actor="~ccleberg", | |
| 105 | repo_name=None, | |
| 106 | resource_id="1", | |
| 107 | external_uid="todo:alias:1", | |
| 108 | occurred_at=datetime(2026, 3, 30, 10, 0, tzinfo=UTC), | |
| 109 | weight=1.0, | |
| 110 | raw_payload_json=None, | |
| 111 | ) | |
| 112 | ) | |
| 113 | session.commit() | |
| 114 | ||
| 115 | with TestClient(app) as client: | |
| 116 | client.headers.update({"X-API-Key": alias_settings.api_key}) | |
| 117 | response = client.get("/api/contributions/cmc@example.com?from=2026-03-30&to=2026-03-30") | |
| 118 | ||
| 119 | assert response.status_code == 200 | |
| 120 | assert response.json()["actor"] == "~ccleberg" | |
tests/test_health.py added +15
| @@ -0,0 +1,15 @@ | ||
| 1 | from fastapi.testclient import TestClient | |
| 2 | ||
| 3 | from srht_contrib.main import create_app | |
| 4 | ||
| 5 | ||
| 6 | def test_health_endpoint() -> None: | |
| 7 | with TestClient(create_app()) as client: | |
| 8 | response = client.get("/health") | |
| 9 | ||
| 10 | assert response.status_code == 200 | |
| 11 | assert response.json() == {"status": "ok"} | |
| 12 | ||
| 13 | ||
| 14 | def test_scheduler_is_disabled_by_default(client: TestClient) -> None: | |
| 15 | assert client.app.state.scheduler is None | |
tests/test_ingestion.py added +337
| @@ -0,0 +1,337 @@ | ||
| 1 | from datetime import UTC, datetime | |
| 2 | ||
| 3 | from sqlalchemy import select | |
| 4 | ||
| 5 | from srht_contrib.config import Settings | |
| 6 | from srht_contrib.jobs.poller import PollerService | |
| 7 | from srht_contrib.models import SyncState, TrackedRepository | |
| 8 | from srht_contrib.schemas import NormalizedEvent | |
| 9 | from srht_contrib.services.git import GitIngestionService, GitPollResult | |
| 10 | from srht_contrib.services.todo import TodoIngestionService, TodoPollResult | |
| 11 | ||
| 12 | ||
| 13 | class StubClient: | |
| 14 | def __init__(self, payload: dict | None = None, payloads_by_query: dict[str, dict] | None = None) -> None: | |
| 15 | self.payload = payload or {} | |
| 16 | self.payloads_by_query = payloads_by_query or {} | |
| 17 | self.calls: list[tuple[str, dict | None]] = [] | |
| 18 | ||
| 19 | def execute(self, query: str, variables: dict | None = None) -> dict: | |
| 20 | self.calls.append((query, variables)) | |
| 21 | for marker, payload in self.payloads_by_query.items(): | |
| 22 | if marker in query: | |
| 23 | return payload | |
| 24 | return self.payload | |
| 25 | ||
| 26 | ||
| 27 | class RecordingTodoService: | |
| 28 | service_name = "todo" | |
| 29 | ||
| 30 | def __init__(self, events_by_call: list[list[NormalizedEvent]]) -> None: | |
| 31 | self.events_by_call = events_by_call | |
| 32 | self.calls: list[datetime] = [] | |
| 33 | ||
| 34 | def fetch_recent_events(self, actor: str, since: datetime | None = None) -> TodoPollResult: | |
| 35 | assert since is not None | |
| 36 | self.calls.append(since) | |
| 37 | events = self.events_by_call.pop(0) | |
| 38 | return TodoPollResult(events=events, cursor="2026-03-31T00:00:00+00:00") | |
| 39 | ||
| 40 | ||
| 41 | class EmptyGitService: | |
| 42 | service_name = "git" | |
| 43 | ||
| 44 | def __init__(self) -> None: | |
| 45 | self.settings = Settings( | |
| 46 | SRHT_TOKEN="x", | |
| 47 | DATABASE_URL="sqlite://", | |
| 48 | DEFAULT_ACTOR="~ccleberg", | |
| 49 | TODO_SRHT_ENDPOINT="https://todo.sr.ht/query", | |
| 50 | GIT_SRHT_ENDPOINT="https://git.sr.ht/query", | |
| 51 | POLL_INTERVAL_SECONDS=60, | |
| 52 | ) | |
| 53 | ||
| 54 | def fetch_recent_events(self, actor: str, since: datetime | None = None, repositories=None) -> GitPollResult: | |
| 55 | return GitPollResult(events=[], cursor="2026-03-31T00:00:00+00:00") | |
| 56 | ||
| 57 | ||
| 58 | def make_settings(**overrides) -> Settings: | |
| 59 | values = { | |
| 60 | "API_KEY": "test-api-key", | |
| 61 | "ENABLE_SCHEDULER": False, | |
| 62 | "SRHT_TOKEN": "x", | |
| 63 | "DATABASE_URL": "sqlite://", | |
| 64 | "DEFAULT_ACTOR": "~ccleberg", | |
| 65 | "TODO_SRHT_ENDPOINT": "https://todo.sr.ht/query", | |
| 66 | "GIT_SRHT_ENDPOINT": "https://git.sr.ht/query", | |
| 67 | "POLL_INTERVAL_SECONDS": 60, | |
| 68 | } | |
| 69 | values.update(overrides) | |
| 70 | return Settings(**values) | |
| 71 | ||
| 72 | ||
| 73 | def test_todo_ingestion_is_idempotent(db_session) -> None: | |
| 74 | settings = make_settings() | |
| 75 | payload = { | |
| 76 | "me": {"canonicalName": "~ccleberg"}, | |
| 77 | "events": { | |
| 78 | "results": [ | |
| 79 | { | |
| 80 | "id": "1001", | |
| 81 | "created": "2026-03-29T10:00:00Z", | |
| 82 | "ticket": { | |
| 83 | "id": "123", | |
| 84 | "ref": "~ccleberg/todo/123", | |
| 85 | "status": "RESOLVED", | |
| 86 | "resolution": "CLOSED", | |
| 87 | "tracker": {"name": "todo"}, | |
| 88 | }, | |
| 89 | "changes": [ | |
| 90 | { | |
| 91 | "__typename": "Created", | |
| 92 | "eventType": "CREATED", | |
| 93 | "ticket": {"id": "123"}, | |
| 94 | "author": {"canonicalName": "~ccleberg"}, | |
| 95 | } | |
| 96 | ], | |
| 97 | }, | |
| 98 | { | |
| 99 | "id": "1002", | |
| 100 | "created": "2026-03-30T09:00:00Z", | |
| 101 | "ticket": { | |
| 102 | "id": "123", | |
| 103 | "ref": "~ccleberg/todo/123", | |
| 104 | "status": "RESOLVED", | |
| 105 | "resolution": "CLOSED", | |
| 106 | "tracker": {"name": "todo"}, | |
| 107 | }, | |
| 108 | "changes": [ | |
| 109 | { | |
| 110 | "__typename": "Comment", | |
| 111 | "eventType": "COMMENT", | |
| 112 | "ticket": {"id": "123"}, | |
| 113 | "author": {"canonicalName": "~ccleberg"}, | |
| 114 | } | |
| 115 | ], | |
| 116 | }, | |
| 117 | { | |
| 118 | "id": "1003", | |
| 119 | "created": "2026-03-30T10:00:00Z", | |
| 120 | "ticket": { | |
| 121 | "id": "123", | |
| 122 | "ref": "~ccleberg/todo/123", | |
| 123 | "status": "RESOLVED", | |
| 124 | "resolution": "CLOSED", | |
| 125 | "tracker": {"name": "todo"}, | |
| 126 | }, | |
| 127 | "changes": [ | |
| 128 | { | |
| 129 | "__typename": "StatusChange", | |
| 130 | "eventType": "STATUS_CHANGE", | |
| 131 | "ticket": {"id": "123"}, | |
| 132 | "editor": {"canonicalName": "~ccleberg"}, | |
| 133 | "oldStatus": "IN_PROGRESS", | |
| 134 | "newStatus": "RESOLVED", | |
| 135 | "oldResolution": "UNRESOLVED", | |
| 136 | "newResolution": "CLOSED", | |
| 137 | } | |
| 138 | ], | |
| 139 | }, | |
| 140 | ], | |
| 141 | "cursor": None, | |
| 142 | }, | |
| 143 | } | |
| 144 | ||
| 145 | todo_service = TodoIngestionService(StubClient(payload), settings) | |
| 146 | git_service = GitIngestionService(StubClient(payload={}), settings) | |
| 147 | poller = PollerService(todo_service=todo_service, git_service=git_service) | |
| 148 | ||
| 149 | first_inserted = poller.poll_all(db_session, "~ccleberg") | |
| 150 | second_inserted = poller.poll_all(db_session, "~ccleberg") | |
| 151 | ||
| 152 | assert first_inserted == 3 | |
| 153 | assert second_inserted == 0 | |
| 154 | ||
| 155 | ||
| 156 | def test_todo_ingestion_falls_back_to_tracker_crawl(db_session) -> None: | |
| 157 | settings = make_settings() | |
| 158 | client = StubClient( | |
| 159 | payloads_by_query={ | |
| 160 | "query TodoActivity": { | |
| 161 | "me": {"canonicalName": "~ccleberg"}, | |
| 162 | "events": {"results": [], "cursor": None}, | |
| 163 | }, | |
| 164 | "query TodoTrackers": { | |
| 165 | "me": { | |
| 166 | "canonicalName": "~ccleberg", | |
| 167 | "trackers": {"results": [{"id": "1", "rid": "tracker-rid", "name": "todo"}], "cursor": None}, | |
| 168 | } | |
| 169 | }, | |
| 170 | "query TodoTrackerTickets": { | |
| 171 | "tracker": { | |
| 172 | "id": "1", | |
| 173 | "name": "todo", | |
| 174 | "tickets": { | |
| 175 | "results": [ | |
| 176 | { | |
| 177 | "id": 123, | |
| 178 | "ref": "~ccleberg/todo/123", | |
| 179 | "created": "2026-03-29T09:00:00Z", | |
| 180 | "updated": "2026-03-30T09:00:00Z", | |
| 181 | "status": "RESOLVED", | |
| 182 | "resolution": "CLOSED", | |
| 183 | "submitter": {"canonicalName": "~ccleberg"}, | |
| 184 | } | |
| 185 | ], | |
| 186 | "cursor": None, | |
| 187 | }, | |
| 188 | } | |
| 189 | }, | |
| 190 | "query TodoTicketEvents": { | |
| 191 | "tracker": { | |
| 192 | "ticket": { | |
| 193 | "id": 123, | |
| 194 | "ref": "~ccleberg/todo/123", | |
| 195 | "status": "RESOLVED", | |
| 196 | "resolution": "CLOSED", | |
| 197 | "events": { | |
| 198 | "results": [ | |
| 199 | { | |
| 200 | "id": "evt-1", | |
| 201 | "created": "2026-03-30T09:00:00Z", | |
| 202 | "changes": [ | |
| 203 | { | |
| 204 | "__typename": "Comment", | |
| 205 | "eventType": "COMMENT", | |
| 206 | "ticket": {"id": "123"}, | |
| 207 | "author": {"canonicalName": "~ccleberg"}, | |
| 208 | } | |
| 209 | ], | |
| 210 | } | |
| 211 | ], | |
| 212 | "cursor": None, | |
| 213 | }, | |
| 214 | } | |
| 215 | } | |
| 216 | }, | |
| 217 | } | |
| 218 | ) | |
| 219 | todo_service = TodoIngestionService(client, settings) | |
| 220 | git_service = GitIngestionService(StubClient(payload={}), settings) | |
| 221 | poller = PollerService(todo_service=todo_service, git_service=git_service) | |
| 222 | ||
| 223 | inserted = poller.poll_all(db_session, "~ccleberg") | |
| 224 | ||
| 225 | assert inserted == 1 | |
| 226 | assert any("query TodoTrackers" in call[0] for call in client.calls) | |
| 227 | ||
| 228 | ||
| 229 | def test_unsupported_todo_changes_are_ignored(db_session) -> None: | |
| 230 | settings = make_settings() | |
| 231 | payload = { | |
| 232 | "me": {"canonicalName": "~ccleberg"}, | |
| 233 | "events": { | |
| 234 | "results": [ | |
| 235 | { | |
| 236 | "id": "1001", | |
| 237 | "created": "2026-03-29T10:00:00Z", | |
| 238 | "ticket": { | |
| 239 | "id": "123", | |
| 240 | "ref": "~ccleberg/todo/123", | |
| 241 | "status": "OPEN", | |
| 242 | "resolution": "UNRESOLVED", | |
| 243 | "tracker": {"name": "todo"}, | |
| 244 | }, | |
| 245 | "changes": [ | |
| 246 | {"__typename": "LabelUpdate", "eventType": "LABEL_UPDATE", "ticket": {"id": "123"}}, | |
| 247 | {"__typename": "TicketMention", "eventType": "TICKET_MENTION", "ticket": {"id": "123"}}, | |
| 248 | ], | |
| 249 | } | |
| 250 | ], | |
| 251 | "cursor": None, | |
| 252 | }, | |
| 253 | } | |
| 254 | ||
| 255 | todo_service = TodoIngestionService(StubClient(payload), settings) | |
| 256 | git_service = GitIngestionService(StubClient(payload={}), settings) | |
| 257 | poller = PollerService(todo_service=todo_service, git_service=git_service) | |
| 258 | ||
| 259 | inserted = poller.poll_all(db_session, "~ccleberg") | |
| 260 | ||
| 261 | assert inserted == 0 | |
| 262 | ||
| 263 | ||
| 264 | def test_git_ingestion_normalizes_commit_aliases_and_repository_names(db_session) -> None: | |
| 265 | settings = make_settings( | |
| 266 | ACTOR_ALIASES_JSON={"~ccleberg": ["cmc@example.com", "Chris Cleberg"]}, | |
| 267 | GIT_TRACKED_REPOSITORIES=["Hutch"], | |
| 268 | ) | |
| 269 | git_payload = { | |
| 270 | "user": { | |
| 271 | "repository": { | |
| 272 | "name": "Hutch", | |
| 273 | "owner": {"canonicalName": "~ccleberg"}, | |
| 274 | "log": { | |
| 275 | "results": [ | |
| 276 | { | |
| 277 | "id": "abc123", | |
| 278 | "shortId": "abc123", | |
| 279 | "author": { | |
| 280 | "name": "Chris Cleberg", | |
| 281 | "email": "cmc@example.com", | |
| 282 | "time": "2026-03-30T12:00:00Z", | |
| 283 | }, | |
| 284 | "committer": { | |
| 285 | "name": "Chris Cleberg", | |
| 286 | "email": "cmc@example.com", | |
| 287 | "time": "2026-03-30T12:00:00Z", | |
| 288 | }, | |
| 289 | "message": "Add contribution calendar", | |
| 290 | } | |
| 291 | ], | |
| 292 | "cursor": None, | |
| 293 | }, | |
| 294 | } | |
| 295 | } | |
| 296 | } | |
| 297 | ||
| 298 | todo_service = TodoIngestionService( | |
| 299 | StubClient(payload={"me": {"canonicalName": "~ccleberg"}, "events": {"results": [], "cursor": None}}), | |
| 300 | settings, | |
| 301 | ) | |
| 302 | git_service = GitIngestionService(StubClient(payloads_by_query={"query RepositoryLog": git_payload}), settings) | |
| 303 | poller = PollerService(todo_service=todo_service, git_service=git_service) | |
| 304 | ||
| 305 | inserted = poller.poll_all(db_session, "~ccleberg") | |
| 306 | ||
| 307 | assert inserted == 1 | |
| 308 | ||
| 309 | tracked_repositories = db_session.scalars(select(TrackedRepository.repo_name)).all() | |
| 310 | assert tracked_repositories == ["~ccleberg/Hutch"] | |
| 311 | ||
| 312 | ||
| 313 | def test_sync_overlap_reuses_cursor_window_and_suppresses_duplicates(db_session) -> None: | |
| 314 | event = NormalizedEvent( | |
| 315 | service="todo", | |
| 316 | event_type="ticket_created", | |
| 317 | actor="~ccleberg", | |
| 318 | repo_name="todo", | |
| 319 | resource_id="123", | |
| 320 | external_uid="todo:event:123:created:123", | |
| 321 | occurred_at=datetime(2026, 3, 30, 10, 0, tzinfo=UTC), | |
| 322 | weight=1.0, | |
| 323 | raw_payload_json=None, | |
| 324 | ) | |
| 325 | todo_service = RecordingTodoService(events_by_call=[[event], [event]]) | |
| 326 | poller = PollerService(todo_service=todo_service, git_service=EmptyGitService()) | |
| 327 | ||
| 328 | first_inserted = poller.poll_all(db_session, "~ccleberg") | |
| 329 | second_inserted = poller.poll_all(db_session, "~ccleberg") | |
| 330 | ||
| 331 | state = db_session.scalar(select(SyncState).where(SyncState.service == "todo").where(SyncState.actor == "~ccleberg")) | |
| 332 | ||
| 333 | assert first_inserted == 1 | |
| 334 | assert second_inserted == 0 | |
| 335 | assert state is not None | |
| 336 | assert len(todo_service.calls) == 2 | |
| 337 | assert todo_service.calls[1].isoformat() == "2026-03-30T00:00:00+00:00" | |
tests/test_migrations.py added +117
| @@ -0,0 +1,117 @@ | ||
| 1 | from pathlib import Path | |
| 2 | ||
| 3 | from alembic import command | |
| 4 | from alembic.config import Config | |
| 5 | from sqlalchemy import create_engine, inspect, text | |
| 6 | ||
| 7 | from srht_contrib.config import Settings | |
| 8 | ||
| 9 | ||
| 10 | def test_alembic_upgrade_creates_schema(tmp_path) -> None: | |
| 11 | database_path = tmp_path / "migrated.db" | |
| 12 | database_url = f"sqlite:///{database_path}" | |
| 13 | config = Config(str(Path(__file__).resolve().parent.parent / "alembic.ini")) | |
| 14 | config.set_main_option("script_location", str(Path(__file__).resolve().parent.parent / "alembic")) | |
| 15 | config.set_main_option("sqlalchemy.url", database_url) | |
| 16 | ||
| 17 | command.upgrade(config, "head") | |
| 18 | ||
| 19 | inspector = inspect(create_engine(database_url)) | |
| 20 | assert sorted(inspector.get_table_names()) == [ | |
| 21 | "actor_aliases", | |
| 22 | "alembic_version", | |
| 23 | "contribution_events", | |
| 24 | "sync_states", | |
| 25 | "tracked_repositories", | |
| 26 | ] | |
| 27 | ||
| 28 | ||
| 29 | def test_alembic_upgrade_adopts_legacy_schema(tmp_path) -> None: | |
| 30 | database_path = tmp_path / "legacy.db" | |
| 31 | database_url = f"sqlite:///{database_path}" | |
| 32 | engine = create_engine(database_url) | |
| 33 | with engine.begin() as connection: | |
| 34 | connection.execute( | |
| 35 | text( | |
| 36 | """ | |
| 37 | CREATE TABLE contribution_events ( | |
| 38 | id INTEGER NOT NULL PRIMARY KEY, | |
| 39 | service VARCHAR(32) NOT NULL, | |
| 40 | event_type VARCHAR(64) NOT NULL, | |
| 41 | actor VARCHAR(255) NOT NULL, | |
| 42 | repo_name VARCHAR(255), | |
| 43 | resource_id VARCHAR(255) NOT NULL, | |
| 44 | external_uid VARCHAR(255) NOT NULL, | |
| 45 | occurred_at DATETIME NOT NULL, | |
| 46 | weight FLOAT NOT NULL, | |
| 47 | raw_payload_json JSON, | |
| 48 | CONSTRAINT uq_contribution_event_service_uid UNIQUE (service, external_uid) | |
| 49 | ) | |
| 50 | """ | |
| 51 | ) | |
| 52 | ) | |
| 53 | connection.execute(text("CREATE INDEX ix_contribution_events_actor_occurred_at ON contribution_events (actor, occurred_at)")) | |
| 54 | connection.execute(text("CREATE INDEX ix_contribution_events_service_occurred_at ON contribution_events (service, occurred_at)")) | |
| 55 | connection.execute( | |
| 56 | text( | |
| 57 | """ | |
| 58 | CREATE TABLE sync_states ( | |
| 59 | id INTEGER NOT NULL PRIMARY KEY, | |
| 60 | service VARCHAR(32) NOT NULL, | |
| 61 | actor VARCHAR(255) NOT NULL, | |
| 62 | cursor_value TEXT, | |
| 63 | updated_at DATETIME NOT NULL, | |
| 64 | CONSTRAINT uq_sync_state_service_actor UNIQUE (service, actor) | |
| 65 | ) | |
| 66 | """ | |
| 67 | ) | |
| 68 | ) | |
| 69 | connection.execute( | |
| 70 | text( | |
| 71 | """ | |
| 72 | CREATE TABLE tracked_repositories ( | |
| 73 | id INTEGER NOT NULL PRIMARY KEY, | |
| 74 | service VARCHAR(32) NOT NULL, | |
| 75 | repo_name VARCHAR(255) NOT NULL, | |
| 76 | actor VARCHAR(255), | |
| 77 | CONSTRAINT uq_tracked_repository_service_name UNIQUE (service, repo_name) | |
| 78 | ) | |
| 79 | """ | |
| 80 | ) | |
| 81 | ) | |
| 82 | connection.execute( | |
| 83 | text( | |
| 84 | """ | |
| 85 | INSERT INTO tracked_repositories (id, service, repo_name, actor) | |
| 86 | VALUES (1, 'git', 'Hutch', NULL) | |
| 87 | """ | |
| 88 | ) | |
| 89 | ) | |
| 90 | connection.execute( | |
| 91 | text( | |
| 92 | """ | |
| 93 | CREATE TABLE actor_aliases ( | |
| 94 | id INTEGER NOT NULL PRIMARY KEY, | |
| 95 | canonical_actor VARCHAR(255) NOT NULL, | |
| 96 | alias VARCHAR(255) NOT NULL, | |
| 97 | CONSTRAINT uq_actor_alias_alias UNIQUE (alias) | |
| 98 | ) | |
| 99 | """ | |
| 100 | ) | |
| 101 | ) | |
| 102 | ||
| 103 | config = Config(str(Path(__file__).resolve().parent.parent / "alembic.ini")) | |
| 104 | config.set_main_option("script_location", str(Path(__file__).resolve().parent.parent / "alembic")) | |
| 105 | config.set_main_option("sqlalchemy.url", database_url) | |
| 106 | ||
| 107 | command.upgrade(config, "head") | |
| 108 | ||
| 109 | inspector = inspect(create_engine(database_url)) | |
| 110 | columns = {column["name"]: column for column in inspector.get_columns("tracked_repositories")} | |
| 111 | unique_constraints = {constraint["name"] for constraint in inspector.get_unique_constraints("tracked_repositories")} | |
| 112 | with create_engine(database_url).connect() as connection: | |
| 113 | actor = connection.execute(text("SELECT actor FROM tracked_repositories WHERE id = 1")).scalar_one() | |
| 114 | ||
| 115 | assert columns["actor"]["nullable"] is False | |
| 116 | assert "uq_tracked_repository_service_actor_name" in unique_constraints | |
| 117 | assert actor == Settings().default_actor | |
tests/test_polling_api.py added +71
| @@ -0,0 +1,71 @@ | ||
| 1 | from datetime import UTC, datetime | |
| 2 | ||
| 3 | from fastapi.testclient import TestClient | |
| 4 | ||
| 5 | from srht_contrib.config import Settings | |
| 6 | from srht_contrib.main import create_app | |
| 7 | from srht_contrib.models import ContributionEvent | |
| 8 | from srht_contrib.services.srht_client import SourceHutClientError | |
| 9 | ||
| 10 | ||
| 11 | class _Closable: | |
| 12 | def close(self) -> None: | |
| 13 | return None | |
| 14 | ||
| 15 | ||
| 16 | class InsertingPoller: | |
| 17 | def __init__(self) -> None: | |
| 18 | service = type("Service", (), {"client": _Closable()})() | |
| 19 | self.todo_service = service | |
| 20 | self.git_service = service | |
| 21 | ||
| 22 | def poll_all(self, db, actor: str) -> int: | |
| 23 | db.add( | |
| 24 | ContributionEvent( | |
| 25 | service="todo", | |
| 26 | event_type="ticket_created", | |
| 27 | actor=actor, | |
| 28 | repo_name="todo", | |
| 29 | resource_id="1", | |
| 30 | external_uid="todo:manual:1", | |
| 31 | occurred_at=datetime(2026, 3, 30, 10, 0, tzinfo=UTC), | |
| 32 | weight=1.0, | |
| 33 | raw_payload_json=None, | |
| 34 | ) | |
| 35 | ) | |
| 36 | db.commit() | |
| 37 | return 1 | |
| 38 | ||
| 39 | ||
| 40 | class FailingPoller: | |
| 41 | def __init__(self) -> None: | |
| 42 | service = type("Service", (), {"client": _Closable()})() | |
| 43 | self.todo_service = service | |
| 44 | self.git_service = service | |
| 45 | ||
| 46 | def poll_all(self, db, actor: str) -> int: | |
| 47 | raise SourceHutClientError("boom") | |
| 48 | ||
| 49 | ||
| 50 | def test_manual_poll_uses_same_database_session(settings: Settings, db_engine, session_factory) -> None: | |
| 51 | app = create_app(settings, engine=db_engine, session_factory=session_factory, poller=InsertingPoller()) | |
| 52 | with TestClient(app) as client: | |
| 53 | client.headers.update({"X-API-Key": settings.api_key}) | |
| 54 | ||
| 55 | poll_response = client.post("/api/contributions/poll?actor=~ccleberg") | |
| 56 | calendar_response = client.get("/api/contributions/~ccleberg?from=2026-03-30&to=2026-03-30") | |
| 57 | ||
| 58 | assert poll_response.status_code == 200 | |
| 59 | assert poll_response.json()["inserted_events"] == 1 | |
| 60 | assert calendar_response.status_code == 200 | |
| 61 | assert calendar_response.json()["days"] == [{"date": "2026-03-30", "count": 1, "score": 1.0}] | |
| 62 | ||
| 63 | ||
| 64 | def test_manual_poll_maps_sourcehut_failures_to_502(settings: Settings, db_engine, session_factory) -> None: | |
| 65 | app = create_app(settings, engine=db_engine, session_factory=session_factory, poller=FailingPoller()) | |
| 66 | with TestClient(app) as client: | |
| 67 | client.headers.update({"X-API-Key": settings.api_key}) | |
| 68 | response = client.post("/api/contributions/poll?actor=~ccleberg") | |
| 69 | ||
| 70 | assert response.status_code == 502 | |
| 71 | assert "SourceHut polling failed" in response.json()["detail"] | |
tests/test_repositories_api.py added +77
| @@ -0,0 +1,77 @@ | ||
| 1 | from fastapi.testclient import TestClient | |
| 2 | ||
| 3 | ||
| 4 | def test_create_list_get_update_delete_tracked_repositories(client: TestClient) -> None: | |
| 5 | create_response = client.post( | |
| 6 | "/api/repositories", | |
| 7 | json={"actor": "~ccleberg", "repo_name": "Hutch"}, | |
| 8 | ) | |
| 9 | ||
| 10 | assert create_response.status_code == 201 | |
| 11 | assert create_response.json()["repo_name"] == "~ccleberg/Hutch" | |
| 12 | ||
| 13 | repository_id = create_response.json()["id"] | |
| 14 | ||
| 15 | get_response = client.get(f"/api/repositories/{repository_id}") | |
| 16 | assert get_response.status_code == 200 | |
| 17 | assert get_response.json()["repo_name"] == "~ccleberg/Hutch" | |
| 18 | ||
| 19 | update_response = client.patch( | |
| 20 | f"/api/repositories/{repository_id}", | |
| 21 | json={"repo_name": "~ccleberg/cleberg.net"}, | |
| 22 | ) | |
| 23 | assert update_response.status_code == 200 | |
| 24 | assert update_response.json()["repo_name"] == "~ccleberg/cleberg.net" | |
| 25 | ||
| 26 | list_response = client.get("/api/repositories?actor=~ccleberg") | |
| 27 | assert list_response.status_code == 200 | |
| 28 | assert list_response.json() == [ | |
| 29 | { | |
| 30 | "id": repository_id, | |
| 31 | "service": "git", | |
| 32 | "actor": "~ccleberg", | |
| 33 | "repo_name": "~ccleberg/cleberg.net", | |
| 34 | } | |
| 35 | ] | |
| 36 | ||
| 37 | delete_response = client.delete(f"/api/repositories/{repository_id}") | |
| 38 | assert delete_response.status_code == 204 | |
| 39 | ||
| 40 | missing_response = client.get(f"/api/repositories/{repository_id}") | |
| 41 | assert missing_response.status_code == 404 | |
| 42 | ||
| 43 | ||
| 44 | def test_repository_validation_and_conflicts(client: TestClient) -> None: | |
| 45 | invalid_response = client.post( | |
| 46 | "/api/repositories", | |
| 47 | json={"actor": "~ccleberg", "repo_name": " "}, | |
| 48 | ) | |
| 49 | assert invalid_response.status_code == 422 | |
| 50 | ||
| 51 | first_response = client.post( | |
| 52 | "/api/repositories", | |
| 53 | json={"actor": "~ccleberg", "repo_name": "Hutch"}, | |
| 54 | ) | |
| 55 | second_response = client.post( | |
| 56 | "/api/repositories", | |
| 57 | json={"actor": "~ccleberg", "repo_name": "~ccleberg/Hutch"}, | |
| 58 | ) | |
| 59 | ||
| 60 | assert first_response.status_code == 201 | |
| 61 | assert second_response.status_code == 409 | |
| 62 | ||
| 63 | ||
| 64 | def test_same_repository_can_be_tracked_by_different_actors(client: TestClient) -> None: | |
| 65 | first_response = client.post( | |
| 66 | "/api/repositories", | |
| 67 | json={"actor": "~ccleberg", "repo_name": "Hutch"}, | |
| 68 | ) | |
| 69 | second_response = client.post( | |
| 70 | "/api/repositories", | |
| 71 | json={"actor": "~other", "repo_name": "Hutch"}, | |
| 72 | ) | |
| 73 | ||
| 74 | assert first_response.status_code == 201 | |
| 75 | assert second_response.status_code == 201 | |
| 76 | assert first_response.json()["repo_name"] == "~ccleberg/Hutch" | |
| 77 | assert second_response.json()["repo_name"] == "~other/Hutch" | |
tests/test_srht_client.py added +56
| @@ -0,0 +1,56 @@ | ||
| 1 | import httpx | |
| 2 | import pytest | |
| 3 | ||
| 4 | from srht_contrib.services.srht_client import SourceHutClientError, SourceHutGraphQLClient | |
| 5 | ||
| 6 | ||
| 7 | def test_graphql_client_retries_http_5xx_and_succeeds() -> None: | |
| 8 | attempts = {"count": 0} | |
| 9 | ||
| 10 | def handler(request: httpx.Request) -> httpx.Response: | |
| 11 | attempts["count"] += 1 | |
| 12 | if attempts["count"] == 1: | |
| 13 | return httpx.Response(502, json={"error": "bad gateway"}) | |
| 14 | return httpx.Response(200, json={"data": {"ok": True}}) | |
| 15 | ||
| 16 | client = SourceHutGraphQLClient( | |
| 17 | "https://todo.sr.ht/query", | |
| 18 | "token", | |
| 19 | transport=httpx.MockTransport(handler), | |
| 20 | ) | |
| 21 | ||
| 22 | data = client.execute("query Ping { ping }") | |
| 23 | ||
| 24 | assert data == {"ok": True} | |
| 25 | assert attempts["count"] == 2 | |
| 26 | client.close() | |
| 27 | ||
| 28 | ||
| 29 | def test_graphql_client_raises_for_graphql_errors() -> None: | |
| 30 | client = SourceHutGraphQLClient( | |
| 31 | "https://todo.sr.ht/query", | |
| 32 | "token", | |
| 33 | transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"errors": [{"message": "nope"}]})), | |
| 34 | ) | |
| 35 | ||
| 36 | with pytest.raises(SourceHutClientError): | |
| 37 | client.execute("query Ping { ping }") | |
| 38 | ||
| 39 | client.close() | |
| 40 | ||
| 41 | ||
| 42 | def test_graphql_client_raises_for_network_errors() -> None: | |
| 43 | def handler(request: httpx.Request) -> httpx.Response: | |
| 44 | raise httpx.ConnectError("offline", request=request) | |
| 45 | ||
| 46 | client = SourceHutGraphQLClient( | |
| 47 | "https://todo.sr.ht/query", | |
| 48 | "token", | |
| 49 | max_retries=0, | |
| 50 | transport=httpx.MockTransport(handler), | |
| 51 | ) | |
| 52 | ||
| 53 | with pytest.raises(SourceHutClientError): | |
| 54 | client.execute("query Ping { ping }") | |
| 55 | ||
| 56 | client.close() | |