krz/hutch-stats

Server-side utility for calculating contributions for sourcehut users. contributions server sourcehut stats

Commit 1ae229d64b

1ae229d64bee19e94bbfd1872879b86496d68104

parent: fdfb7831fe

Verified · cmc

cmc <hello@cleberg.net> · 2026-05-07 03:09 UTC 

feat: index git activity across all SourceHut branches

Layout: unified · split

src/srht_contrib/services/git.py +160 −12
@@ -21,6 +21,23 @@ from srht_contrib.utils.identity import ActorIdentityResolver
21logger = logging.getLogger(__name__) 21logger = logging.getLogger(__name__)
22 22
23 23
24REPOSITORY_BRANCHES_QUERY = """
25query RepositoryBranches($username: String!, $repoName: String!, $cursor: Cursor) {
26 user(username: $username) {
27 repository(name: $repoName) {
28 references(cursor: $cursor) {
29 results {
30 name
31 target
32 }
33 cursor
34 }
35 }
36 }
37}
38""".strip()
39
40
24REPOSITORY_LOG_QUERY = """ 41REPOSITORY_LOG_QUERY = """
25query RepositoryLog($username: String!, $repoName: String!, $cursor: Cursor, $from: String) { 42query RepositoryLog($username: String!, $repoName: String!, $cursor: Cursor, $from: String) {
26 user(username: $username) { 43 user(username: $username) {
@@ -29,10 +46,6 @@ query RepositoryLog($username: String!, $repoName: String!, $cursor: Cursor, $fr
29 owner { 46 owner {
30 canonicalName 47 canonicalName
31 } 48 }
32 HEAD {
33 name
34 target
35 }
36 log(cursor: $cursor, from: $from) { 49 log(cursor: $cursor, from: $from) {
37 results { 50 results {
38 id 51 id
@@ -233,17 +246,80 @@ class GitIngestionService:
233 if state["current_repository"] is None: 246 if state["current_repository"] is None:
234 if not state["repository_queue"]: 247 if not state["repository_queue"]:
235 return BackfillBatchResult(events=[], cursor_state=None, complete=True) 248 return BackfillBatchResult(events=[], cursor_state=None, complete=True)
236 state["current_repository"] = {"name": state["repository_queue"].pop(0), "cursor": None} 249 state["current_repository"] = {
250 "name": state["repository_queue"].pop(0),
251 "reference_cursor": None,
252 "branches_loaded": False,
253 "branch_queue": [],
254 "current_branch": None,
255 }
237 256
238 repository_name = state["current_repository"]["name"] 257 repository_name = state["current_repository"]["name"]
239 owner, repo_name = self._split_repository(actor, repository_name) 258 owner, repo_name = self._split_repository(actor, repository_name)
259 current_repository = state["current_repository"]
260 current_repository.setdefault("reference_cursor", None)
261 current_repository.setdefault("branches_loaded", False)
262 current_repository.setdefault("branch_queue", [])
263 current_repository.setdefault("current_branch", None)
264 if "cursor" in current_repository:
265 current_repository.pop("cursor", None)
266
267 if not current_repository["branches_loaded"]:
268 data = self.client.execute(
269 REPOSITORY_BRANCHES_QUERY,
270 {"username": owner, "repoName": repo_name, "cursor": current_repository["reference_cursor"]},
271 )
272 user = data.get("user") or {}
273 repository = user.get("repository") or {}
274 references_page = repository.get("references") or {}
275 references = references_page.get("results") or []
276 current_repository["reference_cursor"] = references_page.get("cursor")
277 current_repository["branches_loaded"] = not bool(current_repository["reference_cursor"])
278 known_branches = set(current_repository["branch_queue"])
279 if current_repository["current_branch"]:
280 known_branches.add(current_repository["current_branch"]["name"])
281 for reference in references:
282 if not isinstance(reference, dict):
283 continue
284 branch_name = reference.get("name")
285 if not self._is_branch_reference(branch_name):
286 continue
287 if branch_name not in known_branches:
288 current_repository["branch_queue"].append(branch_name)
289 known_branches.add(branch_name)
290 current_repository["branch_queue"] = sorted(current_repository["branch_queue"])
291 logger.info(
292 "git backfill branch discovery actor=%s repository=%s page_count=%s branches=%s next_cursor=%s",
293 actor,
294 repository_name,
295 len(references),
296 len(current_repository["branch_queue"]),
297 bool(current_repository["reference_cursor"]),
298 )
299 if (
300 current_repository["branches_loaded"]
301 and not current_repository["branch_queue"]
302 and not current_repository["current_branch"]
303 ):
304 state["current_repository"] = None
305 complete = state["discovery_complete"] and not state["repository_queue"] and not state["current_repository"]
306 return BackfillBatchResult(events=[], cursor_state=None if complete else state, complete=complete)
307
308 if current_repository["current_branch"] is None:
309 if not current_repository["branch_queue"]:
310 state["current_repository"] = None
311 complete = state["discovery_complete"] and not state["repository_queue"]
312 return BackfillBatchResult(events=[], cursor_state=None if complete else state, complete=complete)
313 current_repository["current_branch"] = {"name": current_repository["branch_queue"].pop(0), "cursor": None}
314
315 current_branch = current_repository["current_branch"]
240 data = self.client.execute( 316 data = self.client.execute(
241 REPOSITORY_LOG_QUERY, 317 REPOSITORY_LOG_QUERY,
242 { 318 {
243 "username": owner, 319 "username": owner,
244 "repoName": repo_name, 320 "repoName": repo_name,
245 "cursor": state["current_repository"]["cursor"], 321 "cursor": current_branch["cursor"],
246 "from": "HEAD", 322 "from": current_branch["name"],
247 }, 323 },
248 ) 324 )
249 user = data.get("user") or {} 325 user = data.get("user") or {}
@@ -264,16 +340,17 @@ class GitIngestionService:
264 if normalized is not None: 340 if normalized is not None:
265 events.append(normalized) 341 events.append(normalized)
266 logger.info( 342 logger.info(
267 "git backfill actor=%s repository=%s commits=%s next_cursor=%s", 343 "git backfill actor=%s repository=%s branch=%s commits=%s next_cursor=%s",
268 actor, 344 actor,
269 repository_name, 345 repository_name,
346 current_branch["name"],
270 len(commits), 347 len(commits),
271 bool(next_cursor), 348 bool(next_cursor),
272 ) 349 )
273 if next_cursor and not stop_repository: 350 if next_cursor and not stop_repository:
274 state["current_repository"]["cursor"] = next_cursor 351 current_branch["cursor"] = next_cursor
275 else: 352 else:
276 state["current_repository"] = None 353 current_repository["current_branch"] = None
277 354
278 complete = state["discovery_complete"] and not state["repository_queue"] and not state["current_repository"] 355 complete = state["discovery_complete"] and not state["repository_queue"] and not state["current_repository"]
279 return BackfillBatchResult(events=events, cursor_state=None if complete else state, complete=complete) 356 return BackfillBatchResult(events=events, cursor_state=None if complete else state, complete=complete)
@@ -355,6 +432,72 @@ class GitIngestionService:
355 owner: str, 432 owner: str,
356 repo_name: str, 433 repo_name: str,
357 since: datetime, 434 since: datetime,
435 ) -> list[NormalizedEvent]:
436 events: list[NormalizedEvent] = []
437 seen_event_uids: set[str] = set()
438 branches = self._fetch_repository_branches(owner=owner, repo_name=repo_name)
439 if not branches:
440 logger.info("git repository=%s/%s has no branch references", owner, repo_name)
441 return events
442
443 for branch in branches:
444 branch_events = self._fetch_repository_branch_commits(
445 actor=actor,
446 owner=owner,
447 repo_name=repo_name,
448 branch=branch,
449 since=since,
450 )
451 for event in branch_events:
452 if event.external_uid in seen_event_uids:
453 continue
454 seen_event_uids.add(event.external_uid)
455 events.append(event)
456
457 return events
458
459 def _fetch_repository_branches(self, *, owner: str, repo_name: str) -> list[str]:
460 branches: list[str] = []
461 cursor: str | None = None
462
463 for _ in range(50):
464 data = self.client.execute(
465 REPOSITORY_BRANCHES_QUERY,
466 {"username": owner, "repoName": repo_name, "cursor": cursor},
467 )
468 user = data.get("user") or {}
469 repository = user.get("repository") or {}
470 references_page = repository.get("references") or {}
471 references = references_page.get("results") or []
472 cursor = references_page.get("cursor")
473 logger.info(
474 "git repository=%s/%s branch page count=%s next_cursor=%s",
475 owner,
476 repo_name,
477 len(references),
478 bool(cursor),
479 )
480
481 for reference in references:
482 if not isinstance(reference, dict):
483 continue
484 branch_name = reference.get("name")
485 if self._is_branch_reference(branch_name):
486 branches.append(branch_name)
487
488 if not cursor:
489 break
490
491 return sorted(set(branches))
492
493 def _fetch_repository_branch_commits(
494 self,
495 *,
496 actor: str,
497 owner: str,
498 repo_name: str,
499 branch: str,
500 since: datetime,
358 ) -> list[NormalizedEvent]: 501 ) -> list[NormalizedEvent]:
359 events: list[NormalizedEvent] = [] 502 events: list[NormalizedEvent] = []
360 cursor: str | None = None 503 cursor: str | None = None
@@ -362,7 +505,7 @@ class GitIngestionService:
362 for _ in range(50): 505 for _ in range(50):
363 data = self.client.execute( 506 data = self.client.execute(
364 REPOSITORY_LOG_QUERY, 507 REPOSITORY_LOG_QUERY,
365 {"username": owner, "repoName": repo_name, "cursor": cursor, "from": "HEAD"}, 508 {"username": owner, "repoName": repo_name, "cursor": cursor, "from": branch},
366 ) 509 )
367 user = data.get("user") or {} 510 user = data.get("user") or {}
368 repository = user.get("repository") or {} 511 repository = user.get("repository") or {}
@@ -370,9 +513,10 @@ class GitIngestionService:
370 commits = log_page.get("results") or [] 513 commits = log_page.get("results") or []
371 cursor = log_page.get("cursor") 514 cursor = log_page.get("cursor")
372 logger.info( 515 logger.info(
373 "git repository=%s/%s commit page count=%s next_cursor=%s", 516 "git repository=%s/%s branch=%s commit page count=%s next_cursor=%s",
374 owner, 517 owner,
375 repo_name, 518 repo_name,
519 branch,
376 len(commits), 520 len(commits),
377 bool(cursor), 521 bool(cursor),
378 ) 522 )
@@ -416,6 +560,10 @@ class GitIngestionService:
416 560
417 return events 561 return events
418 562
563 @staticmethod
564 def _is_branch_reference(reference_name: Any) -> bool:
565 return isinstance(reference_name, str) and reference_name.startswith("refs/heads/")
566
419 def _normalize_commit( 567 def _normalize_commit(
420 self, 568 self,
421 *, 569 *,
tests/test_git_repository_cache.py +14
@@ -40,6 +40,19 @@ def make_settings(**overrides) -> Settings:
40 return Settings(**values) 40 return Settings(**values)
41 41
42 42
43def branch_payload(*branches: str) -> dict:
44 return {
45 "user": {
46 "repository": {
47 "references": {
48 "results": [{"name": branch, "target": "abc123"} for branch in branches],
49 "cursor": None,
50 }
51 }
52 }
53 }
54
55
43def test_git_poll_reuses_cached_discovered_repositories(db_session) -> None: 56def test_git_poll_reuses_cached_discovered_repositories(db_session) -> None:
44 settings = make_settings() 57 settings = make_settings()
45 client = StubClient( 58 client = StubClient(
@@ -58,6 +71,7 @@ def test_git_poll_reuses_cached_discovered_repositories(db_session) -> None:
58 } 71 }
59 } 72 }
60 }, 73 },
74 "query RepositoryBranches": branch_payload("refs/heads/main"),
61 "query RepositoryLog": { 75 "query RepositoryLog": {
62 "user": { 76 "user": {
63 "repository": { 77 "repository": {
tests/test_ingestion.py +61 −37
@@ -202,6 +202,19 @@ def make_settings(**overrides) -> Settings:
202 return Settings(**values) 202 return Settings(**values)
203 203
204 204
205def branch_payload(*branches: str) -> dict:
206 return {
207 "user": {
208 "repository": {
209 "references": {
210 "results": [{"name": branch, "target": "abc123"} for branch in branches],
211 "cursor": None,
212 }
213 }
214 }
215 }
216
217
205def test_todo_ingestion_is_idempotent(db_session) -> None: 218def test_todo_ingestion_is_idempotent(db_session) -> None:
206 settings = make_settings() 219 settings = make_settings()
207 payload = { 220 payload = {
@@ -431,7 +444,15 @@ def test_git_ingestion_normalizes_commit_aliases_and_repository_names(db_session
431 StubClient(payload={"me": {"canonicalName": "~ccleberg"}, "events": {"results": [], "cursor": None}}), 444 StubClient(payload={"me": {"canonicalName": "~ccleberg"}, "events": {"results": [], "cursor": None}}),
432 settings, 445 settings,
433 ) 446 )
434 git_service = GitIngestionService(StubClient(payloads_by_query={"query RepositoryLog": git_payload}), settings) 447 git_service = GitIngestionService(
448 StubClient(
449 payloads_by_query={
450 "query RepositoryBranches": branch_payload("refs/heads/main"),
451 "query RepositoryLog": git_payload,
452 }
453 ),
454 settings,
455 )
435 poller = PollerService(todo_service=todo_service, git_service=git_service, settings=settings) 456 poller = PollerService(todo_service=todo_service, git_service=git_service, settings=settings)
436 457
437 inserted = poller.poll_all(db_session, "~ccleberg") 458 inserted = poller.poll_all(db_session, "~ccleberg")
@@ -463,6 +484,7 @@ def test_git_ingestion_auto_discovers_owned_repositories(db_session) -> None:
463 } 484 }
464 } 485 }
465 }, 486 },
487 "query RepositoryBranches": branch_payload("refs/heads/main"),
466 "query RepositoryLog": { 488 "query RepositoryLog": {
467 "user": { 489 "user": {
468 "repository": { 490 "repository": {
@@ -507,53 +529,55 @@ def test_git_ingestion_auto_discovers_owned_repositories(db_session) -> None:
507 assert any("query UserRepositories" in call[0] for call in client.calls) 529 assert any("query UserRepositories" in call[0] for call in client.calls)
508 530
509 531
510def test_git_ingestion_reads_repository_log_from_default_head_branch(db_session) -> None: 532def test_git_ingestion_reads_repository_logs_from_all_branches(db_session) -> None:
511 settings = make_settings( 533 settings = make_settings(
512 ACTOR_ALIASES_JSON={"~ccleberg": ["cmc@example.com", "Chris Cleberg"]}, 534 ACTOR_ALIASES_JSON={"~ccleberg": ["cmc@example.com", "Chris Cleberg"]},
513 GIT_TRACKED_REPOSITORIES=["Hutch"], 535 GIT_TRACKED_REPOSITORIES=["Hutch"],
514 ) 536 )
515 git_payload = { 537 client = StubClient(
516 "user": { 538 payloads_by_query={
517 "repository": { 539 "query RepositoryBranches": branch_payload(
518 "name": "Hutch", 540 "refs/heads/main",
519 "owner": {"canonicalName": "~ccleberg"}, 541 "refs/heads/trunk",
520 "HEAD": {"name": "refs/heads/trunk", "target": "abc123"}, 542 "refs/tags/v1.0.0",
521 "log": { 543 ),
522 "results": [ 544 "query RepositoryLog": {
523 { 545 "user": {
524 "id": "abc123", 546 "repository": {
525 "shortId": "abc123", 547 "name": "Hutch",
526 "author": { 548 "owner": {"canonicalName": "~ccleberg"},
527 "name": "Chris Cleberg", 549 "log": {
528 "email": "cmc@example.com", 550 "results": [
529 "time": "2026-03-30T12:00:00Z", 551 {
530 }, 552 "id": "abc123",
531 "committer": { 553 "shortId": "abc123",
532 "name": "Chris Cleberg", 554 "author": {
533 "email": "cmc@example.com", 555 "name": "Chris Cleberg",
534 "time": "2026-03-30T12:00:00Z", 556 "email": "cmc@example.com",
535 }, 557 "time": "2026-03-30T12:00:00Z",
536 "message": "Commit on non-standard default branch", 558 },
537 } 559 "committer": {
538 ], 560 "name": "Chris Cleberg",
539 "cursor": None, 561 "email": "cmc@example.com",
540 }, 562 "time": "2026-03-30T12:00:00Z",
541 } 563 },
564 "message": "Commit reachable from more than one branch",
565 }
566 ],
567 "cursor": None,
568 },
569 }
570 }
571 },
542 } 572 }
543 } 573 )
544 client = StubClient(payloads_by_query={"query RepositoryLog": git_payload})
545 git_service = GitIngestionService(client, settings) 574 git_service = GitIngestionService(client, settings)
546 575
547 result = git_service.fetch_recent_events("~ccleberg", since=datetime(2026, 3, 1, tzinfo=UTC)) 576 result = git_service.fetch_recent_events("~ccleberg", since=datetime(2026, 3, 1, tzinfo=UTC))
548 577
549 repository_log_calls = [call for call in client.calls if "query RepositoryLog" in call[0]] 578 repository_log_calls = [call for call in client.calls if "query RepositoryLog" in call[0]]
550 assert len(result.events) == 1 579 assert len(result.events) == 1
551 assert repository_log_calls[0][1] == { 580 assert [call[1]["from"] for call in repository_log_calls] == ["refs/heads/main", "refs/heads/trunk"]
552 "username": "ccleberg",
553 "repoName": "Hutch",
554 "cursor": None,
555 "from": "HEAD",
556 }
557 581
558 582
559def test_sync_overlap_reuses_cursor_window_and_suppresses_duplicates(db_session) -> None: 583def test_sync_overlap_reuses_cursor_window_and_suppresses_duplicates(db_session) -> None: