6149ca9c39 · krz/hutch-stats

API.md +13

 Contribution ranges:
 - Contribution read endpoints return zero-filled days, so clients do not need to patch missing dates.
+- Public contribution reads also register the actor for background indexing. A first lookup may therefore return an empty graph while the scheduler catches up.
 Repository names:
   "actor": "~your-user",
   "from": "2026-03-01",
   "to": "2026-04-15",
+  "is_indexed": false,
+  "last_polled_at": null,
+  "indexing_state": "pending",
   "days": [
     { "date": "2026-03-01", "count": 0, "score": 0.0 },
     { "date": "2026-03-02", "count": 3, "score": 2.5 }
 - `actor` string: canonical actor after alias resolution
 - `from` string: inclusive start date
 - `to` string: inclusive end date
+- `is_indexed` boolean: whether the service has already indexed activity for this actor
+- `last_polled_at` string or `null`: most recent successful poll time, if any
+- `indexing_state` string: one of `pending`, `indexed`, or `error`
 - `days` array:
   - `date` string `YYYY-MM-DD`
   - `count` integer contribution count for the day
   "actor": "~your-user",
   "from": "2026-03-01",
   "to": "2026-04-15",
+  "is_indexed": true,
+  "last_polled_at": "2026-04-11T18:05:00Z",
+  "indexing_state": "indexed",
   "total_events": 126,
   "total_score": 116.75,
   "active_days": 14,
 - `actor` string
 - `from` string
 - `to` string
+- `is_indexed` boolean
+- `last_polled_at` string or `null`
+- `indexing_state` string
 - `total_events` integer
 - `total_score` float
 - `active_days` integer

README.md +5 −1

 }
 ```
-Scheduled polling only runs when `ENABLE_SCHEDULER=true` and uses `DEFAULT_ACTOR`.
+Scheduled polling only runs when `ENABLE_SCHEDULER=true`. The scheduler seeds `DEFAULT_ACTOR` as an initial known actor, and public contribution reads register additional actors for later background polling.
 For `git.sr.ht`, tracked repositories are configured via `GIT_TRACKED_REPOSITORIES`. Entries may be either:
   "actor": "~your-user",
   "from": "2026-01-01",
   "to": "2026-03-30",
+  "is_indexed": true,
+  "last_polled_at": "2026-04-11T18:05:00Z",
+  "indexing_state": "indexed",
   "days": [
     {"date": "2026-03-28", "count": 3, "score": 3.5},
     {"date": "2026-03-29", "count": 0, "score": 0.0},
 - `git.sr.ht` polling is limited to repositories listed in `GIT_TRACKED_REPOSITORIES`
 - scheduled polling runs in-process, so it is not a distributed scheduler
+- newly requested actors are indexed asynchronously, so the first public read may be empty until a scheduler or manual poll runs
 - alias management is config-driven; there is no alias CRUD API yet
 - current deployment model is trusted-operator V1, not a public multi-tenant service

alembic/versions/20260411_0002_tracked_actors.py added +39

@@ -0,0 +1,39 @@
	1	"""tracked actors for lazy indexing"""
	2
	3	from __future__ import annotations
	4
	5	from alembic import op
	6	import sqlalchemy as sa
	7	from sqlalchemy import inspect
	8
	9
	10	revision = "20260411_0002"
	11	down_revision = "20260409_0001"
	12	branch_labels = None
	13	depends_on = None
	14
	15
	16	def _table_names() -> set[str]:
	17	return set(inspect(op.get_bind()).get_table_names())
	18
	19
	20	def upgrade() -> None:
	21	if "tracked_actors" in _table_names():
	22	return
	23
	24	op.create_table(
	25	"tracked_actors",
	26	sa.Column("id", sa.Integer(), primary_key=True),
	27	sa.Column("actor", sa.String(length=255), nullable=False),
	28	sa.Column("is_active", sa.Boolean(), nullable=False, server_default=sa.true()),
	29	sa.Column("last_requested_at", sa.DateTime(timezone=True), nullable=True),
	30	sa.Column("last_polled_at", sa.DateTime(timezone=True), nullable=True),
	31	sa.Column("last_poll_status", sa.String(length=32), nullable=True),
	32	sa.Column("last_poll_error", sa.Text(), nullable=True),
	33	sa.UniqueConstraint("actor", name="uq_tracked_actor_actor"),
	34	)
	35
	36
	37	def downgrade() -> None:
	38	if "tracked_actors" in _table_names():
	39	op.drop_table("tracked_actors")

src/srht_contrib/api/routes_contributions.py +10 −2

     year: int | None = Query(default=None, ge=1970, le=3000),
     from_date: str | None = Query(default=None, alias="from"),
     to_date: str | None = Query(default=None, alias="to"),
+    poller: PollerService = Depends(get_poller),
     db: Session = Depends(get_db),
     actor_identity_resolver: ActorIdentityResolver = Depends(get_actor_identity_resolver),
 ) -> ContributionCalendarResponse:
     start, end = _resolve_range(year, from_date, to_date)
     canonical_actor = actor_identity_resolver.canonicalize(actor, db=db)
-    return ContributionAggregator().build_calendar(db, canonical_actor, start, end)
+    poller.track_actor_request(db, canonical_actor)
+    response = ContributionAggregator().build_calendar(db, canonical_actor, start, end)
+    db.commit()
+    return response
 @router.get("/{actor}/stats", response_model=ContributionStatsResponse)
     year: int | None = Query(default=None, ge=1970, le=3000),
     from_date: str | None = Query(default=None, alias="from"),
     to_date: str | None = Query(default=None, alias="to"),
+    poller: PollerService = Depends(get_poller),
     db: Session = Depends(get_db),
     actor_identity_resolver: ActorIdentityResolver = Depends(get_actor_identity_resolver),
 ) -> ContributionStatsResponse:
     start, end = _resolve_range(year, from_date, to_date)
     canonical_actor = actor_identity_resolver.canonicalize(actor, db=db)
-    return ContributionAggregator().build_stats(db, canonical_actor, start, end)
+    poller.track_actor_request(db, canonical_actor)
+    response = ContributionAggregator().build_stats(db, canonical_actor, start, end)
+    db.commit()
+    return response
 @router.post("/poll", response_model=PollResponse, dependencies=[Depends(require_api_key)])

src/srht_contrib/jobs/poller.py +57 −2

	@@ -7,9 +7,10 @@ from sqlalchemy import select
7	from sqlalchemy.exc import IntegrityError	7	from sqlalchemy.exc import IntegrityError
8	from sqlalchemy.orm import Session	8	from sqlalchemy.orm import Session
9		9
10	from srht_contrib.models import ContributionEvent, SyncState, TrackedRepository	10	from srht_contrib.models import ContributionEvent, SyncState, TrackedActor, TrackedRepository
11	from srht_contrib.schemas import NormalizedEvent	11	from srht_contrib.schemas import NormalizedEvent
12	from srht_contrib.services.git import GitIngestionService	12	from srht_contrib.services.git import GitIngestionService
		13	from srht_contrib.services.srht_client import SourceHutClientError
13	from srht_contrib.services.todo import TodoIngestionService	14	from srht_contrib.services.todo import TodoIngestionService
14	from srht_contrib.utils.repositories import canonicalize_repository_name	15	from srht_contrib.utils.repositories import canonicalize_repository_name
15		16
	@@ -24,6 +25,52 @@ class PollerService:
24	self.git_service = git_service	25	self.git_service = git_service
25		26
26	def poll_all(self, db: Session, actor: str) -> int:	27	def poll_all(self, db: Session, actor: str) -> int:
		28	self.track_actor_request(db, actor, update_last_requested=False)
		29	try:
		30	inserted = self._poll_actor(db, actor)
		31	except Exception as exc:
		32	db.rollback()
		33	self._update_tracked_actor_poll_state(db, actor, status="error", error=str(exc))
		34	db.commit()
		35	raise
		36
		37	self._update_tracked_actor_poll_state(db, actor, status="indexed", error=None)
		38	db.commit()
		39	return inserted
		40
		41	def poll_tracked_actors(self, db: Session, default_actor: str \| None = None) -> dict[str, int]:
		42	if default_actor:
		43	self.track_actor_request(db, default_actor, update_last_requested=False)
		44	db.commit()
		45
		46	results: dict[str, int] = {}
		47	actors = db.scalars(
		48	select(TrackedActor.actor)
		49	.where(TrackedActor.is_active.is_(True))
		50	.order_by(TrackedActor.last_requested_at.is_(None), TrackedActor.last_requested_at.desc(), TrackedActor.actor)
		51	).all()
		52	for actor in actors:
		53	try:
		54	results[actor] = self.poll_all(db, actor)
		55	except SourceHutClientError:
		56	logger.exception("Scheduled poll failed for actor=%s", actor)
		57	except Exception:
		58	logger.exception("Unexpected scheduled poll failure for actor=%s", actor)
		59	return results
		60
		61	def track_actor_request(self, db: Session, actor: str, *, update_last_requested: bool = True) -> TrackedActor:
		62	tracked_actor = db.scalar(select(TrackedActor).where(TrackedActor.actor == actor))
		63	if tracked_actor is None:
		64	tracked_actor = TrackedActor(actor=actor, is_active=True)
		65	db.add(tracked_actor)
		66
		67	tracked_actor.is_active = True
		68	if update_last_requested:
		69	tracked_actor.last_requested_at = datetime.now(tz=UTC)
		70	db.flush()
		71	return tracked_actor
		72
		73	def _poll_actor(self, db: Session, actor: str) -> int:
27	inserted = 0	74	inserted = 0
28	inserted += self._poll_service(db, actor, self.todo_service.service_name, self.todo_service.fetch_recent_events)	75	inserted += self._poll_service(db, actor, self.todo_service.service_name, self.todo_service.fetch_recent_events)
29	self._sync_tracked_repositories(db, actor)	76	self._sync_tracked_repositories(db, actor)
	@@ -38,7 +85,6 @@ class PollerService:
38	repositories=git_repositories,	85	repositories=git_repositories,
39	),	86	),
40	)	87	)
41	db.commit()
42	return inserted	88	return inserted
43		89
44	def _poll_service(self, db: Session, actor: str, service_name: str, fetcher) -> int:	90	def _poll_service(self, db: Session, actor: str, service_name: str, fetcher) -> int:
	@@ -117,3 +163,12 @@ class PollerService:
117	.order_by(TrackedRepository.repo_name)	163	.order_by(TrackedRepository.repo_name)
118	).all()	164	).all()
119	return list(rows)	165	return list(rows)
		166
		167	def _update_tracked_actor_poll_state(self, db: Session, actor: str, status: str, error: str \| None) -> None:
		168	tracked_actor = self.track_actor_request(db, actor, update_last_requested=False)
		169	tracked_actor.last_poll_status = status
		170	tracked_actor.last_poll_error = error
		171	if status == "indexed":
		172	tracked_actor.last_polled_at = datetime.now(tz=UTC)
		173	db.add(tracked_actor)
		174	db.flush()

src/srht_contrib/main.py +1 −1

	@@ -91,7 +91,7 @@ def _scheduled_poll(app: FastAPI) -> None:
91	session_factory: sessionmaker[Session] = app.state.session_factory	91	session_factory: sessionmaker[Session] = app.state.session_factory
92	db = session_factory()	92	db = session_factory()
93	try:	93	try:
94	poller.poll_all(db, settings.default_actor)	94	poller.poll_tracked_actors(db, settings.default_actor)
95	finally:	95	finally:
96	db.close()	96	db.close()
97		97

src/srht_contrib/models.py +14 −1

	@@ -2,7 +2,7 @@ from __future__ import annotations
2		2
3	from datetime import datetime	3	from datetime import datetime
4		4
5	from sqlalchemy import JSON, DateTime, Float, Index, Integer, String, Text, UniqueConstraint	5	from sqlalchemy import JSON, Boolean, DateTime, Float, Index, Integer, String, Text, UniqueConstraint
6	from sqlalchemy.orm import Mapped, mapped_column	6	from sqlalchemy.orm import Mapped, mapped_column
7		7
8	from srht_contrib.db import Base	8	from srht_contrib.db import Base
	@@ -58,3 +58,16 @@ class ActorAlias(Base):
58	id: Mapped[int] = mapped_column(Integer, primary_key=True)	58	id: Mapped[int] = mapped_column(Integer, primary_key=True)
59	canonical_actor: Mapped[str] = mapped_column(String(255), nullable=False)	59	canonical_actor: Mapped[str] = mapped_column(String(255), nullable=False)
60	alias: Mapped[str] = mapped_column(String(255), nullable=False)	60	alias: Mapped[str] = mapped_column(String(255), nullable=False)
		61
		62
		63	class TrackedActor(Base):
		64	__tablename__ = "tracked_actors"
		65	__table_args__ = (UniqueConstraint("actor", name="uq_tracked_actor_actor"),)
		66
		67	id: Mapped[int] = mapped_column(Integer, primary_key=True)
		68	actor: Mapped[str] = mapped_column(String(255), nullable=False)
		69	is_active: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
		70	last_requested_at: Mapped[datetime \| None] = mapped_column(DateTime(timezone=True), nullable=True)
		71	last_polled_at: Mapped[datetime \| None] = mapped_column(DateTime(timezone=True), nullable=True)
		72	last_poll_status: Mapped[str \| None] = mapped_column(String(32), nullable=True)
		73	last_poll_error: Mapped[str \| None] = mapped_column(Text, nullable=True)

src/srht_contrib/schemas.py +9 −2

 from __future__ import annotations
 from datetime import date, datetime
+from typing import Literal
 from pydantic import BaseModel, Field, model_validator
     score: float
-class ContributionCalendarResponse(BaseModel):
+class ContributionIndexMetadata(BaseModel):
+    is_indexed: bool
+    last_polled_at: datetime | None = None
+    indexing_state: Literal["pending", "indexed", "error"]
+class ContributionCalendarResponse(ContributionIndexMetadata):
     actor: str
     from_date: date = Field(alias="from")
     to_date: date = Field(alias="to")
     model_config = {"populate_by_name": True}
-class ContributionStatsResponse(BaseModel):
+class ContributionStatsResponse(ContributionIndexMetadata):
     actor: str
     from_date: date = Field(alias="from")
     to_date: date = Field(alias="to")

src/srht_contrib/services/aggregator.py +37 −3

	@@ -6,8 +6,13 @@ from datetime import date
6	from sqlalchemy import func, select	6	from sqlalchemy import func, select
7	from sqlalchemy.orm import Session	7	from sqlalchemy.orm import Session
8		8
9	from srht_contrib.models import ContributionEvent	9	from srht_contrib.models import ContributionEvent, TrackedActor
10	from srht_contrib.schemas import ContributionCalendarResponse, ContributionDay, ContributionStatsResponse	10	from srht_contrib.schemas import (
		11	ContributionCalendarResponse,
		12	ContributionDay,
		13	ContributionIndexMetadata,
		14	ContributionStatsResponse,
		15	)
11	from srht_contrib.utils.dates import date_range, date_to_utc_bounds	16	from srht_contrib.utils.dates import date_range, date_to_utc_bounds
12		17
13		18
	@@ -30,7 +35,14 @@ class ContributionAggregator:
30	)	35	)
31	for day in date_range(start, end)	36	for day in date_range(start, end)
32	]	37	]
33	return ContributionCalendarResponse(actor=actor, from_date=start, to_date=end, days=days)	38	metadata = self._index_metadata(db, actor)
		39	return ContributionCalendarResponse(
		40	actor=actor,
		41	from_date=start,
		42	to_date=end,
		43	days=days,
		44	**metadata.model_dump(),
		45	)
34		46
35	def build_stats(self, db: Session, actor: str, start: date, end: date) -> ContributionStatsResponse:	47	def build_stats(self, db: Session, actor: str, start: date, end: date) -> ContributionStatsResponse:
36	calendar = self.build_calendar(db, actor, start, end)	48	calendar = self.build_calendar(db, actor, start, end)
	@@ -47,6 +59,28 @@ class ContributionAggregator:
47	active_days=len(active_days),	59	active_days=len(active_days),
48	longest_streak=max(streaks, default=0),	60	longest_streak=max(streaks, default=0),
49	current_streak=current_streak,	61	current_streak=current_streak,
		62	is_indexed=calendar.is_indexed,
		63	last_polled_at=calendar.last_polled_at,
		64	indexing_state=calendar.indexing_state,
		65	)
		66
		67	def _index_metadata(self, db: Session, actor: str) -> ContributionIndexMetadata:
		68	tracked_actor = db.scalar(select(TrackedActor).where(TrackedActor.actor == actor))
		69	has_indexed_events = db.scalar(select(ContributionEvent.id).where(ContributionEvent.actor == actor).limit(1)) is not None
		70	last_poll_status = tracked_actor.last_poll_status if tracked_actor is not None else None
		71	is_indexed = has_indexed_events or (tracked_actor is not None and tracked_actor.last_polled_at is not None)
		72
		73	if last_poll_status == "error":
		74	indexing_state = "error"
		75	elif is_indexed:
		76	indexing_state = "indexed"
		77	else:
		78	indexing_state = "pending"
		79
		80	return ContributionIndexMetadata(
		81	is_indexed=is_indexed,
		82	last_polled_at=tracked_actor.last_polled_at if tracked_actor is not None else None,
		83	indexing_state=indexing_state,
50	)	84	)
51		85
52	def _query_daily_aggregates(self, db: Session, actor: str, start: date, end: date) -> list[DailyAggregate]:	86	def _query_daily_aggregates(self, db: Session, actor: str, start: date, end: date) -> list[DailyAggregate]:

tests/test_contributions_api.py +20 −1

 from datetime import UTC, datetime
 from fastapi.testclient import TestClient
+from sqlalchemy import select
 from srht_contrib.main import create_app
-from srht_contrib.models import ContributionEvent
+from srht_contrib.models import ContributionEvent, TrackedActor
 def test_read_only_contribution_routes_are_public_and_write_routes_require_api_key(settings, db_engine, session_factory) -> None:
     response = client.get("/api/contributions/~ccleberg?from=2026-03-28&to=2026-03-30")
     assert response.status_code == 200
+    assert response.json()["is_indexed"] is True
+    assert response.json()["indexing_state"] == "indexed"
     assert response.json()["days"] == [
         {"date": "2026-03-28", "count": 0, "score": 0.0},
         {"date": "2026-03-29", "count": 0, "score": 0.0},
     ]
+def test_public_read_registers_actor_for_lazy_indexing(client: TestClient, db_session) -> None:
+    response = client.get("/api/contributions/~ccleberg?from=2026-03-28&to=2026-03-30")
+    tracked_actor = db_session.scalar(select(TrackedActor).where(TrackedActor.actor == "~ccleberg"))
+    assert response.status_code == 200
+    assert response.json()["is_indexed"] is False
+    assert response.json()["indexing_state"] == "pending"
+    assert response.json()["last_polled_at"] is None
+    assert tracked_actor is not None
+    assert tracked_actor.is_active is True
+    assert tracked_actor.last_requested_at is not None
 def test_contribution_stats_api(client: TestClient, db_session) -> None:
     db_session.add_all(
         [
     assert response.json()["total_score"] == 1.5
     assert response.json()["longest_streak"] == 2
     assert response.json()["current_streak"] == 2
+    assert response.json()["is_indexed"] is True
+    assert response.json()["indexing_state"] == "indexed"
 def test_invalid_date_input_returns_400(client: TestClient) -> None:

tests/test_ingestion.py +29 −1

	@@ -4,7 +4,7 @@ from sqlalchemy import select
4		4
5	from srht_contrib.config import Settings	5	from srht_contrib.config import Settings
6	from srht_contrib.jobs.poller import PollerService	6	from srht_contrib.jobs.poller import PollerService
7	from srht_contrib.models import SyncState, TrackedRepository	7	from srht_contrib.models import SyncState, TrackedActor, TrackedRepository
8	from srht_contrib.schemas import NormalizedEvent	8	from srht_contrib.schemas import NormalizedEvent
9	from srht_contrib.services.git import GitIngestionService, GitPollResult	9	from srht_contrib.services.git import GitIngestionService, GitPollResult
10	from srht_contrib.services.todo import TodoIngestionService, TodoPollResult	10	from srht_contrib.services.todo import TodoIngestionService, TodoPollResult
	@@ -335,3 +335,31 @@ def test_sync_overlap_reuses_cursor_window_and_suppresses_duplicates(db_session)
335	assert state is not None	335	assert state is not None
336	assert len(todo_service.calls) == 2	336	assert len(todo_service.calls) == 2
337	assert todo_service.calls[1].isoformat() == "2026-03-30T00:00:00+00:00"	337	assert todo_service.calls[1].isoformat() == "2026-03-30T00:00:00+00:00"
		338
		339
		340	def test_scheduled_poll_polls_known_actors_and_seeds_default_actor(db_session) -> None:
		341	event = NormalizedEvent(
		342	service="todo",
		343	event_type="ticket_created",
		344	actor="~known",
		345	repo_name="todo",
		346	resource_id="123",
		347	external_uid="todo:event:known:created:123",
		348	occurred_at=datetime(2026, 3, 30, 10, 0, tzinfo=UTC),
		349	weight=1.0,
		350	raw_payload_json=None,
		351	)
		352	todo_service = RecordingTodoService(events_by_call=[[], [event]])
		353	poller = PollerService(todo_service=todo_service, git_service=EmptyGitService())
		354
		355	db_session.add(TrackedActor(actor="~known", is_active=True))
		356	db_session.commit()
		357
		358	results = poller.poll_tracked_actors(db_session, default_actor="~default")
		359
		360	tracked_actors = db_session.scalars(select(TrackedActor).order_by(TrackedActor.actor)).all()
		361
		362	assert results == {"~default": 0, "~known": 1}
		363	assert [actor.actor for actor in tracked_actors] == ["~default", "~known"]
		364	assert all(actor.last_poll_status == "indexed" for actor in tracked_actors)
		365	assert all(actor.last_polled_at is not None for actor in tracked_actors)

tests/test_migrations.py +3

         "alembic_version",
         "contribution_events",
         "sync_states",
+        "tracked_actors",
         "tracked_repositories",
     ]
     assert columns["actor"]["nullable"] is False
     assert "uq_tracked_repository_service_actor_name" in unique_constraints
     assert actor == Settings().default_actor
+    assert "tracked_actors" in inspector.get_table_names()
 def test_alembic_prefers_database_url_from_environment(tmp_path, monkeypatch) -> None:
     inspector = inspect(create_engine(database_url))
     assert "actor_aliases" in inspector.get_table_names()
+    assert "tracked_actors" in inspector.get_table_names()

tests/test_polling_api.py +21 −1

 from datetime import UTC, datetime
 from fastapi.testclient import TestClient
+from sqlalchemy import select
 from srht_contrib.config import Settings
 from srht_contrib.main import create_app
-from srht_contrib.models import ContributionEvent
+from srht_contrib.models import ContributionEvent, TrackedActor
 from srht_contrib.services.srht_client import SourceHutClientError
         self.todo_service = service
         self.git_service = service
+    def track_actor_request(self, db, actor: str, *, update_last_requested: bool = True):
+        tracked_actor = db.scalar(select(TrackedActor).where(TrackedActor.actor == actor))
+        if tracked_actor is None:
+            tracked_actor = TrackedActor(actor=actor, is_active=True)
+            db.add(tracked_actor)
+        if update_last_requested:
+            tracked_actor.last_requested_at = datetime(2026, 3, 30, 9, 0, tzinfo=UTC)
+        db.flush()
+        return tracked_actor
     def poll_all(self, db, actor: str) -> int:
         db.add(
             ContributionEvent(
         self.todo_service = service
         self.git_service = service
+    def track_actor_request(self, db, actor: str, *, update_last_requested: bool = True):
+        tracked_actor = db.scalar(select(TrackedActor).where(TrackedActor.actor == actor))
+        if tracked_actor is None:
+            tracked_actor = TrackedActor(actor=actor, is_active=True)
+            db.add(tracked_actor)
+        db.flush()
+        return tracked_actor
     def poll_all(self, db, actor: str) -> int:
         raise SourceHutClientError("boom")
     assert poll_response.status_code == 200
     assert poll_response.json()["inserted_events"] == 1
     assert calendar_response.status_code == 200
+    assert calendar_response.json()["is_indexed"] is True
     assert calendar_response.json()["days"] == [{"date": "2026-03-30", "count": 1, "score": 1.0}]

Commit 6149ca9c39

Commit `6149ca9c39`