From ee37803987738ce66befc8205325b2811563d0e3 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Thu, 3 Nov 2022 23:03:52 +0100 Subject: [PATCH] Boostrap full-text search for the outbox --- ...2022_11_02_1914-368f511ad954_outbox_fts.py | 115 ++++++++++++++++++ app/admin.py | 52 ++++++++ app/templates/layout.html | 1 + 3 files changed, 168 insertions(+) create mode 100644 alembic/versions/2022_11_02_1914-368f511ad954_outbox_fts.py diff --git a/alembic/versions/2022_11_02_1914-368f511ad954_outbox_fts.py b/alembic/versions/2022_11_02_1914-368f511ad954_outbox_fts.py new file mode 100644 index 0000000..ec3cf37 --- /dev/null +++ b/alembic/versions/2022_11_02_1914-368f511ad954_outbox_fts.py @@ -0,0 +1,115 @@ +"""Outbox FTS + +Revision ID: 368f511ad954 +Revises: b28c0551c236 +Create Date: 2022-11-02 19:14:37.865923+00:00 + +""" +from sqlalchemy import insert +from sqlalchemy import select +from sqlalchemy.orm.session import Session + +from alembic import op + +# revision identifiers, used by Alembic. +revision = '368f511ad954' +down_revision = 'b28c0551c236' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + # ### end Alembic commands ### + op.execute( + "CREATE VIRTUAL TABLE outbox_fts USING " + "fts5(summary, name, source, content='');" + ) + op.execute( + "CREATE TRIGGER outbox_fts_ai AFTER " + "INSERT ON outbox WHEN new.ap_type in ('Article', 'Note', 'Question') BEGIN" + " INSERT INTO outbox_fts (rowid, source, name, summary)" + " VALUES (" + " new.id, " + " new.source, " + ' json_extract(new.ap_object, "$.name"), ' + ' json_extract(new.ap_object, "$.summary")' + " ); " + "END;" + ) + op.execute( + "CREATE TRIGGER outbox_fts_ad AFTER " + "DELETE ON outbox WHEN old.ap_type in ('Article', 'Note', 'Question') BEGIN" + " INSERT INTO outbox_fts (outbox_fts, rowid, source, name, summary)" + " VALUES (" + " 'delete', " + " old.id, " + " old.source, " + ' json_extract(old.ap_object, "$.name"), ' + ' json_extract(old.ap_object, "$.summary")' + " ); " + "END;" + ) + op.execute( + "CREATE TRIGGER outbox_fts_au_softdelete AFTER " + "UPDATE ON outbox WHEN new.is_deleted = 1 AND " + "new.ap_type in ('Article', 'Note', 'Question') BEGIN" + " INSERT INTO outbox_fts (outbox_fts, rowid, source, name, summary)" + " VALUES (" + " 'delete', " + " old.id, " + " old.source, " + ' json_extract(old.ap_object, "$.name"), ' + ' json_extract(old.ap_object, "$.summary")' + " ); " + "END; " + ) + op.execute( + "CREATE TRIGGER outbox_fts_au AFTER " + "UPDATE ON outbox " + "WHEN (new.source <> old.source OR new.ap_object <> old.ap_object) AND " + "new.ap_type in ('Note', 'Article', 'Quesion') BEGIN" + " INSERT INTO outbox_fts (outbox_fts, rowid, source, name, summary)" + " VALUES (" + " 'delete', " + " old.id, " + " old.source, " + ' json_extract(old.ap_object, "$.name"), ' + ' json_extract(old.ap_object, "$.summary")' + " );" + " INSERT INTO outbox_fts (rowid, source, name, summary)" + " VALUES (" + " new.id, " + " new.source, " + ' json_extract(new.ap_object, "$.name"), ' + ' json_extract(new.ap_object, "$.summary")' + " );" + "END;" + ) + from app.models import OutboxObject + from app.models import outbox_fts + sess = Session(op.get_bind()) + + # Backfill the index + outbox_objects = sess.execute(select(OutboxObject).where( + OutboxObject.ap_type.in_(["Article", "Note", "Question"])) + ).scalars() + for outbox_object in outbox_objects: + row = {"source": outbox_object.source, "rowid": outbox_object.id} + if name := outbox_object.ap_object.get("name"): + row["name"] = name + if summary := outbox_object.ap_object.get("summary"): + row["summary"] = summary + sess.execute(insert(outbox_fts).values(row)) + + sess.commit() + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + # ### end Alembic commands ### + op.drop_table('outbox_fts') + op.execute("DROP TRIGGER outbox_fts_ai;") + op.execute("DROP TRIGGER outbox_fts_ad;") + op.execute("DROP TRIGGER outbox_fts_au_softdelete;") + op.execute("DROP TRIGGER outbox_fts_au;") diff --git a/app/admin.py b/app/admin.py index d8a5ca4..b683aa9 100644 --- a/app/admin.py +++ b/app/admin.py @@ -1,4 +1,5 @@ from datetime import datetime +from typing import Any import httpx from fastapi import APIRouter @@ -149,6 +150,57 @@ async def get_lookup( ) +@router.get("/search") +async def admin_search( + request: Request, + query: str | None = None, + db_session: AsyncSession = Depends(get_db_session), +) -> templates.TemplateResponse | RedirectResponse: + + results: list[Any] = [] + if query: + results = ( + ( + await db_session.execute( + select(models.OutboxObject) + .join( + models.outbox_fts, + models.outbox_fts.c.rowid == models.OutboxObject.id, + ) + .options( + joinedload( + models.OutboxObject.outbox_object_attachments + ).options(joinedload(models.OutboxObjectAttachment.upload)), + joinedload(models.OutboxObject.relates_to_inbox_object).options( + joinedload(models.InboxObject.actor), + ), + joinedload( + models.OutboxObject.relates_to_outbox_object + ).options( + joinedload( + models.OutboxObject.outbox_object_attachments + ).options(joinedload(models.OutboxObjectAttachment.upload)), + ), + ) + .where(models.outbox_fts.c.outbox_fts.op("MATCH")(query)) + .limit(20) + ) + ) # type: ignore + .unique() + .scalars() + ) + + return await templates.render_template( + db_session, + request, + "admin_search.html", + { + "query": query, + "results": results, + }, + ) + + @router.get("/new") async def admin_new( request: Request, diff --git a/app/templates/layout.html b/app/templates/layout.html index 3c30d2a..170d292 100644 --- a/app/templates/layout.html +++ b/app/templates/layout.html @@ -30,6 +30,7 @@
  • {{ admin_link("admin_inbox", "Inbox") }} / {{ admin_link("admin_outbox", "Outbox") }}
  • {{ admin_link("admin_direct_messages", "DMs") }}
  • {{ admin_link("get_notifications", "Notifications") }} {% if notifications_count %}({{ notifications_count }}){% endif %}
    • +
  • {{ admin_link("admin_search", "Search") }}
  • {{ admin_link("get_lookup", "Lookup") }}
  • {{ admin_link("admin_bookmarks", "Bookmarks") }}
  • Logout