[ { "id": "lotus", "name": "LOTUS", "title": "Semantic Operators: A Declarative Model for Rich, AI-based Data Processing", "year": 2024, "category": "Research engine", "interface": "Relational operators", "summary": "Natural-language specifications for filtering, joining, sorting and aggregation, with optimized implementations evaluated against a reference algorithm.", "url": "https://arxiv.org/abs/2407.11418", "tags": [ "Semantic operators", "Quality constraints", "Cascades" ], "verifiedDate": "2026-10-23", "evidence": "Primary source reviewed", "caveat": "Accuracy guarantees are relative to a chosen gold algorithm; they are not guarantees of factual correctness." }, { "id": "uqe", "name": "UQE", "title": "UQE: A Query Engine for Unstructured Databases", "year": 2024, "category": "Research engine", "interface": "UQL / SQL dialect", "summary": "A SQL-like language for reasoning over images, conversations and reviews, combining foundation models with sampling and execution optimization.", "url": "https://arxiv.org/abs/2407.09522", "tags": [ "SQL", "Multimodal", "Sampling" ], "verifiedDate": "2026-10-23", "evidence": "Primary source reviewed", "caveat": "Published evaluation covers selected query types and datasets; capabilities should not be generalized to arbitrary analytics." }, { "id": "palimpzest", "name": "Palimpzest", "title": "A Declarative System for Optimizing AI Workloads", "year": 2024, "category": "Research engine", "interface": "Declarative pipelines", "summary": "Turns declarative AI analytics into executable plans that trade off output quality, runtime and model cost.", "url": "https://arxiv.org/abs/2405.14696", "tags": [ "Pipelines", "Model selection", "Cost models" ], "verifiedDate": "2026-10-23", "evidence": "Primary source reviewed", "caveat": "This record describes the 2024 prototype; it is not a specification of every capability in the current project." }, { "id": "docetl", "name": "DocETL", "title": "DocETL: Agentic Query Rewriting and Evaluation for Complex Document Processing", "year": 2024, "category": "Research engine", "interface": "Declarative pipelines", "summary": "Rewrites complex document workflows by decomposing tasks and data, using agents to propose and evaluate better execution plans.", "url": "https://arxiv.org/abs/2410.12189", "tags": [ "Documents", "Rewriting", "Evaluation" ], "verifiedDate": "2026-10-23", "evidence": "Primary source reviewed", "caveat": "Agent-generated validation and reported accuracy gains depend on the task and evaluation design." }, { "id": "suql", "name": "SUQL", "title": "SUQL: Conversational Search over Structured and Unstructured Data with Large Language Models", "year": 2023, "category": "Research engine", "interface": "SQL extension", "summary": "Composes structured queries with free-text answering and summarization, supporting conversational access to hybrid knowledge sources.", "url": "https://arxiv.org/abs/2311.09818", "tags": [ "SQL", "Hybrid data", "Conversational search" ], "verifiedDate": "2026-10-23", "evidence": "Primary source reviewed", "caveat": "The paper studies hybrid access and semantic parsing; it is not a broad benchmark of semantic engine optimization." }, { "id": "sembench", "name": "SemBench", "title": "SemBench: A Benchmark for Semantic Query Processing Engines", "year": 2025, "category": "Benchmark", "interface": "Benchmark workloads", "summary": "Evaluates semantic engines across text, image and audio scenarios using filters, joins, mappings, ranking and classification.", "url": "https://arxiv.org/abs/2511.01716", "tags": [ "Multimodal", "Evaluation", "End-to-end" ], "verifiedDate": "2026-10-23", "evidence": "Primary source reviewed", "caveat": "Results are a snapshot of evolving systems, models and configurations; they do not establish a permanent leaderboard.", "benchmarkGroup": "engine" }, { "id": "lrobench", "name": "LROBench", "title": "Large Language Model-Enhanced Relational Operators: Taxonomy, Benchmark, and Analysis", "year": 2026, "category": "Benchmark", "interface": "Relational operator workloads", "summary": "Organizes Select, Match, Impute, Cluster and Order operators into single-operator and multi-operator workloads over relational databases.", "url": "https://arxiv.org/abs/2603.02537", "tags": [ "Taxonomy", "Evaluation", "Relational operators" ], "verifiedDate": "2026-10-23", "evidence": "Primary source reviewed", "caveat": "Operator-level findings depend on model, implementation and workload; the benchmark complements rather than replaces end-to-end evaluation.", "benchmarkGroup": "engine" }, { "id": "sema", "name": "Sema", "title": "Sema: A High-performance System for LLM-based Semantic Query Processing", "year": 2026, "category": "Research engine", "interface": "SemaSQL / DuckDB", "summary": "Integrates semantic expressions into SQL and combines logical rewrites with adaptive reordering, fusion and prompt batching.", "url": "https://arxiv.org/abs/2603.11622", "tags": [ "SQL", "DuckDB", "Adaptive execution" ], "verifiedDate": "2026-10-23", "evidence": "Primary source reviewed", "caveat": "Preprint results cover 20 queries and selected baselines; no independent reproduction is claimed here." }, { "id": "nousdb", "name": "NousDB", "title": "NousDB - A Fast Query Engine for Semantic Analytics", "year": 2026, "category": "Research engine", "interface": "Semantic relational algebra", "summary": "A DataFusion-based engine using semantic algebra, token histograms and context-aware batching to optimize AI analytics.", "url": "https://arxiv.org/abs/2610.28551", "tags": [ "DataFusion", "Batching", "Cost models" ], "verifiedDate": "2026-10-23", "evidence": "Primary source reviewed", "caveat": "October 2026 preprint. Batching guarantees have specific statistical assumptions; reported speedups are workload-specific." }, { "id": "jevdb", "name": "JEVDB", "title": "Prune First, Decide Fast: Scalable Semantic Query Processing with JEVDB", "year": 2026, "category": "Research engine", "interface": "SQL / typed decisions", "summary": "Uses decision models for semantic operations, escalates uncertain cases to generative models and prunes semantic-join candidates.", "url": "https://arxiv.org/abs/2610.02046", "tags": [ "Semantic joins", "Cascades", "Candidate pruning" ], "verifiedDate": "2026-10-23", "evidence": "Primary source reviewed", "caveat": "October 2026 preprint. Necessary-condition screening and evaluated decision models determine the scope of pruning claims." }, { "id": "semviews", "name": "SemViews", "title": "Semantic Views: Certified Reuse of LLM Labels Across Semantic Queries", "year": 2026, "category": "Optimization", "interface": "Semantic filter", "summary": "Reuses prior query labels as ranking signals and applies fresh stratified oracle sampling to certify filter precision and recall.", "url": "https://arxiv.org/abs/2610.12768", "tags": [ "Reuse", "Quality constraints", "Sampling" ], "verifiedDate": "2026-10-23", "evidence": "Primary source reviewed", "caveat": "October 2026 preprint. Guarantees concern agreement with an oracle under the stated sampling assumptions, not human truth." }, { "id": "run-to-run-variance", "name": "Same Query, Different Answer", "title": "Same Query, Different Answer: Run-to-Run Variance in Semantic Query Processing", "year": 2026, "category": "Evaluation study", "interface": "Repeated benchmark executions", "summary": "Studies how repeated model decisions propagate into changing query answers and proposes reporting practices for reproducible evaluation.", "url": "https://arxiv.org/abs/2610.13823", "tags": [ "Reproducibility", "Evaluation", "Nondeterminism" ], "verifiedDate": "2026-10-23", "evidence": "Primary source reviewed", "caveat": "October 2026 preprint. This is an evaluation study rather than a new engine; findings concern the tested models and workloads.", "benchmarkGroup": "methodology" }, { "id": "sembaker", "name": "SemBaker", "title": "From Interpretation to Compilation: A Compilation-Based Execution Engine for Semantic Operator Systems", "year": 2026, "category": "Optimization", "interface": "Execution plugin", "summary": "Compiles suitable semantic operations into local Python functions, routing between compiled and native execution using cost estimates.", "url": "https://arxiv.org/abs/2608.06677", "tags": [ "Compilation", "Pipelines", "Cost models" ], "verifiedDate": "2026-10-23", "evidence": "Primary source reviewed", "caveat": "Compilation applies to selected operators; generated code is not a universal substitute for semantic model inference." }, { "id": "kalypso", "name": "Kalypso", "title": "Kalypso: Relational LLM Serving", "year": 2026, "category": "Optimization", "interface": "Query-plan serving API", "summary": "Makes model serving aware of semantic operator dependencies, pipelining tuples to reuse KV-cache state under memory constraints.", "url": "https://arxiv.org/abs/2607.23815", "tags": [ "Serving", "KV cache", "Scheduling" ], "verifiedDate": "2026-10-23", "evidence": "Primary source reviewed", "caveat": "A serving substrate rather than a complete user-facing query engine; benefits depend on cache reuse and workload structure." }, { "id": "flockmtl", "name": "FlockMTL", "title": "Beyond Quacking: Deep Integration of Language Models and RAG into DuckDB", "year": 2025, "category": "Research engine", "interface": "SQL / DuckDB extension", "summary": "Adds model-backed scalar and aggregate functions, with PROMPT and MODEL schema objects plus batching and caching optimizations.", "url": "https://arxiv.org/abs/2504.01157", "tags": [ "SQL", "DuckDB", "RAG" ], "verifiedDate": "2026-10-23", "evidence": "Primary source reviewed", "caveat": "The cited publication is a short system paper; consult project documentation for deployment and supported models." }, { "id": "ipdb", "name": "iPDB", "title": "iPDB -- Optimizing Semantic SQL Queries", "year": 2026, "category": "Research engine", "interface": "Extended SQL", "summary": "Supports in-database ML and LLM inference through a relational predict operator and optimizations across semantic SQL operations.", "url": "https://arxiv.org/abs/2601.16432", "tags": [ "SQL", "Inference", "Query optimization" ], "verifiedDate": "2026-10-23", "evidence": "Primary source reviewed", "caveat": "The current paper title differs from earlier citations. Performance comparisons apply to the evaluated configurations." }, { "id": "plop", "name": "PLOP", "title": "PLOP: Cost-Based Placement of Semantic Operators in Hybrid Query Plans", "year": 2026, "category": "Optimization", "interface": "Hybrid query plans", "summary": "Balances model-call cost against relational processing cost when choosing semantic operator placement in mixed query plans.", "url": "https://arxiv.org/abs/2604.09944", "tags": [ "Operator placement", "Cost models", "Semantic joins" ], "verifiedDate": "2026-10-23", "evidence": "Primary source reviewed", "caveat": "The current record is titled PLOP, not Horrila. Rewrites and cost analysis rely on the paper's stated semantics and caching assumptions." }, { "id": "relational-llm-optimization", "name": "Relational LLM optimization", "title": "Optimizing LLM Queries in Relational Data Analytics Workloads", "year": 2024, "category": "Optimization", "interface": "Relational analytics", "summary": "Reorders table rows and fields to increase shared prompt prefixes and improve KV-cache reuse during batch model inference.", "url": "https://arxiv.org/abs/2403.05821", "tags": [ "KV cache", "Serving", "Batch analytics" ], "verifiedDate": "2026-10-23", "evidence": "Primary source reviewed", "caveat": "Reported cost savings depend on model-serving behavior and provider pricing; this catalog does not repeat them as current estimates." }, { "id": "llm-order-by", "name": "LLM ORDER BY", "title": "Access Paths for Efficient Ordering with Large Language Models", "year": 2025, "category": "Optimization", "interface": "Semantic ranking operator", "summary": "Studies physical implementations of semantic ordering and a budget-aware optimizer that chooses among ranking access paths.", "url": "https://arxiv.org/abs/2509.00303", "tags": [ "Ranking", "Top-k", "Access paths" ], "verifiedDate": "2026-10-23", "evidence": "Primary source reviewed", "caveat": "No single sorting implementation dominates all evaluated datasets; quality assessment includes model-based judging." }, { "id": "larch", "name": "Larch", "title": "Larch: Learned Query Optimization for Semantic Predicates", "year": 2026, "category": "Optimization", "interface": "AI SQL filters", "summary": "Learns per-row evaluation orders for semantic predicates, using either a sequential decision policy or predicted selectivities.", "url": "https://arxiv.org/abs/2606.07923", "tags": [ "Filters", "Learned optimization", "Selectivity" ], "verifiedDate": "2026-10-23", "evidence": "Primary source reviewed", "caveat": "The reported reductions concern tested workloads and baselines; training and runtime overhead should be considered in new deployments." }, { "id": "fast-semantic-filtering", "name": "Fast semantic filtering", "title": "Fast LLM-Based Semantic Filtering: From a Unified Framework to an Adaptive Two-Phase Method", "year": 2026, "category": "Optimization", "interface": "Semantic filter", "summary": "Combines clustering and learned proxies adaptively, using token-aware representations and oracle confidence to reduce expensive filtering calls.", "url": "https://arxiv.org/abs/2606.08090", "tags": [ "Filters", "Cascades", "Proxy models" ], "verifiedDate": "2026-10-23", "evidence": "Primary source reviewed", "caveat": "Empirical attainment of an accuracy target is not an unconditional guarantee; the oracle itself may be wrong." }, { "id": "semjoin", "name": "SemJoin", "title": "SemJoin: Semantic Join Optimization", "year": 2026, "category": "Optimization", "interface": "Semantic join", "summary": "Uses an LLM advisor to route joins between clustering-based candidate pruning and classification into a shared label space.", "url": "https://arxiv.org/abs/2606.29532", "tags": [ "Semantic joins", "Routing", "Clustering" ], "verifiedDate": "2026-10-23", "evidence": "Primary source reviewed", "caveat": "Short preprint evaluated on three datasets; routing success on those workloads does not establish universal optimality." }, { "id": "uda-bench", "name": "UDA-Bench", "title": "Unstructured Data Analysis using LLMs: A Comprehensive Benchmark", "year": 2025, "category": "Benchmark", "interface": "Document analytics / relational ground truth", "benchmarkGroup": "engine", "summary": "Evaluates unstructured-document analysis against manually annotated relational views. The arXiv v1 workload contains five datasets and 240 queries covering extraction, filtering, aggregation, joins, and combinations of operators.", "tags": [ "Documents", "Relational ground truth", "End-to-end" ], "url": "https://arxiv.org/abs/2510.27119", "caveat": "Ground truth is defined over selected schema attributes, so coverage does not extend to arbitrary open-ended document questions. Dataset counts differ between the cited arXiv version and repository descriptions; specify the artifact version when reporting results.", "reviewNote": "Publication record and benchmark-construction sections reviewed. The workload evaluates accuracy, model cost, and latency. No experiments were rerun.", "resources": [ { "label": "Paper: benchmark construction", "url": "https://arxiv.org/html/2510.27119v1#S3" }, { "label": "UDA-Bench repository", "url": "https://github.com/Chandrika2527/UDA-Bench" } ], "verifiedDate": "2026-10-23", "evidence": "Primary source reviewed" }, { "id": "shelob", "name": "Shelob", "title": "Prune First, Decide Fast: Scalable Semantic Query Processing with JEVDB", "year": 2026, "category": "Benchmark", "interface": "TPC-DS-derived semantic joins", "benchmarkGroup": "engine", "summary": "Extends TPC-DS-derived relations with return reports, issue bulletins, and release notes to study semantic-join scaling. The JEVDB paper evaluates five queries at XS, S, and M tiers.", "tags": [ "Semantic joins", "Scalability", "TPC-DS" ], "url": "https://arxiv.org/abs/2610.02046", "caveat": "Introduced and evaluated by the JEVDB authors. The full SF1 workload is larger than the measured tiers; its candidate count must not be presented as an executed comparison. Shelob is not an official TPC benchmark or an independent validation of JEVDB.", "reviewNote": "Reviewed the experimental setup and Shelob evaluation sections of the JEVDB paper. The protocol reports answer F1, cost, latency, and timeout progress. No experiments were rerun.", "resources": [ { "label": "Shelob evaluation in the JEVDB paper", "url": "https://arxiv.org/html/2610.02046v1#S4" } ], "verifiedDate": "2026-10-23", "evidence": "Primary source reviewed" }, { "id": "spider2-aifunc", "name": "Spider 2.0-AIFunc", "title": "Spider 2.0-AIFunc: Extending Real-World Text-to-SQL to AI-Native SQL Workflows", "year": 2026, "category": "Benchmark", "interface": "Natural language to SQL with AI functions", "benchmarkGroup": "generation", "summary": "Tests whether models generate Snowflake SQL that composes relational operations with AI functions for classification, filtering, sentiment analysis, extraction, similarity, and aggregation.", "tags": [ "Query generation", "SQL", "AI functions" ], "url": "https://arxiv.org/abs/2607.06229", "caveat": "Evaluates SQL generation, not engine execution performance. The paper reports 465 instances; the public task release contains 393. Gold SQL and database resources are not bundled, and official scoring requires additional access.", "resources": [ { "label": "Benchmark repository and access requirements", "url": "https://github.com/Leolty/Spider2-AIFunc" }, { "label": "Released task dataset", "url": "https://huggingface.co/datasets/tianyang/spider2-aifunc" } ], "verifiedDate": "2026-10-23", "evidence": "Primary source reviewed" }, { "id": "vectorsqlbench", "name": "VectorSQLBench", "title": "Text2VectorSQL: Towards a Unified Interface for Vector Search and SQL Queries", "year": 2025, "category": "Benchmark", "interface": "Natural language to SQL with vector search", "benchmarkGroup": "generation", "summary": "Benchmarks generation of queries combining relational operations and vector search, spanning three database backends and four source collections. Examines result omissions when SQL filters interact with vector retrieval.", "tags": [ "Query generation", "Vector search", "Hybrid data" ], "url": "https://arxiv.org/abs/2506.23071", "caveat": "An adjacent query-generation benchmark. Vector similarity is a retrieval operation and does not establish the quality or efficiency of arbitrary model-backed filters, joins, or aggregations.", "verifiedDate": "2026-10-23", "evidence": "Primary source reviewed" }, { "id": "hybrid-deep-research", "name": "HybridDeepResearch", "title": "Benchmarking Hybrid Deep Research Across Database Querying and Web Search", "year": 2026, "category": "Benchmark", "interface": "Agents combining SQL and web search", "benchmarkGroup": "adjacent", "summary": "Evaluates agents on 380 tasks requiring both database queries and web evidence. Tasks exercise database-to-search, search-to-database, and parallel reasoning, with an emphasis on preserving constraints across tools.", "tags": [ "Agents", "SQL", "Web search" ], "url": "https://arxiv.org/abs/2609.09410", "caveat": "Measures agent coordination and final-answer correctness across tools. It does not isolate semantic operator execution or query optimizer performance, and web evidence can change over time.", "verifiedDate": "2026-10-23", "evidence": "Primary source reviewed" }, { "id": "stochastic-shift", "name": "The Stochastic Shift", "title": "The Stochastic Shift: A New Evaluation Paradigm for Text-to-SQL with AI Operators", "year": 2026, "category": "Evaluation study", "interface": "Evaluation of SQL containing AI operators", "benchmarkGroup": "methodology", "summary": "Studies failures of exact-result scoring for SQL with stochastic AI operators. Proposes evaluating relational logic and semantic operations separately, with experiments on BigQuery and ThalamusDB.", "tags": [ "Evaluation", "Query generation", "Nondeterminism" ], "url": "https://arxiv.org/abs/2609.21133", "caveat": "An evaluation framework rather than a general-purpose workload benchmark. Reported validation accuracy applies to the tested query sets and judging protocol, not all SQL translations or semantic predicates.", "verifiedDate": "2026-10-23", "evidence": "Primary source reviewed" }, { "id": "bigquery", "name": "BigQuery", "title": "BigQuery generative AI functions", "year": 2026, "category": "Industry platform", "interface": "GoogleSQL", "summary": "Managed AI functions bring generation, filtering, classification and scoring into SQL over structured and unstructured data.", "url": "https://docs.cloud.google.com/bigquery/docs/generative-ai-overview", "tags": [ "SQL", "Multimodal", "Managed inference" ], "verifiedDate": "2026-10-23", "evidence": "Vendor documentation reviewed", "caveat": "Function availability, supported models, regions and billing vary. Product documentation does not establish comparative engine performance." }, { "id": "snowflake", "name": "Snowflake Cortex", "title": "Snowflake Cortex AI Functions / AISQL", "year": 2026, "category": "Industry platform", "interface": "SQL", "summary": "SQL-callable AI functions support classification, filtering, extraction, summarization and aggregation inside the Snowflake platform.", "url": "https://docs.snowflake.com/en/user-guide/snowflake-cortex/aisql", "tags": [ "SQL", "Managed inference", "Semantic operators" ], "verifiedDate": "2026-10-23", "evidence": "Vendor documentation reviewed", "caveat": "Check per-function documentation for model and regional support, pricing and release status. A product feature is not an independent benchmark." }, { "id": "databricks", "name": "Databricks", "title": "Databricks ai_query", "year": 2026, "category": "Industry platform", "interface": "Databricks SQL", "summary": "The ai_query function invokes a model-serving endpoint from SQL and parses the response for real-time or batch inference workflows.", "url": "https://docs.databricks.com/aws/en/sql/language-manual/functions/ai_query", "tags": [ "SQL", "Model serving", "Batch analytics" ], "verifiedDate": "2026-10-23", "evidence": "Vendor documentation reviewed", "caveat": "Requires a supported runtime or SQL environment, region and endpoint permissions; not available on Databricks SQL Classic. No optimizer equivalence with research engines is implied." } ]