B. Grammar

The complete grammar, in EBNF. The pipeline layer (lexer, parser) splits a request into stages and sub-pipelines; each stage then reads its own argument text with the stage grammar below.

In this grammar "|" stands for a stage separator: |, or, in a query that holds no |, ; or a line break (B.6, 3.2). Both styles give the same stages.

B.1 Requests

request        = view-statement
               | [ "explain" ] , { let-header } , query ;
               (* `$name` parameters are bound as literals before parsing *)

let-header     = "let" , binding , { "," , binding } , "|" ;
binding        = variable , ( "=" | "?=" ) , ( expression | "(" , query , ")" ) ;
variable       = "$" , identifier ;

query          = source , { "|" , stage } ;
source         = ( "on" | "from" ) , source-target , [ "as" , identifier ] ;
source-target  = collection , [ "as" , "of" , ( string | integer ) | "history" ]
               | "values" , object-array
               | "view" , view-name , [ "version" , integer ] ;
collection     = identifier , { "." , identifier } ;
view-name      = identifier , { "." , identifier } ;
object-array   = "[" , object-literal , { "," , object-literal } , "]" ;

B.2 View statements

view-statement = [ "explain" ] , ( define-view | undefine-view ) ;
define-view    = "define" , "view" , view-name ,
                 { "|" , view-clause } ,
                 [ "|" , "exec" , "|" , query , "|" , "end" ] ,
                 "|" , "end" ;
view-clause    = "description" , string
               | "version" , positive-integer      (* restore: no exec *)
               | "materialized" ;
undefine-view  = "undefine" , "view" , view-name ;

B.3 Stages

stage          = simple-stage | compound-stage ;
simple-stage   = stage-name , arguments ;
arguments      = { token } ;            (* up to the next top-level "|", brackets balanced *)
sub-pipeline   = { "|" , stage-or-directive } , "|" , "end" ;

compound-stage = lookup | union | group-compound | pivot | window | load ;

Filtering and shaping

where          = "where" , expression ;
select         = ( "select" | "project" ) , select-item , { "," , select-item } ;
select-item    = field-path , [ "as" , field-path ] | expression , "as" , field-path ;
derive         = "derive" , assignment , { "," , assignment } ;
assignment     = field-path , "=" , expression ;
rename         = "rename" , field-path , "as" , field-path ;
drop           = "drop" , field-path , { "," , field-path } ;
root           = "root" , field-path ;
near           = "near" , field-path , "," , ( number-array | variable ) ;

Ordering and paging

sort           = "sort" , sort-key , { "," , sort-key } ;
sort-key       = expression , [ "asc" | "desc" ] ;
limit          = "limit" , count-value ;
skip           = ( "skip" | "offset" ) , count-value ;
first          = "first" , [ field-path ] ;
single         = "single" , [ field-path ] ;
sample         = "sample" , count-value ;
count-value    = non-negative-integer | variable ;

Rows

unwind         = "unwind" , field-path , [ "with" , "index" , field-path ] , [ "keep" , "empty" ] ;
distinct       = "distinct" , [ field-path , { "," , field-path } ] ;
count          = "count" , [ "as" , field-path ] ;

Aggregation and analytics

group-compact  = "group" , group-item , { "," , group-item } ;
group-item     = field-path , [ "as" , field-path ] | expression , "as" , field-path ;

group-compound = "group" , "|" , "by" , group-item , { "," , group-item } ,
                 { "|" , measure } , "|" , "end" ;
measure        = ( "sum" | "avg" | "min" | "max" | "list" | "median" | "stddev" ) , expression , [ measure-filter ] , [ "as" , field-path ]
               | "percentile" , expression , fraction , [ measure-filter ] , [ "as" , field-path ]
               | ( "first" | "last" ) , expression , [ "by" , expression ] , [ measure-filter ] , [ "as" , field-path ]
               | "count" , [ expression ] , [ measure-filter ] , [ "as" , field-path ] ;
measure-filter = "where" , expression ;

bucket         = "bucket" , expression , "by" , array , [ "default" , expression ] , [ "as" , field-path ] ;

pivot          = "pivot" ,
                 "|" , "rows" , group-item , { "," , group-item } ,
                 "|" , "columns" , group-item , { "," , group-item } ,
                 "|" , "values" , group-item , { "," , group-item } ,
                 "|" , "aggregate" , aggregate ,
                 "|" , "end" ;      (* the four directives in any order, each once *)
aggregate      = "sum" | "count" | "avg" | "average" | "min" | "minimum" | "max" | "maximum" | "first" | "last" ;

window         = "window" ,
                 [ "|" , "partition" , "by" , field-path , { "," , field-path } ] ,
                 "|" , "order" , "by" , field-path , [ "asc" | "desc" ] , { "," , field-path , [ "asc" | "desc" ] } ,
                 window-fn , { window-fn } ,
                 "|" , "end" ;
window-fn      = "|" , ( ( "lag" | "lead" ) , field-path , [ integer ]
                       | ( "cumsum" | "first" | "last" ) , field-path
                       | "row_number" ) , [ "as" , field-path ] ;

fill           = "fill" , field-path , "from" , expression , "to" , expression , "step" , expression ,
                 [ "by" , field-path , { "," , field-path } ] ,
                 [ "with" , assignment , { "," , assignment } ] ;

Combining

lookup         = ( "lookup" | "join" ) , lookup-source , [ "as" , identifier ] ,
                 ( "on" , expression , "==" , expression , "into" , field-path       (* one-line *)
                 | { "|" , stage } , "|" , "into" , field-path , { "|" , stage } , "|" , "end" ) ;
lookup-source  = collection | "view" , view-name , [ "version" , integer ] | "values" , object-array ;

union          = "union" , "|" , ( "on" ) , source-target , [ "as" , identifier ] , { "|" , stage } , "|" , "end" ;

Writes

insert         = "insert" , ( object-literal | object-array ) ;
set            = "set" , assignment , { "," , assignment } ;
delete         = "delete" ;
into           = ( "into" | "out" ) , collection ;
load           = "load" , "|" , "with" , ( "replace" | "update" | "merge" ) ,
                 { "|" , "chunk" , json-array } , "|" , "end"
               | "load" , collection ;          (* runtime hook *)

B.4 Expressions

expression   = or-expr ;
or-expr      = and-expr , { ( "or" | "||" ) , and-expr } ;
and-expr     = not-expr , { ( "and" | "&&" ) , not-expr } ;
not-expr     = ( "not" | "!" ) , not-expr | eq-expr ;
eq-expr      = rel-expr , { ( "==" | "!=" ) , rel-expr } ;
rel-expr     = add-expr , { ( "<" | "<=" | ">" | ">=" | "in" | "not" , "in" ) , add-expr
                          | [ "not" ] , "between" , add-expr , ( "and" | "&&" ) , add-expr } ;
add-expr     = mul-expr , { ( "+" | "-" ) , mul-expr } ;
mul-expr     = unary , { ( "*" | "/" | "%" ) , unary } ;
unary        = ( "-" | "+" ) , number            (* folded into a signed literal *)
             | ( "-" | "+" ) , unary
             | primary ;
primary      = literal | variable | call | field-path | array | object-literal | "(" , expression , ")" ;
call         = identifier , "(" , [ expression , { "," , expression } ] , ")" ;
array        = "[" , [ expression , { "," , expression } ] , "]" ;
object-literal = "{" , [ member , { "," , member } ] , "}" ;
member       = ( string | identifier ) , ":" , expression ;   (* bare keys in insert, values and load *)
field-path   = identifier , [ "?" ] , { "." , identifier , [ "?" ] } ;   (* any "?" makes the path optional *)
literal      = "null" | "true" | "false" | number | string ;

B.5 Tokens

identifier   = ( letter | "_" ) , { letter | digit | "_" } ;     (* Unicode letters *)
number       = [ "+" | "-" ] , digit , { digit } , [ "." , digit , { digit } ] , [ exponent ]
             | "." , digit , { digit } , [ exponent ] ;
exponent     = ( "e" | "E" ) , [ "+" | "-" ] , digit , { digit } ;
string       = '"' , { char | escape } , '"' ;
escape       = "\" , ( '"' | "\" | "/" | "b" | "f" | "n" | "r" | "t" | "u" , hex , hex , hex , hex ) ;

Keywords reserved by the pipeline lexer: on from as where set lookup join union load pivot into with chunk replace update merge end and or not true false null; in inside expressions.

B.6 Stage separators

separator    = "|"                                  (* a query holding a top-level "|" *)
             | ";" | line-break ;                   (* any other query *)
line-break   = ? a newline ending a stage: brackets balanced, the line not ending on a continuation,
                 the next line starting with a stage word or a directive of its block ? ;
continuation = "," | "=" | "+" | "-" | "*" | "/" | "%" | "<" | ">" | "!" | "&" | "." | ":" | "$"
             | "and" | "or" | "not" | "in" | "as" | "between" | "by" | "on" | "into" | "with" | "from"
             | "to" | "step" | "default" | "version" | "of" | "keep" | "index" | "view" ;

• A top-level | is one outside strings and brackets, || aside. A query holding one uses | only: there a ; is refused and a line break is a space.
• In the other style a final ; is a space; an empty stage between two ; is refused.
• The words that start a line are the stage names of B.3 and let, end; in a bare group, pivot, window or load, and in define view, the directives of that block (3.2). on and from start a line only after union, exec or a let header.
• Text in parentheses is read on its own, in its own style (a let query).