Welcome to the Tsonnet series!
If you're not following the series so far, you can check out how it all started in the first post of the series.
In the previous post, I started covering the Jsonnet tutorials, starting with the syntax example:
Now I'll cover the variables example:
// A regular definition.
local house_rum = 'Banks Rum';
{
// A definition next to fields.
local pour = 1.5,
Daiquiri: {
ingredients: [
{ kind: house_rum, qty: pour },
{ kind: 'Lime', qty: 1 },
{ kind: 'Simple Syrup', qty: 0.5 },
],
served: 'Straight Up',
},
Mojito: {
ingredients: [
{
kind: 'Mint',
action: 'muddle',
qty: 6,
unit: 'leaves',
},
{ kind: house_rum, qty: pour },
{ kind: 'Lime', qty: 0.5 },
{ kind: 'Simple Syrup', qty: 0.5 },
{ kind: 'Soda', qty: 3 },
],
garnish: 'Lime wedge',
served: 'Over crushed ice',
},
}
Everything should be working with the current implementation, except one thing, the definition next to fields.
So far, objects in Tsonnet are simple JSON objects. Let's make them a bit more dynamic.
Parsing local definitions next to object fields
So far, objects in Tsonnet are simple JSON objects. Let's make them a bit more dynamic.
Let's introduce a new type:
diff --git a/lib/ast.ml b/lib/ast.ml
index 305f79e..5cd601a 100644
--- a/lib/ast.ml
+++ b/lib/ast.ml
@@ -37,12 +37,15 @@ type expr =
| String of position * string
| Ident of position * string
| Array of position * expr list
- | Object of position * (string * expr) list
+ | Object of position * object_entry list
| BinOp of position * bin_op * expr * expr
| UnaryOp of position * unary_op * expr
| Local of position * (string * expr) list
| Seq of expr list
| IndexedExpr of position * string * expr
+and object_entry =
+ | ObjectField of string * expr
+ | ObjectExpr of expr
let dummy_expr = Unit
The object_entry is a new variant type that expresses object entries as both object fields and expressions.. We have two shapes for the entries: ObjectField for regular key-value pairs, and ObjectExpr to hold the local definitions.
The and keyword is used to declare mutually recursive types in OCaml -- pretty handy! The other option is to make object_entry parametric, but we don't need types other than expr.
Then we can apply it to the parser:
diff --git a/lib/parser.mly b/lib/parser.mly
index 91fac9a..efbd372 100644
--- a/lib/parser.mly
+++ b/lib/parser.mly
@@ -66,7 +66,7 @@ literal:
| s = STRING { String (with_pos $startpos $endpos, s) }
| id = ID { Ident (with_pos $startpos $endpos, id) }
| LEFT_SQR_BRACKET; values = array_field_list; RIGHT_SQR_BRACKET { Array (with_pos $startpos $endpos, values) }
- | LEFT_CURLY_BRACKET; attrs = obj_field_list; RIGHT_CURLY_BRACKET; { Object (with_pos $startpos $endpos, attrs) }
+ | LEFT_CURLY_BRACKET; attrs = obj_field_list; RIGHT_CURLY_BRACKET { Object (with_pos $startpos $endpos, attrs) }
;
array_field_list:
@@ -81,13 +81,14 @@ obj_key:
;
obj_field:
- | k = obj_key; COLON; e = assignable_expr { (k, e) }
+ | k = obj_key; COLON; e = assignable_expr { ObjectField (k, e) }
+ | e = single_var { ObjectExpr e }
;
obj_field_list:
| { [] }
| obj_field { [$1] }
- | f = obj_field; COMMA; fs = obj_field_list { f :: fs }
+ | f = obj_field; COMMA; fields = obj_field_list { f :: fields }
;
%inline number:
@@ -114,3 +115,6 @@ var:
vars:
LOCAL; vars = separated_nonempty_list(COMMA, var) { Local (with_pos $startpos $endpos, vars) };
+
+single_var:
+ LOCAL; var_expr = var { Local (with_pos $startpos $endpos, [var_expr]) };
The obj_field_list rule will now contain not just key-value pairs, but also local expressions. Since they are separated by comma, just like object attributes, in the obj_field rule we need a new rule, single_var.
We can't simply use the vars rule because of shift-reduce issues with the parser -- I might write more about this in another post.
Type checking object entries
First, I need a type to represent object_entry in the type system:
diff --git a/lib/type.ml b/lib/type.ml
index e719ffb..0822a7b 100644
--- a/lib/type.ml
+++ b/lib/type.ml
@@ -10,8 +10,11 @@ type tsonnet_type =
| Tstring
| Tany
| Tarray of tsonnet_type
- | Tobject of (string * tsonnet_type) list
+ | Tobject of t_object_entry list
| Lazy of expr
+and t_object_entry =
+ | TobjectField of string * tsonnet_type
+ | TobjectExpr of tsonnet_type
The t_object_entry translates object_entry as-is.
Then we translate each entry:
@@ -93,14 +109,19 @@ let rec translate expr venv =
in ok (venv, Tarray ty)
)
| Object (_pos, elems) ->
- let* fields =
+ let* (fields, _) =
List.fold_left
- (fun acc (attr, expr) ->
- let* attrs = acc in
- let* (_, ty) = translate expr venv in
- ok ((attr, ty) :: attrs)
+ (fun acc entry ->
+ let* (fields, venv) = acc in
+ match entry with
+ | ObjectField (attr, expr) ->
+ let* (venv', ty) = translate expr venv in
+ ok ((TobjectField (attr, ty)) :: fields, venv')
+ | ObjectExpr expr ->
+ let* (venv', _ty) = translate expr venv in
+ ok (fields, venv')
)
- (ok [])
+ (ok ([], venv))
elems
in ok (venv, Tobject fields)
| Local (pos, vars) ->
And check for invalid cycles:
@@ -39,7 +47,15 @@ and check_expr_for_cycles venv expr seen =
match expr with
| Unit | Null _ | Number _ | String _ | Bool _ -> ok ()
| Array (_, exprs) -> iter_for_cycles venv seen exprs
- | Object (_, fields) -> iter_for_cycles venv seen (List.map snd fields)
+ | Object (_, entries) ->
+ List.fold_left
+ (fun ok entry -> ok >>= fun _ ->
+ match entry with
+ | ObjectField (_, expr) -> check_expr_for_cycles venv expr seen
+ | ObjectExpr expr -> check_expr_for_cycles venv expr seen
+ )
+ (ok ())
+ entries
| Ident (pos, varname) -> check_cyclic_refs venv varname seen pos
| BinOp (_, _, e1, e2) -> iter_for_cycles venv seen [e1; e2]
| UnaryOp (_, _, e) -> check_expr_for_cycles venv e seen
Lastly, the to_string needs to include the new type:
let rec to_string = function
@@ -22,8 +26,12 @@ let rec to_string = function
| Tany -> "Any"
| Tarray ty -> "Array of " ^ to_string ty
| Tobject fields ->
+ let field_to_string = function
+ | TobjectField (field, ty) -> field ^ " : " ^ to_string ty
+ | TobjectExpr ty -> to_string ty
+ in
"{" ^ (
- String.concat ", " (List.map (fun (field, ty) -> field ^ " : " ^ to_string ty) fields)
+ String.concat ", " (List.map field_to_string fields)
) ^ "}"
| Lazy ty -> string_of_type ty
Interpreting object entries
Previously, the object could be returned as-is, but now we need to interpret the entries:
diff --git a/lib/interpreter.ml b/lib/interpreter.ml
index 2666a76..fa4b586 100644
--- a/lib/interpreter.ml
+++ b/lib/interpreter.ml
@@ -41,10 +41,29 @@ let interpret_unary_op (op: unary_op) (evaluated_expr: expr) =
| BitwiseNot, Number (pos, Int i) -> ok (Number (pos, Int (lnot i)))
| _ -> error "Invalid unary operation"
+let rec interpret_object env (pos, entries) interpret =
+ let* (result_env, evaluated_entries) = List.fold_left
+ (fun result entry ->
+ let* (env', entries') = result in
+ let* (env'', evaluated_entry) = interpret_entry env' entry interpret in
+ ok (env'', entries' @ [evaluated_entry])
+ )
+ (ok (env, []))
+ entries
+ in
+ ok (result_env, Object (pos, evaluated_entries))
+and interpret_entry env expr interpret =
+ match expr with
+ | ObjectExpr expr ->
+ interpret env expr >>= fun (env', expr') -> ok (env', ObjectExpr expr')
+ | ObjectField (varname, expr) ->
+ interpret env expr >>= fun (env', expr') -> ok (env', ObjectField (varname, expr'))
+
(** [interpret expr] interprets and reduce the intermediate AST [expr] into a result AST. *)
let rec interpret env expr =
match expr with
- | Null _ | Bool _ | String _ | Number _ | Object _ -> ok (env, expr)
+ | Null _ | Bool _ | String _ | Number _ -> ok (env, expr)
+ | Object (pos, entries) -> interpret_object env (pos, entries) interpret
| Array (pos, exprs) ->
(let rec eval' env' exprs' =
match exprs' with
The JSON representation also needs updates:
diff --git a/lib/json.ml b/lib/json.ml
index 2b27718..db19606 100644
--- a/lib/json.ml
+++ b/lib/json.ml
@@ -9,17 +9,20 @@ let rec value_to_yojson : Ast.expr -> (Yojson.t, string) result = function
| Null _ -> ok `Null
| Bool (_, b) -> ok (`Bool b)
| String (_, s) -> ok (`String s)
- | Ident (_, id) -> ok (`String id)
| Array (_, values) ->
let expr_to_list expr' = to_list (value_to_yojson expr') in
let results = values |> List.map expr_to_list |> List.concat in
ok (`List results)
- | Object (_, attrs) ->
- let eval' = fun (k, v) ->
- let result = value_to_yojson v
- in Result.map (fun val' -> (k, val')) result
+ | Object (_, entries) ->
+ let eval' = fun entry ->
+ match entry with
+ | ObjectExpr _ ->
+ error "Object expression(s) not representable as JSON"
+ | ObjectField (k, v) ->
+ let result = value_to_yojson v
+ in Result.map (fun val' -> (k, val')) result
in
- let results = attrs |> List.map eval' |> List.map to_list |> List.concat
+ let results = entries |> List.map eval' |> List.map to_list |> List.concat
in ok (`Assoc results)
| _ -> error "value type not representable as JSON"
The only difference is that during interpretation, the ObjectExpr is thrown away and needs to be ignored here -- if it is found, it must error, as it's not supposed to reach this far.
The Ident case has been removed -- it was never meant to be representable in JSON anyway.
Testing the definitions next to fields
The cram test:
diff --git a/test/cram/tutorials.t b/test/cram/tutorials.t
new file mode 100644
index 0000000..545a65d
--- /dev/null
+++ b/test/cram/tutorials.t
@@ -0,0 +1,50 @@
+ $ tsonnet ../../samples/tutorials/syntax.jsonnet
+ {
+ "cocktails": {
+ "Tom Collins": {
+ "ingredients": [
+ { "kind": "Farmer's Gin", "qty": 1.5 },
+ { "kind": "Lemon", "qty": 1 },
+ { "kind": "Simple Syrup", "qty": 0.5 },
+ { "kind": "Soda", "qty": 2 },
+ { "kind": "Angostura", "qty": "dash" }
+ ],
+ "garnish": "Maraschino Cherry",
+ "served": "Tall",
+ "description": "The Tom Collins is essentially gin and\nlemonade. The bitters add complexity.\n"
+ },
+ "Manhattan": {
+ "ingredients": [
+ { "kind": "Rye", "qty": 2.5 },
+ { "kind": "Sweet Red Vermouth", "qty": 1 },
+ { "kind": "Angostura", "qty": "dash" }
+ ],
+ "garnish": "Maraschino Cherry",
+ "served": "Straight Up",
+ "description": "A clear \\ red drink."
+ }
+ }
+ }
+
+ $ tsonnet ../../samples/tutorials/variables.jsonnet
+ {
+ "Daiquiri": {
+ "ingredients": [
+ { "kind": "Banks Rum", "qty": 1.5 },
+ { "kind": "Lime", "qty": 1 },
+ { "kind": "Simple Syrup", "qty": 0.5 }
+ ],
+ "served": "Straight Up"
+ },
+ "Mojito": {
+ "ingredients": [
+ { "kind": "Mint", "action": "muddle", "qty": 6, "unit": "leaves" },
+ { "kind": "Banks Rum", "qty": 1.5 },
+ { "kind": "Lime", "qty": 0.5 },
+ { "kind": "Simple Syrup", "qty": 0.5 },
+ { "kind": "Soda", "qty": 3 }
+ ],
+ "garnish": "Lime wedge",
+ "served": "Over crushed ice"
+ }
+ }
I missed adding samples/tutorials/syntax.jsonnet in the previous post, but it belongs here too.
The magic of property-based testing -- I only had to specify how each object entry variant should be generated:
diff --git a/test/test_env.ml b/test/test_env.ml
index f16244b..4d59118 100644
--- a/test/test_env.ml
+++ b/test/test_env.ml
@@ -31,11 +31,22 @@ let rec gen_expr_sized n =
(QCheck.Gen.list_size (QCheck.Gen.int_range 0 3) (gen_expr_sized (n-1)))
);
(2, QCheck.Gen.map2
- (fun pos exprs -> Object (pos, exprs))
+ (fun pos entries -> Object (pos, entries))
pos_gen
(QCheck.Gen.list_size
(QCheck.Gen.int_range 0 3)
- (QCheck.Gen.pair QCheck.Gen.string (gen_expr_sized (n-1)))
+ (QCheck.Gen.oneof [
+ (QCheck.Gen.map
+ (fun expr -> ObjectExpr expr)
+ (gen_expr_sized (n-1))
+ );
+ (QCheck.Gen.map2
+ (fun field expr -> ObjectField (field, expr))
+ QCheck.Gen.string
+ (gen_expr_sized (n-1))
+ )
+ ]
+ )
)
);
(1, QCheck.Gen.map4
Bonus round: refactoring specialized interpreting functions
The pattern-matching cases in the interpret function are getting bloated. It's a nice idea to start refactoring some code in specialized functions.
Here's the interpret_array:
diff --git a/lib/interpreter.ml b/lib/interpreter.ml
index fa4b586..1646a14 100644
--- a/lib/interpreter.ml
+++ b/lib/interpreter.ml
@@ -64,16 +64,7 @@ let rec interpret env expr =
match expr with
| Null _ | Bool _ | String _ | Number _ -> ok (env, expr)
| Object (pos, entries) -> interpret_object env (pos, entries) interpret
- | Array (pos, exprs) ->
- (let rec eval' env' exprs' =
- match exprs' with
- | [] -> ok (env', [])
- | e :: exprs ->
- (let* (env1, expr') = interpret env' e in
- let* (env2, rest) = eval' env1 exprs in
- ok (env2, expr' :: rest))
- in eval' env exprs >>= fun (env3, exprs') -> ok (env3, Array (pos, exprs'))
- )
+ | Array (pos, exprs) -> interpret_array env (pos, exprs)
| Ident (pos, varname) ->
Env.find_var varname env
~succ:(fun env' expr -> interpret env' expr)
@@ -116,6 +107,17 @@ let rec interpret env expr =
)
~err:(Error.error_at pos)
+and interpret_array env (pos, exprs) =
+ let* (env', evaluated_exprs) = List.fold_left
+ (fun result expr ->
+ let* (env', result') = result in
+ let* (env'', expr') = interpret env' expr in
+ ok (env'', result' @ [expr'])
+ )
+ (ok (env, []))
+ exprs
+ in ok (env', Array (pos, evaluated_exprs))
+
let eval expr =
let* (_env, evaluated_expr) = interpret Env.empty expr
in ok evaluated_expr
Just like with types, we can have mutually recursive functions. Here we need it to allow interpret_array to call interpret and vice-versa.
Remember the interpret_object function, where I worked around the recursive call by passing the interpret function as a parameter? As a mutually recursive function, there's no need to pass it as a parameter anymore:
diff --git a/lib/interpreter.ml b/lib/interpreter.ml
index 1646a14..4121b9d 100644
--- a/lib/interpreter.ml
+++ b/lib/interpreter.ml
@@ -41,29 +41,11 @@ let interpret_unary_op (op: unary_op) (evaluated_expr: expr) =
| BitwiseNot, Number (pos, Int i) -> ok (Number (pos, Int (lnot i)))
| _ -> error "Invalid unary operation"
-let rec interpret_object env (pos, entries) interpret =
- let* (result_env, evaluated_entries) = List.fold_left
- (fun result entry ->
- let* (env', entries') = result in
- let* (env'', evaluated_entry) = interpret_entry env' entry interpret in
- ok (env'', entries' @ [evaluated_entry])
- )
- (ok (env, []))
- entries
- in
- ok (result_env, Object (pos, evaluated_entries))
-and interpret_entry env expr interpret =
- match expr with
- | ObjectExpr expr ->
- interpret env expr >>= fun (env', expr') -> ok (env', ObjectExpr expr')
- | ObjectField (varname, expr) ->
- interpret env expr >>= fun (env', expr') -> ok (env', ObjectField (varname, expr'))
-
(** [interpret expr] interprets and reduce the intermediate AST [expr] into a result AST. *)
let rec interpret env expr =
match expr with
| Null _ | Bool _ | String _ | Number _ -> ok (env, expr)
- | Object (pos, entries) -> interpret_object env (pos, entries) interpret
+ | Object (pos, entries) -> interpret_object env (pos, entries)
| Array (pos, exprs) -> interpret_array env (pos, exprs)
| Ident (pos, varname) ->
Env.find_var varname env
@@ -118,6 +100,24 @@ and interpret_array env (pos, exprs) =
exprs
in ok (env', Array (pos, evaluated_exprs))
+and interpret_object env (pos, entries) =
+ let* (result_env, evaluated_entries) = List.fold_left
+ (fun result entry ->
+ let* (env', entries') = result in
+ let* (env'', evaluated_entry) = interpret_obj_entry env' entry in
+ ok (env'', entries' @ [evaluated_entry])
+ )
+ (ok (env, []))
+ entries
+ in
+ ok (result_env, Object (pos, evaluated_entries))
+and interpret_obj_entry env expr =
+ match expr with
+ | ObjectExpr expr ->
+ interpret env expr >>= fun (env', expr') -> ok (env', ObjectExpr expr')
+ | ObjectField (varname, expr) ->
+ interpret env expr >>= fun (env', expr') -> ok (env', ObjectField (varname, expr'))
+
let eval expr =
let* (_env, evaluated_expr) = interpret Env.empty expr
in ok evaluated_expr
Neat, huh?!
Conclusion
With local definitions now working inside objects, Tsonnet is starting to feel more like the dynamic configuration language it's meant to be. The ability to define variables right alongside object fields opens up possibilities for cleaner, more maintainable configuration files -- no more repeating the same values throughout your objects!
The entire diff can be seen here.
Next up in the series, we'll explore Jsonnet references. The cocktail recipes are getting more sophisticated, and so is our language!
Thanks for reading Bit Maybe Wise! Ready to mix more variables into your configuration? Subscribe to get fresh Tsonnet updates served up regularly -- like a well-crafted drink, it gets better with the right ingredients! 🍸
Photo by Debby Ledet on Unsplash