Fix serialization errors

conway-cddl/codegen/generators/array.ts

@@ -2,11 +2,11 @@ import { CodeGeneratorBase, CodeGeneratorBaseOptions } from ".";
 import { SchemaTable } from "..";
 
 export type GenArrayOptions = {
-  item: string;
+  item: string | undefined;
 } & CodeGeneratorBaseOptions;
 
 export class GenArray extends CodeGeneratorBase {
-  item: string;
+  item: string | undefined;
 
   constructor(
     name: string,
@@ -18,60 +18,89 @@ export class GenArray extends CodeGeneratorBase {
   }
 
   private itemJsType() {
-    return this.typeUtils.jsType(this.item);
+    return this.item ? `${this.typeUtils.jsType(this.item)}` : undefined;
   }
 
   generateMembers(): string {
-    return `private items: ${this.itemJsType()}[];`;
+    const jsType = this.itemJsType();
+    return `
+      private items: ${jsType ? `${jsType}[]` : `Uint32Array`};
+      private definiteEncoding: boolean;
+    `;
   }
 
   generateConstructor(): string {
+    const jsType = this.itemJsType();
     return `
-        constructor(items: ${this.itemJsType()}[]) {
+        constructor(items: ${jsType ? `${jsType}[]` : `Uint32Array`}, definiteEncoding: boolean = true) {
           this.items = items;
+          this.definiteEncoding = definiteEncoding;
         }
     `;
   }
 
   generateExtraMethods(): string {
-    let itemJsType = this.typeUtils.jsType(this.item);
+    let jsType = this.itemJsType();
     return `
         static new(): ${this.name} {
-          return new ${this.name}([]);
+          return new ${this.name}(${jsType ? "[]" : "new Uint32Array([])"});
         }
 
         len(): number {
           return this.items.length;
         }
 
-        get(index: number): ${itemJsType} {
-          if(index >= this.items.length) throw new Error("Array out of bounds");
-          return this.items[index];
-        }
+        ${jsType ?
+          `
+          get(index: number): ${jsType} {
+            if(index >= this.items.length) throw new Error("Array out of bounds");
+            return this.items[index];
+          }
 
-        add(elem: ${itemJsType}): void {
-          this.items.push(elem);
+          add(elem: ${jsType}): void {
+            this.items.push(elem);
+          }
+          ` : ''
         }
     `;
   }
 
   generateDeserialize(reader: string, path: string): string {
-    return `
-      return new ${this.name}(
-        ${reader}.readArray(
-          (reader, idx) => ${this.typeUtils.readType("reader", this.item, `[...${path}, "Elem#" + idx]`)}
-          , ${path}
-        )
-      );
-    `;
+    if (this.item) {
+      return `
+        const { items, definiteEncoding } = 
+          ${reader}.readArray(
+            (reader, idx) => ${this.typeUtils.readType("reader", this.item, `[...${path}, "Elem#" + idx]`)}
+            , ${path}
+          )
+        return new ${this.name}(items, definiteEncoding);
+      `;
+    } else {
+      return `
+        const { items, definiteEncoding } = 
+          ${reader}.readArray(
+            (reader, idx) => Number(reader.readUint([...${path}, "Byte#" + idx]))
+            , ${path}
+          )
+
+        return new ${this.name}(new Uint32Array(items), definiteEncoding);
+      `;
+    }
   }
 
   generateSerialize(writer: string): string {
-    return `
-      ${writer}.writeArray(
-        this.items,
-        (writer, x) => ${this.typeUtils.writeType("writer", "x", this.item)}
-      );
-    `;
+    if (this.item) {
+      return `
+        ${writer}.writeArray(
+          this.items,
+          (writer, x) => ${this.typeUtils.writeType("writer", "x", this.item)},
+          this.definiteEncoding
+        );
+      `;
+    } else {
+      return `
+        ${writer}.writeArray(this.items, (writer, x) => writer.writeInt(BigInt(x)), this.definiteEncoding)
+      `
+    }
   }
 }

conway-cddl/codegen/generators/index.ts

@@ -267,8 +267,6 @@ export class TypeUtils {
         return yamlType;
       case "bytes":
         return "Uint8Array";
-      case "arrayToUint32Array":
-        return "Uint32Array";
       default:
         console.error("Unknown type: " + yamlType);
         return "unknown";
@@ -292,8 +290,6 @@ export class TypeUtils {
         return `${reader}.readBoolean(${path})`;
       case "bytes":
         return `${reader}.readBytes(${path})`;
-      case "arrayToUint32Array":
-        return `new Uint32Array(${reader}.readArray((reader) => Number(reader.readUint(${path})), ${path}));`;
       default:
         console.error("Can't decode: " + type);
         return `$$CANT_READ("${type}")`;
@@ -317,8 +313,6 @@ export class TypeUtils {
         return `${writer}.writeBoolean(${value})`;
       case "bytes":
         return `${writer}.writeBytes(${value})`;
-      case "arrayToUint32Array":
-        return `${writer}.writeArray(${value}, (writer, x) => writer.writeInt(BigInt(x)))`;
       default:
         console.error("Can't encode: " + type);
         return `$$CANT_WRITE("${type}")`;
@@ -338,8 +332,6 @@ export class TypeUtils {
       case "boolean":
         return `${var1} === ${var2}`;
       case "bytes":
-      case "arrayToUint32Array":
-        return `arrayEq(${var1}, ${var2})`;
       default:
         console.error("Can't compare: " + type);
         return `$$CANT_EQ("${type}")`;

conway-cddl/codegen/generators/set.ts

@@ -20,13 +20,21 @@ export class GenSet extends CodeGeneratorBase {
   generateMembers(): string {
     return `
       private items: ${this.itemJsType()}[];
+      private definiteEncoding: boolean;
+      private nonEmptyTag: boolean;
+
+      private setItems(items: ${this.itemJsType()}[]) {
+        this.items = items;
+      }
     `;
   }
 
   generateConstructor(): string {
     return `
-      constructor() {
+      constructor(definiteEncoding: boolean = true, nonEmptyTag: boolean = true) {
         this.items = [];
+        this.definiteEncoding = definiteEncoding;
+        this.nonEmptyTag = nonEmptyTag;
       }
     `;
   }
@@ -66,19 +74,31 @@ export class GenSet extends CodeGeneratorBase {
 
   generateDeserialize(reader: string, path: string): string {
     return `
-      let ret = new ${this.name}();
+      let nonEmptyTag = false;
       if(${reader}.peekType(${path}) == "tagged") {
         let tag = ${reader}.readTaggedTag(${path});
-        if(tag != 258) throw new Error("Expected tag 258. Got " + tag);
+        if(tag != 258) {
+          throw new Error("Expected tag 258. Got " + tag);
+        } else {
+          nonEmptyTag = true;
+        }
       }
-      ${reader}.readArray((reader, idx) => ret.add(${this.typeUtils.readType(reader, this.item, `[...${path}, '${this.item}#' + idx]`)}), ${path});
+      const { items, definiteEncoding } = ${reader}.readArray(
+        (reader, idx) =>
+          ${this.typeUtils.readType( reader, this.item, `[...${path}, '${this.item}#' + idx]`)},
+          ${path}
+      );
+      let ret = new ${this.name}(definiteEncoding, nonEmptyTag);
+      ret.setItems(items);
       return ret;
     `;
   }
 
   generateSerialize(writer: string): string {
     return `
-      ${writer}.writeTaggedTag(258);
+      if (this.nonEmptyTag) {
+        ${writer}.writeTaggedTag(258);
+      }
       ${writer}.writeArray(this.items, (writer, x) => ${this.typeUtils.writeType("writer", "x", this.item)});
     `;
   }

conway-cddl/codegen/generators/structured/index.ts

@@ -38,6 +38,10 @@ export class GenStructuredBase<
     return this.options.fields;
   }
 
+  getMinFields(): number {
+    return this.options.fields.filter((v) => !v.optional).length
+  }
+
   generateMembers(): string {
     return this.options.fields
       .map((x) => `private _${x.name}: ${this.fieldType(x)};`)

conway-cddl/codegen/generators/structured/record.ts

@@ -5,6 +5,7 @@ export type Field = {
   name: string;
   type: string;
   nullable?: boolean;
+  optional?: boolean;
 };
 
 export type GenRecordOptions = {
@@ -24,21 +25,23 @@ export class GenRecord extends GenStructuredBase<Field> {
     return `
       let len = ${reader}.readArrayTag(${path});
 
-      if(len != null && len < ${this.getFields().length}) {
-        throw new Error("Insufficient number of fields in record. Expected ${this.getFields().length}. Received " + len + "(at " + path.join("/"));
+      if(len != null && len < ${this.getMinFields()}) {
+        throw new Error("Insufficient number of fields in record. Expected at least ${this.getMinFields()}. Received " + len + "(at " + path.join("/"));
       }
 
-      ${this.getFields()
-        .map(
-          (x) => `
-          const ${x.name}_path = [...${path}, '${x.type}(${x.name})'];
-          let ${x.name} = ${
-            x.nullable
-              ? `${reader}.readNullable(r => ${this.typeUtils.readType("r", x.type, `${x.name}_path`)}, ${path})?? undefined`
-              : this.typeUtils.readType(reader, x.type, `${x.name}_path`)
-          };`,
-        )
-        .join("\n")}
+      ${this.getFields().reduce((acc, x, idx) => {
+         acc.push(`
+           const ${x.name}_path = [...${path}, '${x.type}(${x.name})'];
+           let ${x.name} = ${
+             x.optional ?
+               `len != null && len > ${idx} ? ${this.typeUtils.readType(reader, x.type, `${x.name}_path`)} : undefined` :
+             x.nullable ?
+               `${reader}.readNullable(r => ${this.typeUtils.readType("r", x.type, `${x.name}_path`)}, ${path})?? undefined` :
+             this.typeUtils.readType(reader, x.type, `${x.name}_path`)
+           }
+         `);
+         return acc;
+      }, [] as string[]).join("\n")}
 
       return new ${this.name}(${this.getFields()
         .map((x) => x.name)
@@ -48,17 +51,30 @@ export class GenRecord extends GenStructuredBase<Field> {
 
   generateSerialize(writer: string): string {
     return `
-      ${writer}.writeArrayTag(${this.getFields().length});
+      let arrayLen = ${this.getMinFields()};
+      ${this.getFields().
+        map((x) => x.optional
+          ? `if(this._${x.name}) {
+              arrayLen++;            
+          }` :
+          "")
+          .join("\n")}
+
+      ${writer}.writeArrayTag(arrayLen);
 
       ${this.getFields()
         .map((x) =>
+          x.optional
+            ? `if(this._${x.name}) {
+                 ${this.typeUtils.writeType(writer, `this._${x.name}`, x.type)};   
+              }` :
           x.nullable
             ? `if(this._${x.name} == null) { 
                   ${writer}.writeNull();
               } else { 
                   ${this.typeUtils.writeType(writer, `this._${x.name}`, x.type)};
-              }`
-            : `${this.typeUtils.writeType(writer, `this._${x.name}`, x.type)};`,
+              }` :
+          `${this.typeUtils.writeType(writer, `this._${x.name}`, x.type)};`,
         )
         .join("\n")}
     `;

conway-cddl/codegen/generators/tagged_record.ts

@@ -163,14 +163,18 @@ export class GenTaggedRecord extends CodeGeneratorBase {
               break;
             `,
           )
+          .concat([`
+            default:
+              throw new Error("Unexpected tag for ${this.name}: " + tag + "(at " + ${path}.join("/") + ")");
+          `])
           .join("\n")}
       }
 
       if(len == null) {
         ${reader}.readBreak();
       }
-      
-      throw new Error("Unexpected tag for ${this.name}: " + tag + "(at " + ${path}.join("/") + ")");
+
+      return new ${this.name}(variant);
     `;
   }
 

conway-cddl/codegen/types.ts

@@ -16,7 +16,7 @@ export const Schema = Type.Intersect([
     extra_methods: Type.Optional(Type.String()),
   }),
   Type.Union([
-    Type.Object({ type: Type.Literal("array"), item: Type.String() }),
+    Type.Object({ type: Type.Literal("array"), item: Type.Optional(Type.String()) }),
     Type.Object({
       type: Type.Literal("enum"),
       values: Type.Array(
@@ -78,6 +78,7 @@ export const Schema = Type.Intersect([
           name: Type.String(),
           type: Type.String(),
           nullable: Type.Optional(Type.Boolean()),
+          optional: Type.Optional(Type.Boolean())
         }),
       ),
       accessor_get_prefix: Type.Optional(Type.Boolean()),