Add support for strings and byte arrays (#97)

Implement string and byte array support as described in
OpenCyphal/specification#51. The grammar has been modified
as follows:

- Add `utf8` primitive.
- Add `byte` primitive.
- Remove the unnecessary form `saturated bool`, which was shorthand for
`bool`, for regularity. From now on, only unqualified `bool` is allowed.
The expected impact is zero since the long form was never seen in the
wild. Note that `truncated bool` was never legal.

At the moment, the following constraints are enforced; they may be
lifted in the future:

- `utf8` can only be used as an element type of a variable-length array:
```
utf8[<=10] name  # valid
utf8[10]   name  # invalid
utf8       name  # invalid
```

- `byte` can only be used as an element type of an array:
```
byte[<=10] data  # valid
byte[10]   data  # valid
byte       data  # invalid
```

Both `utf8` and `byte` are concrete instances of `UnsignedIntegerType`.
By virtue of this, existing users of PyDSDL (like Nunavut) will
interpret both `utf8` and `byte` as `truncated uint8`, until explicitly
updated to take advantage of the new feature. That is:

```
utf8[<=10] name  # seen by Nunavut as: truncated uint8[<=10]
utf8[10]   name  # frontend error
utf8       name  # frontend error
byte[<=10] data  # seen by Nunavut as: truncated uint8[<=10]
byte[10]   data  # seen by Nunavut as: truncated uint8[10]
byte       data  # frontend error
```

Property `pydsdl.ArrayType.string_like` is now deprecated and should be
replaced with an explicit `isinstance(array.element_type,
pydsdl.UTF8Type)`.

Some internal refactoring has been done to unify the deprecation
consistency checking with the aggregation constraints listed above
(e.g., `utf8[<=10]` is valid but `utf8[10]` is not).

Closes #96
Relates to OpenCyphal/yakut#65

Author: pavel-kirienko

.github/workflows/test-and-release.yml

@@ -1,11 +1,6 @@
 name: 'Test and Release PyDSDL'
 on: [ push, pull_request ]
 
-# Ensures that only one workflow is running at a time
-concurrency:
-  group: ${{ github.workflow_sha }}
-  cancel-in-progress: true
-
 jobs:
   pydsdl-test:
     name: Test PyDSDL

noxfile.py

@@ -98,7 +98,7 @@ def lint(session):
     )
     if is_latest_python(session):
         # we run black only on the newest Python version to ensure that the code is formatted with the latest version
-        session.install("black ~= 24.4")
+        session.install("black ~= 25.1")
         session.run("black", "--check", ".")
 
 

pydsdl/__init__.py

@@ -7,11 +7,11 @@
 import sys as _sys
 from pathlib import Path as _Path
 
-__version__ = "1.22.2"
+__version__ = "1.23.0"
 __version_info__ = tuple(map(int, __version__.split(".")[:3]))
 __license__ = "MIT"
 __author__ = "OpenCyphal"
-__copyright__ = "Copyright (c) 2018 OpenCyphal"
+__copyright__ = "Copyright (c) OpenCyphal development team"
 __email__ = "[email protected]"
 
 # Our unorthodox approach to dependency management requires us to apply certain workarounds.
@@ -42,6 +42,8 @@
 from ._serializable import IntegerType as IntegerType
 from ._serializable import SignedIntegerType as SignedIntegerType
 from ._serializable import UnsignedIntegerType as UnsignedIntegerType
+from ._serializable import ByteType as ByteType
+from ._serializable import UTF8Type as UTF8Type
 from ._serializable import FloatType as FloatType
 from ._serializable import VoidType as VoidType
 from ._serializable import ArrayType as ArrayType

pydsdl/_bit_length_set/_bit_length_set.py

@@ -347,7 +347,7 @@ def elementwise_sum_k_multicombinations(self, k: int) -> "BitLengthSet":  # prag
 
     @staticmethod
     def elementwise_sum_cartesian_product(
-        sets: typing.Iterable[typing.Union[typing.Iterable[int], int]]
+        sets: typing.Iterable[typing.Union[typing.Iterable[int], int]],
     ) -> "BitLengthSet":  # pragma: no cover
         """
         :meta private:

pydsdl/_dsdl.py

@@ -119,6 +119,8 @@ def read(
         definition_visitors: Iterable["DefinitionVisitor"],
         print_output_handler: Callable[[int, str], None],
         allow_unregulated_fixed_port_id: bool,
+        *,
+        strict: bool = False,
     ) -> CompositeType:
         """
         Reads the data type definition and returns its high-level data type representation.
@@ -133,6 +135,7 @@ def read(
         :param definition_visitors:             Visitors to notify about discovered dependencies.
         :param print_output_handler:            Used for @print and for diagnostics: (line_number, text) -> None.
         :param allow_unregulated_fixed_port_id: Do not complain about fixed unregulated port IDs.
+        :param strict:                          Reject features that are not part of the Specification.
         :return: The data type representation.
         """
         raise NotImplementedError()

pydsdl/_dsdl_definition.py

@@ -235,6 +235,8 @@ def read(
         definition_visitors: Iterable[DefinitionVisitor],
         print_output_handler: Callable[[int, str], None],
         allow_unregulated_fixed_port_id: bool,
+        *,
+        strict: bool = False,
     ) -> CompositeType:
         log_prefix = "%s.%d.%d" % (self.full_name, self.version.major, self.version.minor)
         if self._cached_type is not None:
@@ -262,7 +264,7 @@ def read(
                 allow_unregulated_fixed_port_id=allow_unregulated_fixed_port_id,
             )
 
-            _parser.parse(self.text, builder)
+            _parser.parse(self.text, builder, strict=strict)
 
             self._cached_type = builder.finalize()
             _logger.info(
@@ -536,5 +538,5 @@ def _unittest_type_from_path_inference_edge_case(temp_dsdl_factory) -> None:  #
 
 def _unittest_from_first_in(temp_dsdl_factory) -> None:  # type: ignore
     dsdl_file = temp_dsdl_factory.new_file(Path("repo/uavcan/foo/bar/435.baz.1.0.dsdl"), "@sealed")
-    dsdl_def = DSDLDefinition.from_first_in(dsdl_file.resolve(), [(dsdl_file.parent.parent / "..")])
+    dsdl_def = DSDLDefinition.from_first_in(dsdl_file.resolve(), [dsdl_file.parent.parent / ".."])
     assert dsdl_def.full_name == "uavcan.foo.bar.baz"

pydsdl/_expression/_container.py

@@ -32,7 +32,7 @@ class Set(Container):
     class _Decorator:
         @staticmethod
         def homotypic_binary_operator(
-            inferior: typing.Callable[["Set", "Set"], _O]
+            inferior: typing.Callable[["Set", "Set"], _O],
         ) -> typing.Callable[["Set", "Set"], _O]:
             def wrapper(self: "Set", other: "Set") -> _O:
                 assert isinstance(self, Set) and isinstance(other, Set)

pydsdl/_namespace.py

@@ -74,6 +74,8 @@ def read_namespace(
     print_output_handler: PrintOutputHandler | None = None,
     allow_unregulated_fixed_port_id: bool = False,
     allow_root_namespace_name_collision: bool = True,
+    *,
+    strict: bool = False,
 ) -> list[_serializable.CompositeType]:
     """
     This function is a main entry point for the library.
@@ -103,6 +105,8 @@ def read_namespace(
              the same root namespace name multiple times in the lookup dirs. This will enable defining a namespace
              partially and let other entities define new messages or new sub-namespaces in the same root namespace.
 
+    :param strict: Reject features that are not [yet] part of the Cyphal Specification.
+
     :return: A list of :class:`pydsdl.CompositeType` found under the `root_namespace_directory` and sorted
              lexicographically by full data type name, then by major version (newest version first), then by minor
              version (newest version first). The ordering guarantee allows the caller to always find the newest version
@@ -131,7 +135,11 @@ def read_namespace(
         _logger.debug(_LOG_LIST_ITEM_PREFIX + str(x))
 
     return _complete_read_function(
-        target_dsdl_definitions, lookup_directories_path_list, print_output_handler, allow_unregulated_fixed_port_id
+        target_dsdl_definitions,
+        lookup_directories_path_list,
+        print_output_handler,
+        allow_unregulated_fixed_port_id=allow_unregulated_fixed_port_id,
+        strict=strict,
     ).direct
 
 
@@ -142,6 +150,8 @@ def read_files(
     lookup_directories: None | Path | str | Iterable[Path | str] = None,
     print_output_handler: PrintOutputHandler | None = None,
     allow_unregulated_fixed_port_id: bool = False,
+    *,
+    strict: bool = False,
 ) -> tuple[list[_serializable.CompositeType], list[_serializable.CompositeType]]:
     """
     This function is a main entry point for the library.
@@ -232,6 +242,8 @@ def read_files(
         This is a dangerous feature that must not be used unless you understand the risks.
         Please read https://opencyphal.org/guide.
 
+    :param strict: Reject features that are not [yet] part of the Cyphal Specification.
+
     :return: A Tuple of lists of :class:`pydsdl.CompositeType`. The first index in the Tuple are the types parsed from
         the ``dsdl_files`` argument. The second index are types that the target ``dsdl_files`` utilizes.
         A note for using these values to describe build dependencies: each :class:`pydsdl.CompositeType` has two
@@ -266,9 +278,12 @@ def read_files(
     )
 
     definitions = _complete_read_function(
-        target_dsdl_definitions, lookup_directories_path_list, print_output_handler, allow_unregulated_fixed_port_id
+        target_dsdl_definitions,
+        lookup_directories_path_list,
+        print_output_handler,
+        allow_unregulated_fixed_port_id=allow_unregulated_fixed_port_id,
+        strict=strict,
     )
-
     return (definitions.direct, definitions.transitive)
 
 
@@ -286,7 +301,9 @@ def _complete_read_function(
     target_dsdl_definitions: SortedFileList[ReadableDSDLFile],
     lookup_directories_path_list: list[Path],
     print_output_handler: PrintOutputHandler | None,
+    *,
     allow_unregulated_fixed_port_id: bool,
+    strict: bool,
 ) -> DSDLDefinitions:
 
     lookup_dsdl_definitions = _construct_dsdl_definitions_from_namespaces(lookup_directories_path_list)
@@ -307,7 +324,11 @@ def _complete_read_function(
     # This is the biggie. All the rest of the wrangling is just to get to this point. This will take the
     # most time and memory.
     definitions = read_definitions(
-        target_dsdl_definitions, lookup_dsdl_definitions, print_output_handler, allow_unregulated_fixed_port_id
+        target_dsdl_definitions,
+        lookup_dsdl_definitions,
+        print_output_handler,
+        allow_unregulated_fixed_port_id=allow_unregulated_fixed_port_id,
+        strict=strict,
     )
 
     # Note that we check for collisions in the read namespace only.

pydsdl/_namespace_reader.py

@@ -19,12 +19,14 @@
 def _read_definitions(
     target_definitions: SortedFileList[ReadableDSDLFile],
     lookup_definitions: SortedFileList[ReadableDSDLFile],
+    *,
     print_output_handler: PrintOutputHandler | None,
     allow_unregulated_fixed_port_id: bool,
+    strict: bool,
     direct: set[CompositeType],
     transitive: set[CompositeType],
     file_pool: dict[Path, ReadableDSDLFile],
-    level: int,
+    level: int = 0,
 ) -> None:
     """
     Don't look at me! I'm hideous!
@@ -87,12 +89,13 @@ def print_handler(file: Path, line: int, message: str) -> None:
             _read_definitions(
                 dsdl_file_sort(_pending_definitions),
                 lookup_definitions,
-                print_output_handler,
-                allow_unregulated_fixed_port_id,
-                direct,
-                transitive,
-                file_pool,
-                level + 1,
+                print_output_handler=print_output_handler,
+                allow_unregulated_fixed_port_id=allow_unregulated_fixed_port_id,
+                strict=strict,
+                direct=direct,
+                transitive=transitive,
+                file_pool=file_pool,
+                level=level + 1,
             )
             _pending_definitions.clear()
 
@@ -116,6 +119,8 @@ def read_definitions(
     lookup_definitions: SortedFileList[ReadableDSDLFile],
     print_output_handler: PrintOutputHandler | None,
     allow_unregulated_fixed_port_id: bool,
+    *,
+    strict: bool = False,
 ) -> DSDLDefinitions:
     """
     Given a set of DSDL files, this method reads the text and invokes the parser for each and for any files found in the
@@ -125,6 +130,7 @@ def read_definitions(
     :param lookup_definitions:              List of definitions available for referring to.
     :param print_output_handler:            Used for @print and for diagnostics: (line_number, text) -> None.
     :param allow_unregulated_fixed_port_id: Do not complain about fixed unregulated port IDs.
+    :param strict:                          Reject features that are not part of the Specification.
     :return: The data type representation.
     :raises InvalidDefinitionError: If a dependency is missing.
     :raises InternalError: If an unexpected error occurs.
@@ -135,12 +141,12 @@ def read_definitions(
     _read_definitions(
         target_definitions,
         lookup_definitions,
-        print_output_handler,
-        allow_unregulated_fixed_port_id,
-        _direct,
-        _transitive,
-        _file_pool,
-        0,
+        print_output_handler=print_output_handler,
+        allow_unregulated_fixed_port_id=allow_unregulated_fixed_port_id,
+        strict=strict,
+        direct=_direct,
+        transitive=_transitive,
+        file_pool=_file_pool,
     )
     return DSDLDefinitions(
         dsdl_file_sort(_direct),
@@ -186,6 +192,7 @@ def _unittest_namespace_reader_read_definitions_multiple(temp_dsdl_factory) -> N
     assert len(definitions.transitive) == 2
 
 
+# noinspection PyProtectedMember
 def _unittest_namespace_reader_read_definitions_multiple_no_load(temp_dsdl_factory) -> None:  # type: ignore
     """
     Ensure that the loader does not load files that are not in the transitive closure of the target files.

pydsdl/_parser.py

@@ -21,12 +21,12 @@ class DSDLSyntaxError(_error.InvalidDefinitionError):
     pass
 
 
-def parse(text: str, statement_stream_processor: "StatementStreamProcessor") -> None:
+def parse(text: str, statement_stream_processor: "StatementStreamProcessor", *, strict: bool) -> None:
     """
     The entry point of the parser. As the text is being parsed, the parser invokes appropriate
     methods in the statement stream processor.
     """
-    pr = _ParseTreeProcessor(statement_stream_processor)
+    pr = _ParseTreeProcessor(statement_stream_processor, strict=strict)
     try:
         pr.visit(_get_grammar().parse(text))  # type: ignore
     except _error.FrontendError as ex:
@@ -134,12 +134,13 @@ class _ParseTreeProcessor(parsimonious.NodeVisitor):
     # Beware that those might be propagated from recursive parser instances!
     unwrapped_exceptions = (_error.FrontendError, SystemError, MemoryError, SystemExit)  # type: ignore
 
-    def __init__(self, statement_stream_processor: StatementStreamProcessor):
+    def __init__(self, statement_stream_processor: StatementStreamProcessor, *, strict: bool):
         assert isinstance(statement_stream_processor, StatementStreamProcessor)
         self._statement_stream_processor = statement_stream_processor  # type: StatementStreamProcessor
         self._current_line_number = 1  # Lines are numbered from one
         self._comment = ""
         self._comment_is_header = True
+        self._strict = bool(strict)
         super().__init__()
 
     @property
@@ -265,6 +266,15 @@ def visit_type_version_specifier(self, _n: _Node, children: _Children) -> _seria
         assert isinstance(major, _expression.Rational) and isinstance(minor, _expression.Rational)
         return _serializable.Version(major=major.as_native_integer(), minor=minor.as_native_integer())
 
+    def visit_type_primitive_boolean(self, _n: _Node, _c: _Children) -> _serializable.PrimitiveType:
+        return _serializable.BooleanType()
+
+    def visit_type_primitive_byte(self, _n: _Node, _c: _Children) -> _serializable.PrimitiveType:
+        return _serializable.ByteType()
+
+    def visit_type_primitive_utf8(self, _n: _Node, _c: _Children) -> _serializable.PrimitiveType:
+        return _serializable.UTF8Type()
+
     def visit_type_primitive_truncated(self, _n: _Node, children: _Children) -> _serializable.PrimitiveType:
         _kw, _sp, cons = cast(Tuple[_Node, _Node, _PrimitiveTypeConstructor], children)
         return cons(_serializable.PrimitiveType.CastMode.TRUNCATED)
@@ -273,9 +283,6 @@ def visit_type_primitive_saturated(self, _n: _Node, children: _Children) -> _ser
         _, cons = cast(Tuple[_Node, _PrimitiveTypeConstructor], children)
         return cons(_serializable.PrimitiveType.CastMode.SATURATED)
 
-    def visit_type_primitive_name_boolean(self, _n: _Node, _c: _Children) -> _PrimitiveTypeConstructor:
-        return typing.cast(_PrimitiveTypeConstructor, _serializable.BooleanType)
-
     def visit_type_primitive_name_unsigned_integer(self, _n: _Node, children: _Children) -> _PrimitiveTypeConstructor:
         return lambda cm: _serializable.UnsignedIntegerType(children[-1], cm)