Something went wrong. Try again.
Decode the protobuf wire format in Gleam! protobin.hexdocs.pm
gleam protobuf
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423import gleam/bit_arrayimport gleam/dictimport gleam/dynamic.{type Dynamic}import gleam/dynamic/decode.{type Decoder}import gleam/intimport gleam/listimport gleam/optionimport gleam/result
import protobin/internal/utilimport protobin/internal/wire_type.{type WireType}
pub type BytePos = Int
pub type ParseError { UnknownWireType(Int, pos: BytePos) InvalidVarInt(leftover_bits: BitArray, acc: BitArray, pos: BytePos) InvalidFixed(size: Int, bits: BitArray, pos: BytePos) InvalidLen(len: Int, value_bits: BitArray, pos: BytePos) UnableToDecode(List(decode.DecodeError)) UnexpectedGroup(wire_type: WireType, pos: BytePos)}
pub type DecodeResult(t) = Result(t, ParseError)
pub type Parsed(t) { Parsed(value: t, rest: BitArray, pos: BytePos)}
fn parsed_map(of parsed: Parsed(t), with fun: fn(t) -> u) -> Parsed(u) { let Parsed(value:, rest:, pos:) = parsed Parsed(value: fun(value), rest:, pos:)}
type Field { Field(key: Dynamic, value: Dynamic)}
pub type ValueResult = DecodeResult(Parsed(BitArray))
pub type ValueParser = fn(BitArray, BytePos) -> ValueResult
pub type Config { Config( /// Whether or not to ignore groups and allow parsing to succeed if one is /// encountered. If ignored, all nested fields will be treated as if they /// were not nested. /// /// Defaults to `False`. ignore_groups: Bool, )}
const config_default = Config(ignore_groups: False)
pub fn parse( from bits: BitArray, using decoder: Decoder(t),) -> DecodeResult(Parsed(t)) { parse_with_config(from: bits, using: decoder, config: config_default)}
pub fn parse_with_config( from bits: BitArray, using decoder: Decoder(t), config config: Config,) -> DecodeResult(Parsed(t)) { use Parsed(value: data, rest:, pos:) <- result.try(read_fields( bits, [], 0, config, )) decode.run(data, decoder) |> result.map(fn(value) { Parsed(value:, rest:, pos:) }) |> result.map_error(UnableToDecode)}
fn read_fields( bits: BitArray, acc: List(Field), pos: BytePos, config: Config,) -> DecodeResult(Parsed(Dynamic)) { case bits { <<>> -> acc |> repeated_to_list |> dict.to_list |> dynamic.properties |> Parsed(value: _, rest: bits, pos:) |> Ok bits -> { use Parsed(value: prop, rest: bits, pos:) <- result.try(read_field( bits, pos, config, )) read_fields(bits, [prop, ..acc], pos, config) } }}
fn repeated_to_list( reversed_fields: List(Field),) -> dict.Dict(Dynamic, Dynamic) { let fields = { // Every field is a list of values for two reasons: // a) expanded repeated values are encoded as repeated fields // b) if a non-repeating field is defined twice, then the last value should // be considered the correct one. since the parser doesn't know which // fields are repeating, it parses all fields as a list and then the // decoders will handle choosing which value(s) to keep. let acc: dict.Dict(Dynamic, List(Dynamic)) = dict.new() use fields, Field(key:, value:) <- list.fold( over: reversed_fields, from: acc, )
use existing_values <- dict.upsert(in: fields, update: key) let existing_values = option.unwrap(existing_values, or: []) // The repeated values were previously reversed, now being reversed again [value, ..existing_values] }
dict.map_values(in: fields, with: fn(_key, field) { field |> dynamic.list })}
fn read_field( bits: BitArray, tag_pos: BytePos, config: Config,) -> DecodeResult(Parsed(Field)) { use Parsed(value: tag, rest: bits, pos:) <- result.try(parse_varint( bits, tag_pos, )) let tag = util.bit_array_to_uint(tag)
let field_id = tag |> int.bitwise_shift_right(3) let wire_type = tag |> int.bitwise_and(0b111)
use wire_type <- result.try(option.to_result( wire_type.parse(wire_type), UnknownWireType(wire_type, pos: tag_pos), ))
// Fail when groups are found use wire_type <- result.try({ case wire_type, config { wire_type.SGroup, Config(ignore_groups: False) | wire_type.EGroup, Config(ignore_groups: False) -> Error(UnexpectedGroup(wire_type:, pos: tag_pos)) _, _ -> Ok(wire_type) } })
let read = wire_type_read_fn(wire_type) use value: Parsed(Dynamic) <- result.try( read(bits, pos) |> result.map(parsed_map(_, dynamic.bit_array)), )
let field = parsed_map(value, fn(value) { Field(key: dynamic.int(field_id), value:) })
Ok(field)}
fn wire_type_read_fn(ty: WireType) -> ValueParser { case ty { wire_type.VarInt -> read_varint wire_type.I64 -> read_fixed(64) wire_type.Len -> read_len wire_type.I32 -> read_fixed(32) wire_type.SGroup | wire_type.EGroup -> read_group }}
/// Reads the bits from a varint and returns *all* of them. They cannot be/// parsed as a uint.////// Specifically, the continuation bits are included, so the value's bit size/// will be a multiple of 8.pub fn read_varint(bits: BitArray, pos: BytePos) -> ValueResult { read_varint_acc(bits, <<>>, pos)}
fn read_varint_acc(bits: BitArray, acc: BitArray, pos: BytePos) -> ValueResult { case bits { <<0:size(1), n:bits-size(7), rest:bytes>> -> { let bit = <<0:size(1), n:bits>> let acc = bit_array.concat([acc, bit]) Ok(Parsed(value: acc, rest:, pos: pos + 1)) } <<1:size(1), n:bits-size(7), rest:bytes>> -> { let bit = <<1:size(1), n:bits>> read_varint_acc(rest, bit_array.concat([acc, bit]), pos + 1) } bits -> Error(InvalidVarInt(leftover_bits: bits, acc:, pos:)) }}
/// Reads the bits from a varint and parses them a `BitArray`. The returned bits/// can be parsed as a uint.////// For a decoder that does this, use `decoders.uint()`.////// The continuation bits are not included, so the value's bit size will be a/// multiple of 7.pub fn parse_varint(bits: BitArray, pos: BytePos) -> ValueResult { parse_varint_acc(bits, <<>>, pos)}
fn parse_varint_acc( bits: BitArray, acc: BitArray, pos: BytePos,) -> ValueResult { case bits { <<0:size(1), n:bits-size(7), rest:bytes>> -> { let acc = bit_array.concat([n, acc]) Ok(Parsed(value: acc, rest:, pos: pos + 1)) } <<1:size(1), n:bits-size(7), rest:bytes>> -> parse_varint_acc(rest, bit_array.concat([n, acc]), pos + 1) bits -> { Error(InvalidVarInt(leftover_bits: bits, acc:, pos:)) } }}
/// Size is in bits, usually either 32 or 64.////// Size must be a multiple of 8, or the returned position will be incorrect.pub fn read_fixed(size: Int) -> ValueParser { fn(bits: BitArray, pos: BytePos) -> ValueResult { case bits { <<num:bits-size(size), rest:bytes>> -> Ok(Parsed(value: num, rest:, pos: pos + size / 8)) bits -> Error(InvalidFixed(size:, bits:, pos:)) } }}
fn read_len(bits: BitArray, len_pos: BytePos) -> ValueResult { // First, read the length of the value // It is encoded as a varint immediately after the tag use Parsed(value: len, rest: bits, pos:) <- result.try(parse_varint( bits, len_pos, )) let len = util.bit_array_to_uint(len)
use value <- result.try( bit_array.slice(from: bits, at: 0, take: len) |> result.map_error(fn(_) { InvalidLen(len:, value_bits: bits, pos: len_pos) }), ) // Assert b/c if the len was too long, it would've errored in the prev slice let assert Ok(rest) = bit_array.slice(from: bits, at: len, take: bit_array.byte_size(bits) - len)
Ok(Parsed(value:, rest:, pos: pos + len))}
/// Currently just does nothing.fn read_group(bits: BitArray, pos: BytePos) -> ValueResult { Ok(Parsed(value: <<>>, rest: bits, pos: pos))}
/// Decode a repeated field that may be either packed or expanded./// /// If it is impossible for a field to be packed (ie because it is encoded as/// a `LEN`) and therefore there is no `ValueParser` for it, then use the/// stdlib's `decode.list()` instead.pub fn decode_multiple( of decoder: Decoder(t), using parser: ValueParser,) -> Decoder(List(t)) { use values <- decode.then(packed_values(of: decoder, using: parser)) values |> decode.success}
fn packed_values( of decoder: Decoder(t), using parser: ValueParser,) -> Decoder(List(t)) { use bits <- decode.then(decode.list(of: decode.bit_array)) let bits = list.fold(bits, <<>>, fn(acc, elem) { bit_array.concat([acc, elem]) })
let values = { use values <- result.try(unpack_bits(bits, [], at: 0, using: parser)) values |> list.map(dynamic.bit_array) |> list.try_map(fn(value) { decode.run(value, decoder) }) |> result.map_error(UnableToDecode) }
case values { Ok(values) -> decode.success(values) Error(_) -> decode.failure([], "Packed values") }}
fn unpack_bits( bits: BitArray, acc: List(BitArray), at pos: BytePos, using parser: ValueParser,) -> DecodeResult(List(BitArray)) { case bits { <<>> -> acc |> list.reverse |> Ok bits -> { use Parsed(value:, rest:, pos:) <- result.try(parser(bits, pos)) unpack_bits(rest, [value, ..acc], at: pos, using: parser) } }}
/// Takes the last value from the list.fn single( of decoder: Decoder(t), named name: String, default default: t,) -> Decoder(t) { use values <- decode.then(decode.list(of: decoder))
case list.last(values) { Ok(value) -> decode.success(value) Error(Nil) -> decode.failure(default, name) }}
// Allows the decoders to be used for either single or repeated fieldsfn single_or_raw( of decoder: Decoder(t), named name: String, default default: t,) -> Decoder(t) { decode.one_of(decoder, or: [ single(of: decoder, named: name, default: default), ])}
fn single_or_raw_bit_array() -> Decoder(BitArray) { single_or_raw(of: decode.bit_array, named: "BitArray", default: <<>>)}
/// Decode a `bytes` field. The result is guaranteed to be byte-aligned.pub fn decode_bytes() -> Decoder(BitArray) { use bits <- decode.then(single_or_raw_bit_array()) case bits { <<_:bytes>> -> decode.success(bits) _ -> decode.failure(<<>>, "Bytes") }}
pub fn decode_protobuf( // Passed as a function so recursive decoders are easier using decoder: fn() -> Decoder(t), named name: String, default default: t,) -> Decoder(t) { use bits <- decode.then(single_or_raw_bit_array())
let value = parse(from: bits, using: decoder()) case value { Ok(Parsed(value:, rest: <<>>, pos: _)) -> decode.success(value) _ -> decode.failure(default, name) }}
pub fn decode_uint() -> Decoder(Int) { use bits <- decode.then(single_or_raw_bit_array()) use bits <- decode.then(case parse_varint(bits, 0) { Ok(Parsed(value:, rest: <<>>, pos: _)) -> decode.success(value) _ -> decode.failure(<<>>, "uint") }) bits |> util.bit_array_to_uint |> decode.success}
pub fn decode_sint() -> Decoder(Int) { use n <- decode.then(decode_uint()) case int.is_even(n) { True -> n / 2 False -> int.negate({ n + 1 } / 2) } |> decode.success}
pub fn decode_fixed(size: Int) -> Decoder(Int) { use bits <- decode.then(single_or_raw_bit_array()) case bits { <<num:unsigned-little-size(size)>> -> decode.success(num) _ -> decode.failure(0, "Fixed(" <> int.to_string(size) <> ")") }}
pub fn decode_bool() -> Decoder(Bool) { use bool <- decode.then(decode_uint()) case bool { 0 -> False |> decode.success 1 -> True |> decode.success _ -> decode.failure(False, "Bool") }}
pub fn decode_string() -> Decoder(String) { use bits <- decode.then(single_or_raw_bit_array())
let str = bit_array.to_string(bits) case str { Ok(str) -> decode.success(str) Error(_) -> decode.failure("", "String") }}