From fa9a3b35d73bf586a2feee651982dc04818e7e88 Mon Sep 17 00:00:00 2001 From: joemphilips Date: Mon, 15 Aug 2022 23:59:06 +0900 Subject: [PATCH 1/2] cln: add JsonConverter for handling enum Unlike Newtonsoft.Json, System.Text.Json does not respect EnumMemberAttribute. (see: https://github.com/dotnet/runtime/issues/31081) This caused an issue when sending request to c-lightning, when you use System.Text.Json for serialization lib, and there is a enum value in the request field, the field will be serialized into an int, instead of a string. Thus causing RPC error. This commit fixes it by using `JsonStringEnumConverterEx` (which is taken from an issue comment of issue above). --- .../DotNetLightning.ClnRpc.fsproj | 6 +-- src/DotNetLightning.ClnRpc/Requests.fs | 22 ++++++++ .../SystemTextJsonConverterExtensions.fs | 17 ++++++ .../SystemTextJsonConverters.fs | 52 +++++++++++++++++-- .../SerializerTests.fs | 26 ++++++++++ tools/fsharp_msggen/fsharp_msggen/fsharp.py | 22 ++++++-- 6 files changed, 135 insertions(+), 10 deletions(-) create mode 100644 src/DotNetLightning.ClnRpc/SystemTextJsonConverterExtensions.fs diff --git a/src/DotNetLightning.ClnRpc/DotNetLightning.ClnRpc.fsproj b/src/DotNetLightning.ClnRpc/DotNetLightning.ClnRpc.fsproj index f6c81e233..d77f7dc42 100644 --- a/src/DotNetLightning.ClnRpc/DotNetLightning.ClnRpc.fsproj +++ b/src/DotNetLightning.ClnRpc/DotNetLightning.ClnRpc.fsproj @@ -17,6 +17,7 @@ + @@ -30,14 +31,13 @@ - + - + diff --git a/src/DotNetLightning.ClnRpc/Requests.fs b/src/DotNetLightning.ClnRpc/Requests.fs index 6f831c914..8b071b2e9 100644 --- a/src/DotNetLightning.ClnRpc/Requests.fs +++ b/src/DotNetLightning.ClnRpc/Requests.fs @@ -3105,3 +3105,25 @@ type private Response = | Ping of Responses.PingResponse | SignMessage of Responses.SignmessageResponse + +open DotNetLightning.ClnRpc.SystemTextJsonConverters +module internal AddJsonConverters = + let addEnumConverters(opts: JsonSerializerOptions) = + opts.Converters.Add(JsonStringEnumConverterEx()) + opts.Converters.Add(JsonStringEnumConverterEx()) + opts.Converters.Add(JsonStringEnumConverterEx()) + opts.Converters.Add(JsonStringEnumConverterEx()) + opts.Converters.Add(JsonStringEnumConverterEx()) + opts.Converters.Add(JsonStringEnumConverterEx()) + opts.Converters.Add(JsonStringEnumConverterEx()) + opts.Converters.Add(JsonStringEnumConverterEx()) + opts.Converters.Add(JsonStringEnumConverterEx()) + opts.Converters.Add(JsonStringEnumConverterEx()) + opts.Converters.Add(JsonStringEnumConverterEx()) + opts.Converters.Add(JsonStringEnumConverterEx()) + opts.Converters.Add(JsonStringEnumConverterEx()) + opts.Converters.Add(JsonStringEnumConverterEx()) + opts.Converters.Add(JsonStringEnumConverterEx()) + opts.Converters.Add(JsonStringEnumConverterEx()) + opts.Converters.Add(JsonStringEnumConverterEx()) + opts.Converters.Add(JsonStringEnumConverterEx()) diff --git a/src/DotNetLightning.ClnRpc/SystemTextJsonConverterExtensions.fs b/src/DotNetLightning.ClnRpc/SystemTextJsonConverterExtensions.fs new file mode 100644 index 000000000..6b2308782 --- /dev/null +++ b/src/DotNetLightning.ClnRpc/SystemTextJsonConverterExtensions.fs @@ -0,0 +1,17 @@ +namespace DotNetLightning.ClnRpc.SystemTextJsonConverters + +open System.Text.Json + +open System.Runtime.CompilerServices +open NBitcoin + +[] +type ClnSharpClientHelpers = + [] + static member internal AddDNLJsonConverters + ( + this: JsonSerializerOptions, + n: Network + ) = + this._AddDNLJsonConverters(n) + DotNetLightning.ClnRpc.AddJsonConverters.addEnumConverters(this) diff --git a/src/DotNetLightning.ClnRpc/SystemTextJsonConverters.fs b/src/DotNetLightning.ClnRpc/SystemTextJsonConverters.fs index 49fe1fd2d..d8dcf8fac 100644 --- a/src/DotNetLightning.ClnRpc/SystemTextJsonConverters.fs +++ b/src/DotNetLightning.ClnRpc/SystemTextJsonConverters.fs @@ -107,8 +107,8 @@ type OutPointJsonConverter() = raise <| JsonException("not a valid txid:output tuple") else let o = OutPoint() - o.Hash <- splits.[0] |> uint256.Parse - o.N <- splits.[1] |> uint32 + o.Hash <- splits[0] |> uint256.Parse + o.N <- splits[1] |> uint32 o type FeerateJsonConverter() = @@ -154,10 +154,54 @@ type OutputDescriptorJsonConverter(network: Network) = reader.GetString() |> fun s -> OutputDescriptor.Parse(s, network) +open System.Collections.Generic +open System.Runtime.Serialization +open System.Linq + +/// Taken from https://github.com/dotnet/runtime/issues/31081#issuecomment-848697673 +type JsonStringEnumConverterEx<'TEnum when 'TEnum: enum and 'TEnum: equality and 'TEnum: (new: + unit -> 'TEnum) and 'TEnum: struct and 'TEnum :> Enum>() = + inherit JsonConverter<'TEnum>() + + let _enumToString = Dictionary<'TEnum, string>() + let _stringToEnum = Dictionary() + + do + let ty = typeof<'TEnum> + + for v in Enum.GetValues<'TEnum>() do + let enumMember = ty.GetMember(v.ToString())[0] + + let maybeAttr = + enumMember + .GetCustomAttributes(typeof, false) + .Cast() + .FirstOrDefault() + |> Option.ofObj + + _stringToEnum.Add(v.ToString(), v) + + match maybeAttr with + | Some attr -> + _enumToString.Add(v, attr.Value) + _stringToEnum.Add(attr.Value, v) + | None -> _enumToString.Add(v, v.ToString()) + + override this.Read(reader, _typeToConvert, _options) = + let stringV = reader.GetString() + + match _stringToEnum.TryGetValue stringV with + | true, v -> v + | _ -> Unchecked.defaultof<'TEnum> + + override this.Write(writer, value, _options) = + writer.WriteStringValue(_enumToString[value]) + + [] -type ClnSharpClientHelpers = +type internal ClnSharpClientHelpersCore = [] - static member AddDNLJsonConverters + static member internal _AddDNLJsonConverters ( this: JsonSerializerOptions, n: Network diff --git a/tests/DotNetLightning.ClnRpc.Tests/SerializerTests.fs b/tests/DotNetLightning.ClnRpc.Tests/SerializerTests.fs index c15c0d627..770c1ec46 100644 --- a/tests/DotNetLightning.ClnRpc.Tests/SerializerTests.fs +++ b/tests/DotNetLightning.ClnRpc.Tests/SerializerTests.fs @@ -10,6 +10,7 @@ open DotNetLightning.ClnRpc open DotNetLightning.ClnRpc.Plugin open DotNetLightning.ClnRpc.NewtonsoftJsonConverters open DotNetLightning.ClnRpc.Requests +open DotNetLightning.ClnRpc.Responses open DotNetLightning.ClnRpc.SystemTextJsonConverters open DotNetLightning.Serialization open NBitcoin @@ -213,3 +214,28 @@ type SerializerTests() = Assert.Null(jObj.Root.["exclude"]) Assert.Null(jObj.Root.["cltv"]) () + + [] + member this.SerializeListPays() = + let req = + { + ListpaysRequest.Bolt11 = + "lnbcrt500u1p305fnmpp5vzsjps8uptzedfmrw8jsuw37m4mdlyjjua0qfzceph3a0nz7rtfqdql2djkuepqw3hjqsj5gvsxzerywfjhxuccqzptxqrrsssp5fak5cm2c3r5wtezcflfg6cs3psrp4kczvp4wly66h85y4m4hsrds9qyyssqqxemaw5w9r6hteaxmmhvqe4nkv654nyk88gahjt5mxfjjzkj945xe6frwuavv8u0fzwcst0mvrxj8nxlj3qad9dxgzv8rg9dup3r5kcqnwpqjk" + |> Some + PaymentHash = None + Status = ListpaysStatus.PENDING |> Some + } + + let opts = JsonSerializerOptions() + + let data1 = + opts.AddDNLJsonConverters(Network.RegTest) + JsonSerializer.SerializeToDocument(req, opts) + + Assert.Equal( + "pending", + data1 + .RootElement + .GetProperty("status") + .GetString() + ) diff --git a/tools/fsharp_msggen/fsharp_msggen/fsharp.py b/tools/fsharp_msggen/fsharp_msggen/fsharp.py index 9a597cc62..884e6e782 100644 --- a/tools/fsharp_msggen/fsharp_msggen/fsharp.py +++ b/tools/fsharp_msggen/fsharp_msggen/fsharp.py @@ -276,12 +276,30 @@ def write_header(self): """ self.write(opens) + def _gen_stj_converter(self, prefix: str, field: EnumField): + self.write(f"opts.Converters.Add(JsonStringEnumConverterEx<{prefix}.{field.typename}>())\n", numindent=2) + + def generate_stj_method(self, service: Service): + template = """ +open DotNetLightning.ClnRpc.SystemTextJsonConverters +module internal AddJsonConverters = + let addEnumConverters(opts: JsonSerializerOptions) = +""" + self.write(template) + for method in service.methods: + for field in method.request.fields: + if isinstance(field, EnumField): + self._gen_stj_converter("Requests", field) + for field in method.response.fields: + if isinstance(field, EnumField): + self._gen_stj_converter("Responses", field) + def generate(self, service: Service): self.write_header() self.generate_requests(service) self.generate_responses(service) self.generate_enums(service) - + self.generate_stj_method(service) class FSharpClientExtensionGenerator: def __init__(self, dest: TextIO): @@ -340,8 +358,6 @@ def write_header(self): """ self.write(opens) - def generate(self, service: Service): self.write_header() self.generate_methods(service) - From 69462d639c0939ba86c4ad07f28a2d1d6015a3d7 Mon Sep 17 00:00:00 2001 From: joemphilips Date: Tue, 16 Aug 2022 21:24:38 +0900 Subject: [PATCH 2/2] Use old version of .NET SDK for fsdocs. It seems that fsdocs has a bug in the latest .NET SDK (6.0.400). It fails to build a API doc for a package which uses 3rd party lib. Avoid this by specifying an older version explicitly for CI build. --- .github/workflows/build+test.yml | 2 +- .github/workflows/gh-pages.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build+test.yml b/.github/workflows/build+test.yml index 5a2baa0d6..4946e5968 100644 --- a/.github/workflows/build+test.yml +++ b/.github/workflows/build+test.yml @@ -62,7 +62,7 @@ jobs: - name: Setup .NET 6 uses: actions/setup-dotnet@v1 with: - dotnet-version: 6.0.x + dotnet-version: 6.0.303 - name: build project assemblies so that fsdocs can read assemblies and .xml files from bin/ . run: diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 2c2211e30..cd103b962 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -15,7 +15,7 @@ jobs: - name: Setup .NET 6 uses: actions/setup-dotnet@v1 with: - dotnet-version: 6.0.x + dotnet-version: 6.0.303 - name: build project assemblies so that fsdocs can read assemblies and .xml files from bin/ . run: