fix: Chinese character support on web loader (#600)

davidmigloz · Nov 29, 2024 · 48e64d5 · 48e64d5
1 parent 44291a0
commit 48e64d5
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/packages/langchain_community/lib/src/document_loaders/web.dart b/packages/langchain_community/lib/src/document_loaders/web.dart
@@ -3,6 +3,7 @@ import 'package:http/http.dart' as http;
 
 import 'package:langchain_core/document_loaders.dart';
 import 'package:langchain_core/documents.dart';
+import 'dart:convert';
 
 /// {@template web_base_loader}
 /// A document loader that loads [Document]s from web pages.
@@ -57,7 +58,7 @@ class WebBaseLoader extends BaseDocumentLoader {
 
   Future<String> _fetchUrl(final String url) async {
     final response = await http.get(Uri.parse(url), headers: requestHeaders);
-    return response.body;
+    return utf8.decode(response.bodyBytes);
   }
 
   Map<String, dynamic> _buildMetadata(