diff --git a/docs/tutorial/docfx.exe_user_manual.md b/docs/tutorial/docfx.exe_user_manual.md index c3330042abf..f29e1717df0 100644 --- a/docs/tutorial/docfx.exe_user_manual.md +++ b/docs/tutorial/docfx.exe_user_manual.md @@ -226,7 +226,6 @@ fileMetadataFiles | Specify a list of JSON file path containing fileMetad template | The templates applied to each file in the documentation. It can be a string or an array. The latter ones will override the former ones if the name of the file inside the template collides. If omitted, embedded `default` template will be used. theme | The themes applied to the documentation. Theme is used to customize the styles generated by `template`. It can be a string or an array. The latter ones will override the former ones if the name of the file inside the template collides. If omitted, no theme will be applied, the default theme inside the template will be used. xref | Specifies the urls of xrefmap used by content files. Currently, it supports following scheme: http, https, ftp, file, embedded. -xrefService | Specifies the url patterns of xref service. Please read **Section3.2.4** for detail. exportRawModel | If set to true, data model to run template script will be extracted in `.raw.json` extension. rawModelOutputFolder | Specify the output folder for the raw model. If not set, the raw model will be generated to the same folder as the output documentation. exportViewModel | If set to true, data model to apply template will be extracted in `.view.json` extension. @@ -425,38 +424,6 @@ Note that, metadata set in command line will merge with metadata set in `docfx.j Given multiple metadata files, the behavior would be **undetermined**, if same key is set in these files. -#### 3.2.4 Xref service url pattern - -Xref service url pattern will exact url by following steps: - -1. Replace environment variables. - - * Syntax: `{%variableName%}` - * Process: load variable value from enviroment variables, replace the source content. -2. Extract post pipelines. - - * Syntax: `|> pipelineName parameter1 parameter2 ...` - * Process: load pipeline, and extract parameters, remove from url. - * Current pipeline only contains: `removeHost`, `addQueryString`, pluggable pipeline will be add later. -3. Runtime replace variables. - - * Syntax: `{varName}` - * Process: replace the source content to the value of variable (uri data encoded). - -e.g.:\ -Environment variables: test = hello\ -Available Pipeline: removeHost\ -Variable: uid = testuid1\ -Url pattern: `http://{%test%}.contoso.com/?uid={uid}|> removeHost` - -It will run as following steps: - -1. Replace environment variables, `http://hello.contoso.com/?uid={uid}|> removeHost`. -2. Extract post pipelines, `http://hello.contoso.com/?uid={uid}`, and append a post pipeline `removeHost`. -3. Runtime replace variables, `http://hello.contoso.com/?uid=testuid`. -4. Send request to `http://hello.contoso.com/?uid=testuid`, and get response from site. -5. Run post pipeline `removeHost`. - #### 3.2.5 SitemapOptions The SitemapOptions is to configure the values for generating [sitemap.xml](https://www.sitemaps.org/protocol.html) file. diff --git a/docs/tutorial/links_and_cross_references.md b/docs/tutorial/links_and_cross_references.md index ebcb8212137..921f501c6af 100644 --- a/docs/tutorial/links_and_cross_references.md +++ b/docs/tutorial/links_and_cross_references.md @@ -290,21 +290,6 @@ The value of `xref` could be a string or a list of strings that contain the path > the website together with topic files so that others can directly use its url in `docfx.json` instead of downloading it to > local. -### Cross reference services - -Cross reference services are hosted services that can be queried for cross reference information. When DocFX generates the metadata for your project, it will perform cross reference lookups against the service. - -To use a cross reference service, add a `xrefservice` config to the `build` section of `docfx.json`: - -```json -{ - "build": { - "xrefService": [ "" ], - ... - } -} -``` - ## Advanced: more options for cross reference You can create a cross link with following options: diff --git a/src/Microsoft.DocAsCode.App/Config/BuildJsonConfig.cs b/src/Microsoft.DocAsCode.App/Config/BuildJsonConfig.cs index 2b6b113bb8d..09d000054f1 100644 --- a/src/Microsoft.DocAsCode.App/Config/BuildJsonConfig.cs +++ b/src/Microsoft.DocAsCode.App/Config/BuildJsonConfig.cs @@ -36,9 +36,6 @@ internal class BuildJsonConfig [JsonProperty("xref")] public ListWithStringFallback XRefMaps { get; set; } - [JsonProperty("xrefService")] - public ListWithStringFallback XRefServiceUrls { get; set; } - [JsonProperty("dest")] public string Destination { get; set; } diff --git a/src/Microsoft.DocAsCode.App/Helpers/DocumentBuilderWrapper.cs b/src/Microsoft.DocAsCode.App/Helpers/DocumentBuilderWrapper.cs index 3c9dedb3350..36214c7a1fe 100644 --- a/src/Microsoft.DocAsCode.App/Helpers/DocumentBuilderWrapper.cs +++ b/src/Microsoft.DocAsCode.App/Helpers/DocumentBuilderWrapper.cs @@ -148,10 +148,6 @@ private static List ConfigToParameter(BuildJsonConfig c { parameters.XRefMaps = config.XRefMaps.ToImmutableArray(); } - if (config.XRefServiceUrls != null) - { - parameters.XRefServiceUrls = config.XRefServiceUrls.ToImmutableArray(); - } string outputFolderForDebugFiles = null; if (!string.IsNullOrEmpty(config.OutputFolderForDebugFiles)) diff --git a/src/Microsoft.DocAsCode.Build.Engine/DocumentBuildContext.cs b/src/Microsoft.DocAsCode.Build.Engine/DocumentBuildContext.cs index dc1d18ed8c9..d9a271b29c0 100644 --- a/src/Microsoft.DocAsCode.Build.Engine/DocumentBuildContext.cs +++ b/src/Microsoft.DocAsCode.Build.Engine/DocumentBuildContext.cs @@ -18,16 +18,15 @@ public sealed class DocumentBuildContext : IDocumentBuildContext private readonly ConcurrentDictionary _tableOfContents = new(FilePathComparer.OSPlatformSensitiveStringComparer); private readonly Task _reader; private ImmutableArray _xrefMapUrls { get; } - private ImmutableArray _xrefServiceUrls { get; } public DocumentBuildContext(string buildOutputFolder) : this(buildOutputFolder, Enumerable.Empty(), ImmutableArray.Empty, ImmutableArray.Empty, 1, Directory.GetCurrentDirectory(), string.Empty, null, null) { } public DocumentBuildContext(string buildOutputFolder, IEnumerable allSourceFiles, ImmutableArray externalReferencePackages, ImmutableArray xrefMaps, int maxParallelism, string baseFolder, string versionName, ApplyTemplateSettings applyTemplateSetting, string rootTocPath) - : this(buildOutputFolder, allSourceFiles, externalReferencePackages, xrefMaps, maxParallelism, baseFolder, versionName, applyTemplateSetting, rootTocPath, null, ImmutableArray.Empty) { } + : this(buildOutputFolder, allSourceFiles, externalReferencePackages, xrefMaps, maxParallelism, baseFolder, versionName, applyTemplateSetting, rootTocPath, null) { } - public DocumentBuildContext(string buildOutputFolder, IEnumerable allSourceFiles, ImmutableArray externalReferencePackages, ImmutableArray xrefMaps, int maxParallelism, string baseFolder, string versionName, ApplyTemplateSettings applyTemplateSetting, string rootTocPath, string versionFolder, ImmutableArray xrefServiceUrls) - : this(buildOutputFolder, allSourceFiles, externalReferencePackages, xrefMaps, maxParallelism, baseFolder, versionName, applyTemplateSetting, rootTocPath, null, ImmutableArray.Empty, null, null) { } + public DocumentBuildContext(string buildOutputFolder, IEnumerable allSourceFiles, ImmutableArray externalReferencePackages, ImmutableArray xrefMaps, int maxParallelism, string baseFolder, string versionName, ApplyTemplateSettings applyTemplateSetting, string rootTocPath, string versionFolder) + : this(buildOutputFolder, allSourceFiles, externalReferencePackages, xrefMaps, maxParallelism, baseFolder, versionName, applyTemplateSetting, rootTocPath, null, null, null) { } public DocumentBuildContext(DocumentBuildParameters parameters) { @@ -37,7 +36,6 @@ public DocumentBuildContext(DocumentBuildParameters parameters) HrefGenerator = parameters.ApplyTemplateSettings?.HrefGenerator; AllSourceFiles = GetAllSourceFiles(parameters.Files.EnumerateFiles()); _xrefMapUrls = parameters.XRefMaps; - _xrefServiceUrls = parameters.XRefServiceUrls; GroupInfo = parameters.GroupInfo; XRefTags = parameters.XRefTags; MaxParallelism = parameters.MaxParallelism; @@ -78,7 +76,6 @@ public DocumentBuildContext( ApplyTemplateSettings applyTemplateSetting, string rootTocPath, string versionFolder, - ImmutableArray xrefServiceUrls, GroupInfo groupInfo, List xrefTags) { @@ -89,7 +86,6 @@ public DocumentBuildContext( AllSourceFiles = GetAllSourceFiles(allSourceFiles); ExternalReferencePackages = externalReferencePackages; _xrefMapUrls = xrefMaps; - _xrefServiceUrls = xrefServiceUrls; GroupInfo = groupInfo; XRefTags = xrefTags; MaxParallelism = maxParallelism; @@ -204,7 +200,7 @@ private void ResolveExternalXRefSpecForSpecs() } } - public async Task ResolveExternalXRefSpecForNoneSpecsAsync() + public void ResolveExternalXRefSpecForNoneSpecsAsync() { // remove internal xref. var uidList = @@ -226,10 +222,6 @@ from spec in ExternalXRefSpec.Values { uidList = ResolveByExternalReferencePackages(uidList, ExternalXRefSpec); } - if (uidList.Count > 0) - { - uidList = await ResolveByXRefServiceAsync(uidList, ExternalXRefSpec); - } Logger.LogVerbose($"{uidList.Count} uids are unresolved."); @@ -268,56 +260,6 @@ private List ResolveByExternalReferencePackages(List uidList, Co return list; } - private async Task> ResolveByXRefServiceAsync(List uidList, ConcurrentDictionary externalXRefSpec) - { - if (_xrefServiceUrls == null || _xrefServiceUrls.Length == 0) - { - return uidList; - } - - var unresolvedUidList = await new XrefServiceResolver(_xrefServiceUrls, MaxHttpParallelism).ResolveAsync(uidList, externalXRefSpec); - Logger.LogInfo($"{uidList.Count - unresolvedUidList.Count} uids found in {_xrefServiceUrls.Length} xrefservice(s)."); - return unresolvedUidList; - } - - internal async Task> QueryByHttpRequestAsync(HttpClient client, string requestUrl, string uid) - { - string url = requestUrl.Replace("{uid}", Uri.EscapeDataString(uid)); - try - { - var data = await client.GetStreamAsync(url); - using var sr = new StreamReader(data); - var xsList = JsonUtility.Deserialize>>(sr); - return xsList.ConvertAll(item => - { - var spec = new XRefSpec(); - foreach (var pair in item) - { - if (pair.Value is string s) - { - spec[pair.Key] = s; - } - } - return spec; - }); - } - catch (HttpRequestException e) - { - Logger.LogWarning($"Error occurs when resolve {uid} from {requestUrl}.{e.InnerException.Message}"); - return null; - } - catch (Newtonsoft.Json.JsonReaderException e) - { - Logger.LogWarning($"Response from {requestUrl} is not in valid JSON format.{e.Message}"); - return null; - } - catch (Exception e) - { - Logger.LogWarning($"Error occurs when resolve {uid} from {requestUrl}.{e.Message}"); - return null; - } - } - private List ResolveByXRefMaps(List uidList, ConcurrentDictionary externalXRefSpec) { if (_reader == null) @@ -561,12 +503,6 @@ public XRefSpec GetXrefSpec(string uid) } } - var uidList = ResolveByXRefServiceAsync(new List { uid }, ExternalXRefSpec).Result; - if (uidList.Count == 0) - { - return ExternalXRefSpec[uid]; - } - UnknownUids.TryAdd(uid, null); return null; } diff --git a/src/Microsoft.DocAsCode.Build.Engine/DocumentBuildParameters.cs b/src/Microsoft.DocAsCode.Build.Engine/DocumentBuildParameters.cs index f0bd06d9196..84bfe1fe1e9 100644 --- a/src/Microsoft.DocAsCode.Build.Engine/DocumentBuildParameters.cs +++ b/src/Microsoft.DocAsCode.Build.Engine/DocumentBuildParameters.cs @@ -22,8 +22,6 @@ public sealed class DocumentBuildParameters : IBuildParameters public ImmutableArray XRefMaps { get; set; } = ImmutableArray.Empty; - public ImmutableArray XRefServiceUrls { get; set; } = ImmutableArray.Empty; - public ImmutableDictionary Metadata { get; set; } = ImmutableDictionary.Empty; public FileMetadata FileMetadata { get; set; } diff --git a/src/Microsoft.DocAsCode.Build.Engine/XrefServiceResolver.cs b/src/Microsoft.DocAsCode.Build.Engine/XrefServiceResolver.cs deleted file mode 100644 index c9b8100ee6e..00000000000 --- a/src/Microsoft.DocAsCode.Build.Engine/XrefServiceResolver.cs +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System.Collections.Concurrent; -using System.Collections.Immutable; -using System.Text; -using System.Web; - -using Microsoft.DocAsCode.Common; -using Microsoft.DocAsCode.Plugins; - -namespace Microsoft.DocAsCode.Build.Engine; - -public class XrefServiceResolver -{ - private readonly List>>> _uriTemplates; - - public XrefServiceResolver(ImmutableArray xrefServiceUrls, int maxParallelism) - : this(null, xrefServiceUrls, maxParallelism) { } - - public XrefServiceResolver(HttpClient client, ImmutableArray xrefServiceUrls, int maxParallelism) - { - _uriTemplates = - (from url in xrefServiceUrls - select UriTemplate.Create( - url, - new XrefClient(client, maxParallelism).ResolveAsync, - GetPipeline)).ToList(); - } - - public async Task> ResolveAsync(List uidList, ConcurrentDictionary externalXRefSpec) - { - if (_uriTemplates.Count == 0) - { - return uidList; - } - - var unresolvedUidList = new List(); - - var xrefObjects = await Task.WhenAll( - from uid in uidList - select ResolveAsync(uid)); - foreach (var tuple in uidList.Zip(xrefObjects, Tuple.Create)) - { - if (tuple.Item2 == null) - { - unresolvedUidList.Add(tuple.Item1); - } - else - { - externalXRefSpec.AddOrUpdate(tuple.Item1, tuple.Item2, (s, x) => x + tuple.Item2); - } - } - if (unresolvedUidList.Count > 0 && Logger.LogLevelThreshold <= LogLevel.Verbose) - { - var capacity = 256 + 64 * (Math.Min(100, unresolvedUidList.Count)) + 64 * _uriTemplates.Count; - var sb = new StringBuilder(capacity); - sb.Append("Cannot resolve "); - sb.Append(unresolvedUidList.Count); - sb.Append(" uids by xref service, top 100:"); - foreach (var uid in unresolvedUidList.Take(100)) - { - sb.AppendLine().Append(" ").Append(uid); - } - sb.AppendLine().Append(" ").Append("xref service:"); - foreach (var t in _uriTemplates) - { - sb.AppendLine().Append(" ").Append(t.Template); - } - Logger.LogVerbose(sb.ToString()); - } - return unresolvedUidList; - } - - public async Task ResolveAsync(string uid) - { - var d = new Dictionary { ["uid"] = uid }; - foreach (var t in _uriTemplates) - { - List value = null; - try - { - value = await t.Evaluate(d); - } - catch (Exception ex) - { - Logger.LogInfo($"Unable to resolve uid ({uid}) from {t.Template}, details: {ex.Message}"); - } - if (value?.Count > 0) - { - return value[0]; - } - } - return null; - } - - private IUriTemplatePipeline>> GetPipeline(string name) - { - // todo: pluggable. - switch (name) - { - case "removeHost": - return RemoveHostUriTemplatePipeline.Default; - case "addQueryString": - return AddQueryStringUriTemplatePipeline.Default; - default: - Logger.LogWarning($"Unknown uri template pipeline: {name}.", code: WarningCodes.Build.UnknownUriTemplatePipeline); - return EmptyUriTemplatePipeline.Default; - } - } - - private sealed class RemoveHostUriTemplatePipeline : IUriTemplatePipeline>> - { - public static readonly RemoveHostUriTemplatePipeline Default = new(); - - public async Task> Handle(Task> value, string[] parameters) - { - var list = await value; - foreach (var item in list) - { - if (string.IsNullOrEmpty(item.Href)) - { - continue; - } - if (Uri.TryCreate(item.Href, UriKind.Absolute, out var uri)) - { - if (parameters.Length == 0 || - Array.IndexOf(parameters, uri.Host) != -1) - { - item.Href = item.Href.Substring(uri.GetLeftPart(UriPartial.Authority).Length); - } - } - } - return list; - } - } - - private sealed class AddQueryStringUriTemplatePipeline : IUriTemplatePipeline>> - { - public static readonly AddQueryStringUriTemplatePipeline Default = new(); - - public Task> Handle(Task> value, string[] parameters) - { - if (parameters.Length == 2 && - !string.IsNullOrEmpty(parameters[0]) && - !string.IsNullOrEmpty(parameters[1])) - { - return HandleCoreAsync(value, parameters[0], parameters[1]); - } - return value; - } - - private async Task> HandleCoreAsync(Task> task, string name, string value) - { - var list = await task; - foreach (var item in list) - { - if (string.IsNullOrEmpty(item.Href)) - { - continue; - } - var mvc = HttpUtility.ParseQueryString(UriUtility.GetQueryString(item.Href)); - mvc[name] = value; - item.Href = UriUtility.GetPath(item.Href) + - "?" + mvc.ToString() + - UriUtility.GetFragment(item.Href); - } - return list; - } - } - - private sealed class EmptyUriTemplatePipeline : IUriTemplatePipeline>> - { - public static readonly EmptyUriTemplatePipeline Default = new(); - - public Task> Handle(Task> value, string[] parameters) - { - return value; - } - } -} diff --git a/src/docfx/Models/BuildCommandOptions.cs b/src/docfx/Models/BuildCommandOptions.cs index 745fa1e4a4e..c050208a904 100644 --- a/src/docfx/Models/BuildCommandOptions.cs +++ b/src/docfx/Models/BuildCommandOptions.cs @@ -29,9 +29,6 @@ internal class BuildCommandOptions : LogOptions, ICanPrintHelpMessage [OptionList('x', "xref", Separator = ',', HelpText = "Specify the urls of xrefmap used by content files.")] public List XRefMaps { get; set; } - [OptionList("xrefService", Separator = ',', HelpText = "Specify the urls of xrefService for resolving xref used by content files.")] - public List XRefService { get; set; } - [OptionList('t', "template", Separator = ',', HelpText = "Specify the template name to apply to. If not specified, output YAML file will not be transformed.")] public List Templates { get; set; } diff --git a/src/docfx/SubCommands/BuildCommand.cs b/src/docfx/SubCommands/BuildCommand.cs index 3416654d3d7..1022e8e4657 100644 --- a/src/docfx/SubCommands/BuildCommand.cs +++ b/src/docfx/SubCommands/BuildCommand.cs @@ -153,18 +153,6 @@ internal static void MergeOptionsToConfig(BuildCommandOptions options, BuildJson .Distinct()); } - if (options.XRefService != null) - { - config.XRefServiceUrls = - new ListWithStringFallback( - (config.XRefServiceUrls ?? new ListWithStringFallback()) - .Concat(options.XRefService) - .Where(x => !string.IsNullOrWhiteSpace(x)) - .Distinct()); - } - - //to-do: get changelist from options - if (options.Serve) { config.Serve = options.Serve; diff --git a/test/Microsoft.DocAsCode.Build.Engine.Tests/DocumentBuilderTest.cs b/test/Microsoft.DocAsCode.Build.Engine.Tests/DocumentBuilderTest.cs index 52b2806fbe5..23787f9c2c2 100644 --- a/test/Microsoft.DocAsCode.Build.Engine.Tests/DocumentBuilderTest.cs +++ b/test/Microsoft.DocAsCode.Build.Engine.Tests/DocumentBuilderTest.cs @@ -879,63 +879,6 @@ protected override Task SendAsync(HttpRequestMessage reques } } - [Fact] - public async Task TestBuildWithXrefService() - { - var fakeResponseHandler = new FakeResponseHandler(); - fakeResponseHandler.AddFakeResponse(new Uri("http://example.org/test1"), new HttpResponseMessage - { - StatusCode = HttpStatusCode.OK, - Content = new StringContent("[]") - }); - fakeResponseHandler.AddFakeResponse(new Uri("http://example.org/test2"), new HttpResponseMessage - { - StatusCode = HttpStatusCode.OK, - Content = new StringContent("[{'uid':'csharp_coding_standards', 'name':'C# Coding Standards', 'href':'http://dotnet.github.io/docfx/guideline/csharp_coding_standards.html'}]") - }); - - var httpClient = new HttpClient(fakeResponseHandler); // lgtm[cs/httpclient-checkcertrevlist-disabled] - var result = await new XrefServiceResolver(httpClient, ImmutableArray.Create("http://example.org/test1"), 1).ResolveAsync("xx"); - Assert.Null(result); - result = await new XrefServiceResolver(httpClient, ImmutableArray.Create("http://example.org/test2|> removeHost |> addQueryString x y"), 1).ResolveAsync("xx"); - Assert.Equal("csharp_coding_standards", result.Uid); - Assert.Equal("/docfx/guideline/csharp_coding_standards.html?x=y", result.Href); - } - - [Fact] - public async Task TestBuildWithXrefServiceRemoveHostWithParameters() - { - var fakeResponseHandler = new FakeResponseHandler(); - var httpClient = new HttpClient(fakeResponseHandler); // lgtm[cs/httpclient-checkcertrevlist-disabled] - - fakeResponseHandler.AddFakeResponse(new Uri("http://example.org/test1"), new HttpResponseMessage - { - StatusCode = HttpStatusCode.OK, - Content = new StringContent("[{'uid':'csharp_coding_standards', 'name':'C# Coding Standards', 'href':'http://dotnet.github.io/docfx/guideline/csharp_coding_standards.html'}]") - }); - var result = await new XrefServiceResolver(httpClient, ImmutableArray.Create("http://example.org/test1|> removeHost www.microsoft.com"), 1).ResolveAsync("xx"); - Assert.Equal("csharp_coding_standards", result.Uid); - Assert.Equal("http://dotnet.github.io/docfx/guideline/csharp_coding_standards.html", result.Href); - - fakeResponseHandler.AddFakeResponse(new Uri("http://example.org/test2"), new HttpResponseMessage - { - StatusCode = HttpStatusCode.OK, - Content = new StringContent("[{'uid':'csharp_coding_standards', 'name':'C# Coding Standards', 'href':'http://www.microsoft.com/docfx/guideline/csharp_coding_standards.html'}]") - }); - result = await new XrefServiceResolver(httpClient, ImmutableArray.Create("http://example.org/test2|> removeHost www.microsoft.com"), 1).ResolveAsync("xx"); - Assert.Equal("csharp_coding_standards", result.Uid); - Assert.Equal("/docfx/guideline/csharp_coding_standards.html", result.Href); - - fakeResponseHandler.AddFakeResponse(new Uri("http://example.org/test3"), new HttpResponseMessage - { - StatusCode = HttpStatusCode.OK, - Content = new StringContent("[{'uid':'csharp_coding_standards', 'name':'C# Coding Standards', 'href':'http://dotnet.github.io/docfx/guideline/csharp_coding_standards.html'}]") - }); - result = await new XrefServiceResolver(httpClient, ImmutableArray.Create("http://example.org/test3|> removeHost www.microsoft.com dotnet.github.io"), 1).ResolveAsync("xx"); - Assert.Equal("csharp_coding_standards", result.Uid); - Assert.Equal("/docfx/guideline/csharp_coding_standards.html", result.Href); - } - [Fact] public void TestBuildWithMultipleVersion() {