Merge pull request #1352 from samchon/feat/llm

Enhance test programs for the LLM schemas.

.github/workflows/build.yml

@@ -4,13 +4,15 @@ on:
     paths:
       - 'src/**'
       - 'test/**'
+      - 'test-error/**'
       - 'test-esm/**'
       - 'package.json'
       - '.github/workflows/build.yml'
   pull_request:
     paths:
       - 'src/**'
       - 'test/**'
+      - 'test-error/**'
       - 'test-esm/**'
       - 'package.json'
       - '.github/workflows/build.yml'

benchmark/package.json

@@ -72,6 +72,6 @@
     "suppress-warnings": "^1.0.2",
     "tstl": "^3.0.0",
     "uuid": "^9.0.1",
-    "typia": "../typia-7.0.0-dev.20241110.tgz"
+    "typia": "../typia-7.0.0-dev.20241111-2.tgz"
   }
 }

deploy/index.ts

@@ -39,7 +39,7 @@ const main = async (): Promise<void> => {
         commands: ["npm run build", "npm start"],
       },
       {
-        name: "errors",
+        name: "test-error",
         commands: ["npm start"],
       },
       {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "typia",
-  "version": "7.0.0-dev.20241110",
+  "version": "7.0.0-dev.20241111-2",
   "description": "Superfast runtime validators with only one line",
   "main": "lib/index.js",
   "typings": "lib/index.d.ts",
@@ -68,7 +68,7 @@
   },
   "homepage": "https://typia.io",
   "dependencies": {
-    "@samchon/openapi": "^2.0.0-dev.20241110-2",
+    "@samchon/openapi": "2.0.0-dev.20241111-3",
     "commander": "^10.0.0",
     "comment-json": "^4.2.3",
     "inquirer": "^8.2.5",

packages/typescript-json/package.json

@@ -1,6 +1,6 @@
 {
   "name": "typescript-json",
-  "version": "7.0.0-dev.20241110",
+  "version": "7.0.0-dev.20241111-2",
   "description": "Superfast runtime validators with only one line",
   "main": "lib/index.js",
   "typings": "lib/index.d.ts",
@@ -64,7 +64,7 @@
   },
   "homepage": "https://typia.io",
   "dependencies": {
-    "typia": "7.0.0-dev.20241110"
+    "typia": "7.0.0-dev.20241111-2"
   },
   "peerDependencies": {
     "typescript": ">=4.8.0 <5.7.0"

src/factories/MetadataCollection.ts

@@ -123,9 +123,9 @@ export class MetadataCollection {
     const oldbie = this.objects_.get(type);
     if (oldbie !== undefined) return [oldbie, false];
 
-    const $id: string = this.getName(checker, type);
+    const id: string = this.getName(checker, type);
     const obj: MetadataObjectType = MetadataObjectType.create({
-      name: $id,
+      name: id,
       properties: [],
       description:
         (type.aliasSymbol && CommentFactory.description(type.aliasSymbol)) ??
@@ -150,9 +150,9 @@ export class MetadataCollection {
     const oldbie = this.aliases_.get(type);
     if (oldbie !== undefined) return [oldbie, false, () => {}];
 
-    const $id: string = this.getName(checker, type);
+    const id: string = this.getName(checker, type);
     const alias: MetadataAliasType = MetadataAliasType.create({
-      name: $id,
+      name: id,
       value: null!,
       description: CommentFactory.description(symbol) ?? null,
       recursive: null!,
@@ -170,9 +170,9 @@ export class MetadataCollection {
     const oldbie = this.arrays_.get(type);
     if (oldbie !== undefined) return [oldbie, false, () => {}];
 
-    const $id = this.getName(checker, type);
+    const id = this.getName(checker, type);
     const array: MetadataArrayType = MetadataArrayType.create({
-      name: $id,
+      name: id,
       value: null!,
       index: null,
       recursive: null!,
@@ -189,9 +189,9 @@ export class MetadataCollection {
     const oldbie = this.tuples_.get(type);
     if (oldbie !== undefined) return [oldbie, false, () => {}];
 
-    const $id = this.getName(checker, type);
+    const id = this.getName(checker, type);
     const tuple: MetadataTupleType = MetadataTupleType.create({
-      name: $id,
+      name: id,
       elements: null!,
       index: null,
       recursive: null!,

src/programmers/internal/json_schema_array.ts

@@ -23,16 +23,20 @@ export const json_schema_array = (props: {
       tags: props.array.tags,
     });
   if (props.array.type.recursive === true) {
-    const out = () => [{ $ref }];
-    const $ref: string = `#/components/schemas/${props.array.type.name}`;
-    if (props.components.schemas?.[$ref] !== undefined) return out();
+    const out = () => [
+      {
+        $ref: `#/components/schemas/${props.array.type.name}`,
+      },
+    ];
+    if (props.components.schemas?.[props.array.type.name] !== undefined)
+      return out();
 
     props.components.schemas ??= {};
-    props.components.schemas[$ref] ??= {};
+    props.components.schemas[props.array.type.name] ??= {};
 
     const oneOf: OpenApi.IJsonSchema.IArray[] = factory();
     Object.assign(
-      props.components.schemas[$ref]!,
+      props.components.schemas[props.array.type.name]!,
       oneOf.length === 1 ? oneOf[0] : { oneOf },
     );
     return out();

src/programmers/llm/LlmApplicationProgrammer.ts

@@ -11,7 +11,7 @@ import { MetadataObjectType } from "../../schemas/metadata/MetadataObjectType";
 import { LlmSchemaProgrammer } from "./LlmSchemaProgrammer";
 
 export namespace LlmApplicationProgrammer {
-  export const validate = () => {
+  export const validate = (model: ILlmApplication.Model) => {
     let top: Metadata | undefined;
     return (
       metadata: Metadata,
@@ -28,7 +28,7 @@ export namespace LlmApplicationProgrammer {
           metadata.functions.length === 1
         )
           return validateFunction(metadata.functions[0]!);
-        else return LlmSchemaProgrammer.validate(metadata);
+        else return LlmSchemaProgrammer.validate(model)(metadata);
 
       const output: string[] = [];
       const valid: boolean =
@@ -88,7 +88,7 @@ export namespace LlmApplicationProgrammer {
     model: Model;
     metadata: Metadata;
   }): ILlmApplication<Model> => {
-    const errors: string[] = validate()(props.metadata, {
+    const errors: string[] = validate(props.model)(props.metadata, {
       top: true,
       object: null,
       property: null,

src/programmers/llm/LlmSchemaProgrammer.ts

@@ -5,45 +5,14 @@ import { IJsonSchemaCollection } from "../../schemas/json/IJsonSchemaCollection"
 import { Metadata } from "../../schemas/metadata/Metadata";
 
 import { AtomicPredicator } from "../helpers/AtomicPredicator";
+import { json_schema_bigint } from "../internal/json_schema_bigint";
+import { json_schema_boolean } from "../internal/json_schema_boolean";
+import { json_schema_native } from "../internal/json_schema_native";
+import { json_schema_number } from "../internal/json_schema_number";
+import { json_schema_string } from "../internal/json_schema_string";
 import { JsonSchemasProgrammer } from "../json/JsonSchemasProgrammer";
 
 export namespace LlmSchemaProgrammer {
-  export const validate = (metadata: Metadata): string[] => {
-    const output: string[] = [];
-    if (
-      metadata.atomics.some((a) => a.type === "bigint") ||
-      metadata.constants.some((c) => c.type === "bigint")
-    )
-      output.push("LLM schema does not support bigint type.");
-    if (
-      metadata.tuples.some((t) =>
-        t.type.elements.some((e) => e.isRequired() === false),
-      ) ||
-      metadata.arrays.some((a) => a.type.value.isRequired() === false)
-    )
-      output.push("LLM schema does not support undefined type in array.");
-    if (metadata.maps.length)
-      output.push("LLM schema does not support Map type.");
-    if (metadata.sets.length)
-      output.push("LLM schema does not support Set type.");
-    for (const native of metadata.natives)
-      if (
-        AtomicPredicator.native(native.name) === false &&
-        native.name !== "Date" &&
-        native.name !== "Blob" &&
-        native.name !== "File"
-      )
-        output.push(`LLM schema does not support ${native.name} type.`);
-    // if (
-    //   metadata.aliases.some((a) => a.type.recursive) ||
-    //   metadata.arrays.some((a) => a.type.recursive) ||
-    //   metadata.objects.some((o) => o.type.recursive) ||
-    //   metadata.tuples.some((t) => t.type.recursive)
-    // )
-    //   output.push("LLM schema does not support recursive type.");
-    return output;
-  };
-
   export const write = <Model extends ILlmApplication.Model>(props: {
     model: Model;
     metadata: Metadata;
@@ -64,4 +33,82 @@ export namespace LlmSchemaProgrammer {
       throw new Error("Failed to convert JSON schema to LLM schema.");
     return schema;
   };
+
+  export const validate =
+    (model: ILlmApplication.Model) =>
+    (metadata: Metadata): string[] => {
+      const output: string[] = [];
+      if (
+        metadata.atomics.some((a) => a.type === "bigint") ||
+        metadata.constants.some((c) => c.type === "bigint")
+      )
+        output.push("LLM schema does not support bigint type.");
+      if (
+        metadata.tuples.some((t) =>
+          t.type.elements.some((e) => e.isRequired() === false),
+        ) ||
+        metadata.arrays.some((a) => a.type.value.isRequired() === false)
+      )
+        output.push("LLM schema does not support undefined type in array.");
+      if (metadata.maps.length)
+        output.push("LLM schema does not support Map type.");
+      if (metadata.sets.length)
+        output.push("LLM schema does not support Set type.");
+      for (const native of metadata.natives)
+        if (
+          AtomicPredicator.native(native.name) === false &&
+          native.name !== "Date" &&
+          native.name !== "Blob" &&
+          native.name !== "File"
+        )
+          output.push(`LLM schema does not support ${native.name} type.`);
+      if (model === "gemini" && size(metadata) > 1)
+        output.push("Gemini model does not support the union type.");
+      // if (
+      //   metadata.aliases.some((a) => a.type.recursive) ||
+      //   metadata.arrays.some((a) => a.type.recursive) ||
+      //   metadata.objects.some((o) => o.type.recursive) ||
+      //   metadata.tuples.some((t) => t.type.recursive)
+      // )
+      //   output.push("LLM schema does not support recursive type.");
+      return output;
+    };
 }
+
+const size = (metadata: Metadata): number =>
+  (metadata.escaped ? size(metadata.escaped.returns) : 0) +
+  metadata.aliases.length +
+  metadata.objects.length +
+  metadata.arrays.length +
+  metadata.tuples.length +
+  (metadata.maps.length ? 1 : 0) +
+  (metadata.sets.length ? 1 : 0) +
+  metadata.atomics
+    .map((a) =>
+      a.type === "boolean"
+        ? json_schema_boolean(a).length
+        : a.type === "bigint"
+          ? json_schema_bigint(a).length
+          : a.type === "number"
+            ? json_schema_number(a).length
+            : json_schema_string(a).length,
+    )
+    .reduce((a, b) => a + b, 0) +
+  metadata.constants.filter(
+    (c) => metadata.atomics.some((a) => a.type === c.type) === false,
+  ).length +
+  metadata.templates.length +
+  metadata.natives
+    .filter(
+      (n) =>
+        metadata.atomics.some((a) => a.type === n.name) === false &&
+        metadata.constants.some((c) => c.type === n.name) === false,
+    )
+    .map(
+      (n) =>
+        json_schema_native({
+          components: {},
+          native: n,
+        }).length,
+    )
+    .reduce((a, b) => a + b, 0);

src/transformers/features/llm/LlmApplicationTransformer.ts

@@ -28,6 +28,19 @@ export namespace LlmApplicationTransformer {
     const top: ts.Node = props.expression.typeArguments[0]!;
     if (ts.isTypeNode(top) === false) return props.expression;
 
+    // GET MODEL
+    const model: ILlmApplication.Model = get_parameter<ILlmApplication.Model>({
+      checker: props.context.checker,
+      name: "Model",
+      is: (value) =>
+        value === "3.1" ||
+        value === "3.0" ||
+        value === "chatgpt" ||
+        value === "gemini",
+      cast: (value) => value as ILlmApplication.Model,
+      default: () => "3.1",
+    })(props.expression.typeArguments[1]);
+
     // GET TYPE
     const type: ts.Type = props.context.checker.getTypeFromTypeNode(top);
     const collection: MetadataCollection = new MetadataCollection({
@@ -42,7 +55,7 @@ export namespace LlmApplicationTransformer {
           constant: true,
           absorb: false,
           functional: true,
-          validate: LlmApplicationProgrammer.validate(),
+          validate: LlmApplicationProgrammer.validate(model),
         },
         collection,
         type,
@@ -54,17 +67,6 @@ export namespace LlmApplicationTransformer {
       });
 
     // GENERATE LLM APPLICATION
-    const model: ILlmApplication.Model = get_parameter<ILlmApplication.Model>({
-      checker: props.context.checker,
-      name: "Model",
-      is: (value) =>
-        value === "3.1" ||
-        value === "3.0" ||
-        value === "chatgpt" ||
-        value === "gemini",
-      cast: (value) => value as ILlmApplication.Model,
-      default: () => "3.1",
-    })(props.expression.typeArguments[1]);
     const schema: ILlmApplication<ILlmApplication.Model> =
       LlmApplicationProgrammer.write({
         model,

src/transformers/features/llm/LlmSchemaTransformer.ts

@@ -28,6 +28,19 @@ export namespace LlmSchemaTransformer {
     const top: ts.Node = props.expression.typeArguments[0]!;
     if (ts.isTypeNode(top) === false) return props.expression;
 
+    // GET MODEL
+    const model: ILlmApplication.Model = get_parameter<ILlmApplication.Model>({
+      checker: props.context.checker,
+      name: "Model",
+      is: (value) =>
+        value === "3.1" ||
+        value === "3.0" ||
+        value === "chatgpt" ||
+        value === "gemini",
+      cast: (value) => value as ILlmApplication.Model,
+      default: () => "3.1",
+    })(props.expression.typeArguments[1]);
+
     // GET TYPE
     const type: ts.Type = props.context.checker.getTypeFromTypeNode(top);
     const collection: MetadataCollection = new MetadataCollection({
@@ -41,7 +54,7 @@ export namespace LlmSchemaTransformer {
           escape: true,
           constant: true,
           absorb: false,
-          validate: LlmSchemaProgrammer.validate,
+          validate: LlmSchemaProgrammer.validate(model),
         },
         collection,
         type,
@@ -53,17 +66,6 @@ export namespace LlmSchemaTransformer {
       });
 
     // GENERATE LLM SCHEMA
-    const model: ILlmApplication.Model = get_parameter<ILlmApplication.Model>({
-      checker: props.context.checker,
-      name: "Model",
-      is: (value) =>
-        value === "3.1" ||
-        value === "3.0" ||
-        value === "chatgpt" ||
-        value === "gemini",
-      cast: (value) => value as ILlmApplication.Model,
-      default: () => "3.1",
-    })(props.expression.typeArguments[1]);
     const schema: ILlmApplication.ModelSchema[ILlmApplication.Model] =
       LlmSchemaProgrammer.write({
         model,