Drive web frontend using ontario (#53)

* Drive web frontend using ontario

Stop crashes

Change default vals

Fix upload_image breakage

* It works, somewhat

.gitignore

@@ -2,4 +2,4 @@ output/
 node_modules
 lib
 build/
-**/__pycache__
+**/__pycache__

backend/ontario-web/ontario_web/__init__.py

@@ -75,7 +75,6 @@ def upload_image():
         # The body of the request should be an image
         image = request.files["image"]
         new_id = im.add_image(image.filename)
-        os.remove(image.filename)
         return {"id": new_id}
 
     # For /api/process, take the body of the request and process it

backend/ontario-web/ontario_web/image_manager.py

@@ -76,9 +76,6 @@ def add_image(self, path: str) -> int:
         Adds an image to the image manager.
         """
 
-        # Get the image size.
-        image_size = os.path.getsize(path)
-
         # Create a new image ID.
         image_id = self.__last_image_id
         self.__last_image_id += 1
@@ -91,6 +88,9 @@ def add_image(self, path: str) -> int:
         context = ontario.ImageContext()
         ontario.ImageBuilder(context).load_from_file(
             path).save_to_file(image_path).process()
+
+        # get the image size
+        image_size = os.path.getsize(image_path)
 
         # Add the image to the image map.
         image_info = _ImageInfo(image_id, image_path, image_size)

backend/ontario-web/ontario_web/nodes.py

@@ -622,6 +622,7 @@ def link(
         toId: int,
         toIndex: int,
         meta: M,
+        id = None,
     ) -> Link[T, M]:
         """
         Link two nodes together and return the link object.
@@ -637,6 +638,11 @@ def link(
         #    raise ValueError("Input is occupied")
 
         link = to_node._linkFrom(from_node, fromIndex, toIndex, meta)
+        if not id:
+            id = self.__nextId
+            self.__nextId += 1
+        link.setId(id)
+        #print(link.getId())
         self.__links[link.getId()] = link
         return link
 
@@ -749,7 +755,8 @@ def deserializePipeline(
             serializedLink["fromIndex"],
             serializedLink["to"],
             serializedLink["toIndex"],
-            serializedLink.get("metadata", None)
+            serializedLink.get("metadata", None),
+            serializedLink.get("id", None),
         )
 
         if serializedLink.get("defaultValue") is not None:

backend/ontario-web/ontario_web/processor.py

@@ -16,13 +16,13 @@
 PipelineUnit = Union[ImageBuilder, int]
 
 
-def process(pipeline: str, images: ImageManager, target: str) -> None:
+def process(pipeline, images: ImageManager, target: str) -> None:
     """
     Processes a pipeline.
     """
 
     # Deserialize from JSON
-    pipeline = nodes.deserializePipeline(json.loads(pipeline), make_template_table())
+    pipeline = nodes.deserializePipeline(pipeline, make_template_table())
 
     # set metadata for all nodes
     context = ImageContext()

frontend/pictonode-web/package.json

@@ -12,12 +12,14 @@
     "@mdi/font": "^7.0.96",
     "@vue-flow/additional-components": "^1.2.4",
     "@vue-flow/core": "^1.4.1",
+    "axios": "^1.3.3",
     "libnode": "file:../../libraries/libnode",
     "vue": "^3.2.45",
     "vue-router": "^4.1.6",
     "vuetify": "^3.0.1"
   },
   "devDependencies": {
+    "@types/axios": "^0.14.0",
     "css-loader": "^6.7.2",
     "sass": "^1.56.1",
     "sass-loader": "^13.2.0",

frontend/pictonode-web/src/api.ts

@@ -0,0 +1,30 @@
+// GNU AGPL v3 License
+// Written by John Nunley and Grace Meredith.
+
+import { API_URL } from "./consts";
+
+import axios from "axios";
+
+const API = axios.create({
+  baseURL: API_URL,
+  timeout: 10000,
+});
+
+export function uploadImage(file: File): Promise<number> {
+  const formData = new FormData();
+  formData.append("image", file);
+
+  // Response will be JSON with number field "id"
+  return API.post("/upload_image", formData, { responseType: "json" }).then(
+    (response) => response.data.id
+  );
+}
+
+export function processPipeline(pipeline: any): Promise<File> {
+  // The body of the returning request will be an image file.
+  return API.post(
+    "/process",
+    pipeline,
+    { responseType: "blob" }
+  ).then((response) => response.data);
+}

frontend/pictonode-web/src/components/nodes/Flow.vue

@@ -18,6 +18,8 @@ import {
   nodeToViewFlow,
 } from "./NodeTree";
 import getTemplates from "./Templates";
+import { serializePipeline } from "libnode";
+import { processPipeline } from "../../api";
 
 const pipeline = defaultPipeline();
 const elements = pipelineToVueFlow(pipeline);
@@ -112,17 +114,44 @@ export default defineComponent({
         ctx.fillStyle = "#FFFFFF";
         ctx.font = "30px Arial";
         ctx.fillText("No output node", 10, 50);
+
+        this.$emit("canvas-update", canvas);
       } else {
-        // Get the first input link and get the canvas from it.
-        const inputLink = outputNode.getInputs()[0];
-        const data = inputLink.get();
-        if (data.type !== NodeDataType.Image) {
-          throw new Error("Output node does not have an image input");
+        // Parse the tree to a JSON format.
+        // @ts-ignore
+        const formatted = serializePipeline(this.pipeline);
+
+        // Replace default values with IDs.
+        for (const link of formatted.links) {
+          if ("defaultValue" in link && link.defaultValue) {
+            // @ts-ignore
+            link.defaultValue = link.defaultValue.id;
+          }
         }
-        canvas = data.canvas;
-      }
-
-      this.$emit("canvas-update", canvas);
+
+        formatted.output = outputNode.getId();
+
+        // Run the API.
+        processPipeline(formatted).then((image_file) => {
+          // Conver the image_file `Blob` to a canvas
+          const img = new Image();
+          img.src = URL.createObjectURL(image_file);
+
+          img.onload = () => {
+            canvas = document.createElement("canvas");
+            canvas.width = img.width;
+            canvas.height = img.height;
+            const ctx = canvas.getContext("2d");
+
+            if (!ctx) {
+              throw new Error("Could not get 2d context");
+            }
+
+            ctx.drawImage(img, 0, 0);
+            this.$emit("canvas-update", canvas);
+          };
+        });
+      } 
     },
 
     onConnect(connection: Connection | Edge) {

frontend/pictonode-web/src/components/nodes/InnerSwitch.vue

@@ -7,6 +7,7 @@
 <script lang="ts">
 import { defineComponent } from "vue";
 import { Node, SpecialNodeType, MetadataType, NodeDataType } from "./NodeTree";
+import { uploadImage } from "../../api";
 
 export default defineComponent({
   props: {
@@ -27,35 +28,13 @@ export default defineComponent({
       // @ts-ignore
       const imageBlob = this.$refs.imageInput.files[0];
 
-      // Convert it to a canvas.
-      const reader = new FileReader();
-      reader.readAsDataURL(imageBlob);
-      reader.onload = (e) => {
-        const img = new Image();
-        img.onload = () => {
-          // Create a canvas.
-          const canvas = document.createElement("canvas");
-          canvas.width = img.width;
-          canvas.height = img.height;
-
-          // Draw the image on the canvas.
-          const ctx = canvas.getContext("2d");
-          if (!ctx) {
-            throw new Error("Could not get canvas context");
-          }
-
-          ctx.drawImage(img, 0, 0);
-
-          this.node.getOutputs()[0].set({
-            type: NodeDataType.Image,
-            canvas,
-          });
-          this.$emit("input-update");
-        };
-
-        // @ts-ignore
-        img.src = e.target!.result;
-      };
+      // Upload to the server.
+      uploadImage(imageBlob).then((id) => {
+        this.node.getOutputs()[0].set({
+          type: NodeDataType.Image,
+          id,
+        });
+      });
     },
 
     onColorInputUpdate(color: string) {

frontend/pictonode-web/src/components/nodes/NodeTree.ts

@@ -51,7 +51,11 @@ export enum SpecialNodeType {
 export type NodeData =
   | {
       type: NodeDataType.Image;
-      canvas: HTMLCanvasElement;
+      id: number;
+    } |
+    {
+      type: NodeDataType.Image;
+      image: HTMLCanvasElement;
     }
   | {
       type: NodeDataType.Color;

frontend/pictonode-web/src/components/nodes/Templates.ts

@@ -87,55 +87,11 @@ function initializeTemplates() {
 type Link2 = Link<NodeData, NodeMetadata>;
 
 function composite(input: Array<Link2>): Array<NodeData> {
-  // Check both inputs.
-  if (input.length != 2) {
-    throw new Error("Composite node must have two inputs.");
-  }
-
-  const data1 = input[0].get();
-  const data2 = input[1].get();
-
-  // Convert both to images.
-  const image1 = transformToImage(data1);
-  const image2 = transformToImage(data2);
-
-  // Get the minimum width and height.
-  const width = Math.min(image1.width, image2.width);
-  const height = Math.min(image1.height, image2.height);
-
-  // Create a new canvas.
-  const canvas = document.createElement("canvas");
-  canvas.width = width;
-  canvas.height = height;
-
-  // Get the context.
-  const ctx = canvas.getContext("2d")!;
-
-  // Composite the images.
-  ctx.drawImage(image2, 0, 0, width, height);
-  ctx.drawImage(image1, 0, 0, width, height);
-
-  // Return the new image.
-  return [{ type: NodeDataType.Image, canvas }];
+  throw new Error("Not implemented.");
 }
 
 function transformToImage(data: NodeData): HTMLCanvasElement {
-  switch (data.type) {
-    case NodeDataType.Image:
-      return data.canvas;
-    case NodeDataType.Color:
-      const width = 10000;
-      const height = 10000;
-      const canvas = document.createElement("canvas");
-      canvas.width = width;
-      canvas.height = height;
-      const ctx = canvas.getContext("2d")!;
-      ctx.fillStyle = data.color;
-      ctx.fillRect(0, 0, width, height);
-      return canvas;
-    default:
-      throw new Error("Unknown node data type.");
-  }
+  throw new Error("Not implemented.");
 }
 
 function ntMeta(
@@ -163,32 +119,44 @@ function defaultImage(): NodeData {
   const canvas = document.createElement("canvas");
   canvas.width = 100;
   canvas.height = 100;
-  const ctx = canvas.getContext("2d")!;
-  ctx.fillStyle = "#FF4433";
-  ctx.fillRect(0, 0, 100, 100);
+  const ctx = canvas.getContext("2d");
+  if (ctx === null) {
+    throw new Error("Could not get context.");
+  }
 
-  // Write black text saying "error".
+  ctx.fillStyle = "#FF00FF";
+  ctx.fillRect(0, 0, 100, 100);
+
+  // Write "default image"
   ctx.fillStyle = "#000000";
   ctx.font = "20px Arial";
-  ctx.fillText("Error", 10, 50);
+  ctx.fillText("default image", 10, 50);
 
-  return { type: NodeDataType.Image, canvas };
+  return {
+    type: NodeDataType.Image,
+    image: canvas,
+  }
 }
 
 function noOutputImage(): NodeData {
   const canvas = document.createElement("canvas");
-  canvas.width = 200;
+  canvas.width = 100;
   canvas.height = 100;
-  const ctx = canvas.getContext("2d")!;
+  const ctx = canvas.getContext("2d");
+  if (ctx === null) {
+    throw new Error("Could not get context.");
+  }
 
-  // Black background.
+  ctx.fillStyle = "#FF00FF";
+  ctx.fillRect(0, 0, 100, 100);
+
+  // Write "no output image"
   ctx.fillStyle = "#000000";
-  ctx.fillRect(0, 0, 200, 100);
-
-  // White text saying "no image selected".
-  ctx.fillStyle = "#FFFFFF";
   ctx.font = "20px Arial";
-  ctx.fillText("No image selected", 10, 50);
+  ctx.fillText("no output image", 10, 50);
 
-  return { type: NodeDataType.Image, canvas };
+  return {
+    type: NodeDataType.Image,
+    image: canvas,
+  }
 }

frontend/pictonode-web/src/consts.ts

@@ -0,0 +1,5 @@
+// GNU AGPL v3 License
+
+// This file in its entirety was written by John Nunley and Grace Meredith.
+
+export const API_URL = "/api/";