Merge pull request #168 from biggraph/darabos-ws-docs

examples/fake_data.py

@@ -7,6 +7,11 @@ faker = Faker()
 
 @op("LynxKite Graph Analytics", "Fake data")
 def fake(*, n=10):
+    """Creates a DataFrame with random-generated names and postal addresses.
+
+    Parameters:
+        n: Number of rows to create.
+    """
     df = pd.DataFrame(
         {
             "name": [faker.name() for _ in range(n)],

lynxkite-app/pyproject.toml

@@ -10,6 +10,7 @@ dependencies = [
     "orjson>=3.10.13",
     "pycrdt-websocket>=0.15.3",
     "sse-starlette>=2.2.1",
+    "griffe>=1.7.3",
 ]
 
 [project.optional-dependencies]

lynxkite-app/web/package-lock.json

@@ -30,6 +30,7 @@
         "react-error-boundary": "^5.0.0",
         "react-markdown": "^9.0.1",
         "react-router-dom": "^7.5.2",
+        "react-tooltip": "^5.28.1",
         "swr": "^2.2.5",
         "unplugin-icons": "^0.21.0",
         "y-monaco": "^0.1.6",
@@ -940,6 +941,31 @@
         "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
       }
     },
+    "node_modules/@floating-ui/core": {
+      "version": "1.6.9",
+      "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.6.9.tgz",
+      "integrity": "sha512-uMXCuQ3BItDUbAMhIXw7UPXRfAlOAvZzdK9BWpE60MCn+Svt3aLn9jsPTi/WNGlRUu2uI0v5S7JiIUsbsvh3fw==",
+      "license": "MIT",
+      "dependencies": {
+        "@floating-ui/utils": "^0.2.9"
+      }
+    },
+    "node_modules/@floating-ui/dom": {
+      "version": "1.6.13",
+      "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.6.13.tgz",
+      "integrity": "sha512-umqzocjDgNRGTuO7Q8CU32dkHkECqI8ZdMZ5Swb6QAM0t5rnlrN3lGo1hdpscRd3WS8T6DKYK4ephgIH9iRh3w==",
+      "license": "MIT",
+      "dependencies": {
+        "@floating-ui/core": "^1.6.0",
+        "@floating-ui/utils": "^0.2.9"
+      }
+    },
+    "node_modules/@floating-ui/utils": {
+      "version": "0.2.9",
+      "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.9.tgz",
+      "integrity": "sha512-MDWhGtE+eHw5JW7lq4qhc5yRLS11ERl1c7Z6Xd0a58DozHES6EnNNwUWbMiG4J9Cgj053Bhk8zvlhFYKVhULwg==",
+      "license": "MIT"
+    },
     "node_modules/@humanfs/core": {
       "version": "0.19.1",
       "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz",
@@ -2879,6 +2905,12 @@
       "integrity": "sha512-JhZUT7JFcQy/EzW605k/ktHtncoo9vnyW/2GspNYwFlN1C/WmjuV/xtS04e9SOkL2sTdw0VAZ2UGCcQ9lR6p6w==",
       "license": "MIT"
     },
+    "node_modules/classnames": {
+      "version": "2.5.1",
+      "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.5.1.tgz",
+      "integrity": "sha512-saHYOzhIQs6wy2sVxTM6bUDsQO4F50V9RQ22qBpEdCW+I+/Wmke2HOl6lS6dTpdxVhb88/I6+Hs+438c3lfUow==",
+      "license": "MIT"
+    },
     "node_modules/client-only": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz",
@@ -6239,6 +6271,20 @@
         "react-dom": ">=18"
       }
     },
+    "node_modules/react-tooltip": {
+      "version": "5.28.1",
+      "resolved": "https://registry.npmjs.org/react-tooltip/-/react-tooltip-5.28.1.tgz",
+      "integrity": "sha512-ZA4oHwoIIK09TS7PvSLFcRlje1wGZaxw6xHvfrzn6T82UcMEfEmHVCad16Gnr4NDNDh93HyN037VK4HDi5odfQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@floating-ui/dom": "^1.6.1",
+        "classnames": "^2.3.0"
+      },
+      "peerDependencies": {
+        "react": ">=16.14.0",
+        "react-dom": ">=16.14.0"
+      }
+    },
     "node_modules/read-cache": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz",

lynxkite-app/web/package.json

@@ -33,6 +33,7 @@
     "react-error-boundary": "^5.0.0",
     "react-markdown": "^9.0.1",
     "react-router-dom": "^7.5.2",
+    "react-tooltip": "^5.28.1",
     "swr": "^2.2.5",
     "unplugin-icons": "^0.21.0",
     "y-monaco": "^0.1.6",

lynxkite-app/web/src/Tooltip.tsx

@@ -0,0 +1,21 @@
+import { useId } from "react";
+import Markdown from "react-markdown";
+import { Tooltip as ReactTooltip } from "react-tooltip";
+
+export default function Tooltip(props: any) {
+  const id = useId();
+  if (!props.doc) return null;
+  return (
+    <>
+      <a data-tooltip-id={id} tabIndex={0}>
+        {props.children}
+      </a>
+      <ReactTooltip id={id} className="tooltip" place="top-end">
+        {props.doc.map?.(
+          (section: any, i: number) =>
+            section.kind === "text" && <Markdown key={i}>{section.value}</Markdown>,
+        ) ?? <Markdown>{props.doc}</Markdown>}
+      </ReactTooltip>
+    </>
+  );
+}

lynxkite-app/web/src/index.css

@@ -68,11 +68,6 @@ body {
     font-size: 12px;
   }
 
-  .title-icon {
-    margin-left: 5px;
-    float: right;
-  }
-
   .node-container {
     padding: 8px;
     position: relative;
@@ -93,8 +88,21 @@ body {
     }
   }
 
+  .tooltip {
+    padding: 8px;
+    border-radius: 4px;
+    opacity: 1;
+    text-align: left;
+    background: #fffa;
+    color: black;
+    box-shadow: 0px 5px 50px 0px rgba(0, 0, 0, 0.1);
+    backdrop-filter: blur(10px);
+    font-size: 16px;
+    font-weight: initial;
+    max-width: 300px;
+  }
+
   .expanded .lynxkite-node {
-    overflow-y: auto;
     height: 100%;
   }
 
@@ -114,6 +122,13 @@ body {
     --status-color-2: #0000;
     --status-color-3: #0000;
     transition: --status-color-1 0.3s, --status-color-2 0.3s, --status-color-3 0.3s;
+    display: flex;
+    flex-direction: row;
+    gap: 10px;
+
+    .title-title {
+      flex: 1;
+    }
   }
 
   .lynxkite-node .title.active {
@@ -143,24 +158,40 @@ body {
     visibility: hidden;
   }
 
-  .react-flow__handle-left .handle-name {
-    right: 20px;
+  .react-flow__handle-left {
+    left: -5px;
+
+    .handle-name {
+      right: 30px;
+    }
   }
 
-  .react-flow__handle-right .handle-name {
-    left: 20px;
+  .react-flow__handle-right {
+    right: -5px;
+
+    .handle-name {
+      left: 30px;
+    }
   }
 
-  .react-flow__handle-top .handle-name {
-    top: -20px;
-    left: 5px;
-    backdrop-filter: none;
+  .react-flow__handle-top {
+    top: -5px;
+
+    .handle-name {
+      top: -3px;
+      left: 13px;
+      backdrop-filter: none;
+    }
   }
 
-  .react-flow__handle-bottom .handle-name {
-    top: 0px;
-    left: 5px;
-    backdrop-filter: none;
+  .react-flow__handle-bottom {
+    bottom: -5px;
+
+    .handle-name {
+      top: 0px;
+      left: 13px;
+      backdrop-filter: none;
+    }
   }
 
   .node-container:hover .handle-name {
@@ -180,6 +211,13 @@ body {
       display: block;
     }
 
+    .param-name-row {
+      display: flex;
+      flex-direction: row;
+      justify-content: space-between;
+      align-items: end;
+    }
+
     .param-name {
       display: block;
       font-size: 10px;
@@ -453,8 +491,9 @@ body {
 .react-flow__handle {
   border-color: black;
   background: white;
-  width: 10px;
-  height: 10px;
+  width: 20px;
+  height: 20px;
+  border-width: 2px;
 }
 
 .react-flow__arrowhead * {

lynxkite-app/web/src/workspace/LynxKiteEdge.tsx

@@ -0,0 +1,30 @@
+import { BaseEdge, Position } from "@xyflow/react";
+
+function addOffset(x: number, y: number, p: Position, offset: number) {
+  if (p === Position.Top) return `${x},${y - offset}`;
+  if (p === Position.Bottom) return `${x},${y + offset}`;
+  if (p === Position.Left) return `${x - offset},${y}`;
+  return `${x + offset},${y}`;
+}
+
+export default function LynxKiteEdge(props: any) {
+  const offset = 0.3 * Math.hypot(props.targetX - props.sourceX, props.targetY - props.sourceY);
+  const s = addOffset(props.sourceX, props.sourceY, props.sourcePosition, 0);
+  const sc = addOffset(props.sourceX, props.sourceY, props.sourcePosition, offset);
+  const tc = addOffset(props.targetX, props.targetY, props.targetPosition, offset);
+  const t = addOffset(props.targetX, props.targetY, props.targetPosition, 0);
+  const path = `M${s} C${sc} ${tc} ${t}`;
+  return (
+    <>
+      <BaseEdge
+        id={props.id}
+        path={path}
+        {...props}
+        style={{
+          strokeWidth: 2,
+          stroke: "black",
+        }}
+      />
+    </>
+  );
+}

lynxkite-app/web/src/workspace/Workspace.tsx

@@ -34,6 +34,7 @@ import favicon from "../assets/favicon.ico";
 import { usePath } from "../common.ts";
 // import NodeWithTableView from './NodeWithTableView';
 import EnvironmentSelector from "./EnvironmentSelector";
+import LynxKiteEdge from "./LynxKiteEdge.tsx";
 import { LynxKiteState } from "./LynxKiteState";
 import NodeSearch, { type OpsOp, type Catalog, type Catalogs } from "./NodeSearch.tsx";
 import NodeWithGraphCreationView from "./nodes/GraphCreationNode.tsx";
@@ -186,6 +187,12 @@ function LynxKiteFlow() {
     }),
     [],
   );
+  const edgeTypes = useMemo(
+    () => ({
+      default: LynxKiteEdge,
+    }),
+    [],
+  );
 
   // Global keyboard shortcuts.
   useEffect(() => {
@@ -280,7 +287,7 @@ function LynxKiteFlow() {
     (connection: Connection) => {
       setSuppressSearchUntil(Date.now() + 200);
       const edge = {
-        id: `${connection.source} ${connection.target}`,
+        id: `${connection.source} ${connection.sourceHandle} ${connection.target} ${connection.targetHandle}`,
         source: connection.source,
         sourceHandle: connection.sourceHandle!,
         target: connection.target,
@@ -374,6 +381,7 @@ function LynxKiteFlow() {
             nodes={nodes}
             edges={edges}
             nodeTypes={nodeTypes}
+            edgeTypes={edgeTypes}
             fitView
             onNodesChange={onNodesChange}
             onEdgesChange={onEdgesChange}

lynxkite-app/web/src/workspace/nodes/LynxKiteNode.tsx

@@ -1,9 +1,17 @@
 import { Handle, NodeResizeControl, type Position, useReactFlow } from "@xyflow/react";
+import type React from "react";
 import { ErrorBoundary } from "react-error-boundary";
 // @ts-ignore
+import AlertTriangle from "~icons/tabler/alert-triangle-filled.jsx";
+// @ts-ignore
 import ChevronDownRight from "~icons/tabler/chevron-down-right.jsx";
 // @ts-ignore
+import Dots from "~icons/tabler/dots.jsx";
+// @ts-ignore
+import Help from "~icons/tabler/question-mark.jsx";
+// @ts-ignore
 import Skull from "~icons/tabler/skull.jsx";
+import Tooltip from "../../Tooltip";
 
 interface LynxKiteNodeProps {
   id: string;
@@ -44,9 +52,11 @@ function getHandles(inputs: any[], outputs: any[]) {
 }
 
 const OP_COLORS: { [key: string]: string } = {
+  pink: "oklch(75% 0.2 0)",
   orange: "oklch(75% 0.2 55)",
+  green: "oklch(75% 0.2 150)",
   blue: "oklch(75% 0.2 230)",
-  green: "oklch(75% 0.2 130)",
+  purple: "oklch(75% 0.2 290)",
 };
 
 function LynxKiteNodeComponent(props: LynxKiteNodeProps) {
@@ -81,9 +91,20 @@ function LynxKiteNodeComponent(props: LynxKiteNodeProps) {
           style={titleStyle}
           onClick={titleClicked}
         >
-          {data.title}
-          {data.error && <span className="title-icon">⚠️</span>}
-          {expanded || <span className="title-icon">⋯</span>}
+          <span className="title-title">{data.title}</span>
+          {data.error && (
+            <Tooltip doc={`Error: ${data.error}`}>
+              <AlertTriangle />
+            </Tooltip>
+          )}
+          {expanded || (
+            <Tooltip doc="Click to expand node">
+              <Dots />
+            </Tooltip>
+          )}
+          <Tooltip doc={data.meta?.value?.doc}>
+            <Help />
+          </Tooltip>
         </div>
         {expanded && (
           <>

lynxkite-app/web/src/workspace/nodes/NodeParameter.tsx

@@ -1,6 +1,9 @@
 import { useRef } from "react";
 // @ts-ignore
 import ArrowsHorizontal from "~icons/tabler/arrows-horizontal.jsx";
+// @ts-ignore
+import Help from "~icons/tabler/question-mark.jsx";
+import Tooltip from "../../Tooltip";
 import NodeGroupParameter from "./NodeGroupParameter";
 
 const BOOLEAN = "<class 'bool'>";
@@ -9,8 +12,19 @@ const MODEL_TRAINING_INPUT_MAPPING =
 const MODEL_INFERENCE_INPUT_MAPPING =
   "<class 'lynxkite_graph_analytics.ml_ops.ModelInferenceInputMapping'>";
 const MODEL_OUTPUT_MAPPING = "<class 'lynxkite_graph_analytics.ml_ops.ModelOutputMapping'>";
-function ParamName({ name }: { name: string }) {
-  return <span className="param-name bg-base-200">{name.replace(/_/g, " ")}</span>;
+
+function ParamName({ name, doc }: { name: string; doc: string }) {
+  const help = doc && (
+    <Tooltip doc={doc} width={200}>
+      <Help />
+    </Tooltip>
+  );
+  return (
+    <div className="param-name-row">
+      <span className="param-name bg-base-200">{name.replace(/_/g, " ")}</span>
+      {help}
+    </div>
+  );
 }
 
 function Input({
@@ -195,18 +209,31 @@ interface NodeParameterProps {
 
 export type UpdateOptions = { delay?: number };
 
+function findDocs(docs: any, parameter: string) {
+  for (const sec of docs) {
+    if (sec.kind === "parameters") {
+      for (const p of sec.value) {
+        if (p.name === parameter) {
+          return p.description;
+        }
+      }
+    }
+  }
+}
+
 export default function NodeParameter({ name, value, meta, data, setParam }: NodeParameterProps) {
+  const doc = findDocs(data.meta?.value?.doc ?? [], name);
   function onChange(value: any, opts?: UpdateOptions) {
     setParam(meta.name, value, opts || {});
   }
   return meta?.type?.format === "collapsed" ? (
     <label className="param">
-      <ParamName name={name} />
+      <ParamName name={name} doc={doc} />
       <button className="collapsed-param">⋯</button>
     </label>
   ) : meta?.type?.format === "textarea" ? (
     <label className="param">
-      <ParamName name={name} />
+      <ParamName name={name} doc={doc} />
       <textarea
         className="textarea textarea-bordered w-full"
         rows={6}
@@ -219,7 +246,7 @@ export default function NodeParameter({ name, value, meta, data, setParam }: Nod
     <NodeGroupParameter meta={meta} data={data} setParam={setParam} />
   ) : meta?.type?.enum ? (
     <label className="param">
-      <ParamName name={name} />
+      <ParamName name={name} doc={doc} />
       <select
         className="select select-bordered w-full"
         value={value || meta.type.enum[0]}
@@ -246,22 +273,22 @@ export default function NodeParameter({ name, value, meta, data, setParam }: Nod
     </div>
   ) : meta?.type?.type === MODEL_TRAINING_INPUT_MAPPING ? (
     <label className="param">
-      <ParamName name={name} />
+      <ParamName name={name} doc={doc} />
       <ModelMapping value={value} data={data} variant="training input" onChange={onChange} />
     </label>
   ) : meta?.type?.type === MODEL_INFERENCE_INPUT_MAPPING ? (
     <label className="param">
-      <ParamName name={name} />
+      <ParamName name={name} doc={doc} />
       <ModelMapping value={value} data={data} variant="inference input" onChange={onChange} />
     </label>
   ) : meta?.type?.type === MODEL_OUTPUT_MAPPING ? (
     <label className="param">
-      <ParamName name={name} />
+      <ParamName name={name} doc={doc} />
       <ModelMapping value={value} data={data} variant="output" onChange={onChange} />
     </label>
   ) : (
     <label className="param">
-      <ParamName name={name} />
+      <ParamName name={name} doc={doc} />
       <Input value={value} onChange={onChange} />
     </label>
   );

lynxkite-core/pyproject.toml

@@ -5,6 +5,7 @@ description = "A lightweight dependency for authoring LynxKite operations and ex
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
+    "griffe>=1.7.3",
 ]
 
 [project.optional-dependencies]

lynxkite-core/src/lynxkite/core/ops.py

@@ -1,19 +1,22 @@
 """API for implementing LynxKite operations."""
 
 from __future__ import annotations
+
 import asyncio
 import enum
 import functools
+import json
 import importlib
 import inspect
 import pathlib
 import subprocess
 import traceback
-import joblib
 import types
-import pydantic
 import typing
 from dataclasses import dataclass
+
+import joblib
+import pydantic
 from typing_extensions import Annotated
 
 if typing.TYPE_CHECKING:
@@ -180,6 +183,7 @@ class Op(BaseConfig):
     # TODO: Make type an enum with the possible values.
     type: str = "basic"  # The UI to use for this operation.
     color: str = "orange"  # The color of the operation in the UI.
+    doc: object = None
 
     def __call__(self, *inputs, **params):
         # Convert parameters.
@@ -236,6 +240,7 @@ def op(
     """Decorator for defining an operation."""
 
     def decorator(func):
+        doc = get_doc(func)
         sig = inspect.signature(func)
         _view = view
         if view == "matplotlib":
@@ -262,6 +267,7 @@ def op(
             _outputs = [Output(name="output", type=None)] if view == "basic" else []
         op = Op(
             func=func,
+            doc=doc,
             name=name,
             params=_params,
             inputs=inputs,
@@ -279,10 +285,11 @@ def op(
 
 def matplotlib_to_image(func):
     """Decorator for converting a matplotlib figure to an image."""
-    import matplotlib.pyplot as plt
     import base64
     import io
 
+    import matplotlib.pyplot as plt
+
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
         func(*args, **kwargs)
@@ -423,3 +430,15 @@ def run_user_script(script_path: pathlib.Path):
     spec = importlib.util.spec_from_file_location(script_path.stem, str(script_path))
     module = importlib.util.module_from_spec(spec)
     spec.loader.exec_module(module)
+
+
+def get_doc(func):
+    """Griffe is an optional dependency. When available, we returned the parsed docstring."""
+    try:
+        import griffe
+    except ImportError:
+        return func.__doc__
+    if func.__doc__ is None:
+        return None
+    doc = griffe.Docstring(func.__doc__).parse("google")
+    return json.loads(json.dumps(doc, cls=griffe.JSONEncoder))

lynxkite-graph-analytics/src/lynxkite_graph_analytics/lynxkite_ops.py

@@ -8,7 +8,6 @@ from collections import deque
 
 from . import core
 import grandcypher
-import joblib
 import matplotlib
 import networkx as nx
 import pandas as pd
@@ -16,7 +15,6 @@ import polars as pl
 import json
 
 
-mem = joblib.Memory(".joblib-cache")
 op = ops.op_registration(core.ENV)
 
 
@@ -82,8 +80,7 @@ def import_parquet(*, filename: str):
     return pd.read_parquet(filename)
 
 
-@op("Import CSV")
-@mem.cache
+@op("Import CSV", slow=True)
 def import_csv(*, filename: str, columns: str = "<from file>", separator: str = "<auto>"):
     """Imports a CSV file."""
     return pd.read_csv(
@@ -93,8 +90,7 @@ def import_csv(*, filename: str, columns: str = "<from file>", separator: str =
     )
 
 
-@op("Import GraphML")
-@mem.cache
+@op("Import GraphML", slow=True)
 def import_graphml(*, filename: str):
     """Imports a GraphML file."""
     files = fsspec.open_files(filename, compression="infer")
@@ -105,8 +101,7 @@ def import_graphml(*, filename: str):
     raise ValueError(f"No .graphml file found at {filename}")
 
 
-@op("Graph from OSM")
-@mem.cache
+@op("Graph from OSM", slow=True)
 def import_osm(*, location: str):
     import osmnx as ox
 

lynxkite-graph-analytics/src/lynxkite_graph_analytics/pytorch/pytorch_ops.py

@@ -8,7 +8,7 @@ from .pytorch_core import op, reg, ENV
 
 reg("Input: tensor", outputs=["output"], params=[P.basic("name")])
 reg("Input: graph edges", outputs=["edges"])
-reg("Input: sequential", outputs=["y"])
+reg("Input: sequential", outputs=["y"], params=[P.basic("name")])
 reg("Output", inputs=["x"], outputs=["x"], params=[P.basic("name")])
 
 
@@ -19,6 +19,7 @@ def lstm(x, *, input_size=1024, hidden_size=1024, dropout=0.0):
 
 reg(
     "Neural ODE",
+    color="blue",
     inputs=["x"],
     params=[
         P.basic("relative_tolerance"),
@@ -99,6 +100,7 @@ def concatenate(a, b):
 
 reg(
     "Graph conv",
+    color="blue",
     inputs=["x", "edges"],
     outputs=["x"],
     params=[P.options("type", ["GCNConv", "GATConv", "GATv2Conv", "SAGEConv"])],