Update the LynxScribe ops with OpenAI API change.

lynxkite-app/.gitignore

@@ -0,0 +1,3 @@
+/src/lynxkite/app/web_assets
+!/src/lynxkite/app/web_assets/__init__.py
+!/src/lynxkite/app/web_assets/assets/__init__.py

lynxkite-app/src/lynxkite/app/main.py

@@ -20,7 +20,6 @@ def detect_plugins():
         print("No modules found in lynxkite_plugins. Be sure to install some plugins.")
         return {}
 
-    print(list(pkgutil.iter_modules(lynxkite_plugins.__path__)))
     plugins = {}
     for _, name, _ in pkgutil.iter_modules(lynxkite_plugins.__path__):
         name = f"lynxkite_plugins.{name}"

lynxkite-core/build/lib/lynxkite/executors/one_by_one.py

@@ -1,175 +0,0 @@
-"""A LynxKite executor that assumes most operations operate on their input one by one."""
-
-from .. import ops
-from .. import workspace
-import orjson
-import pandas as pd
-import pydantic
-import traceback
-import inspect
-import typing
-
-
-class Context(ops.BaseConfig):
-    """Passed to operation functions as "_ctx" if they have such a parameter."""
-
-    node: workspace.WorkspaceNode
-    last_result: typing.Any = None
-
-
-class Output(ops.BaseConfig):
-    """Return this to send values to specific outputs of a node."""
-
-    output_handle: str
-    value: dict
-
-
-def df_to_list(df):
-    return df.to_dict(orient="records")
-
-
-def has_ctx(op):
-    sig = inspect.signature(op.func)
-    return "_ctx" in sig.parameters
-
-
-CACHES = {}
-
-
-def register(env: str, cache: bool = True):
-    """Registers the one-by-one executor."""
-    if cache:
-        CACHES[env] = {}
-        cache = CACHES[env]
-    else:
-        cache = None
-    ops.EXECUTORS[env] = lambda ws: execute(ws, ops.CATALOGS[env], cache=cache)
-
-
-def get_stages(ws, catalog):
-    """Inputs on top/bottom are batch inputs. We decompose the graph into a DAG of components along these edges."""
-    nodes = {n.id: n for n in ws.nodes}
-    batch_inputs = {}
-    inputs = {}
-    for edge in ws.edges:
-        inputs.setdefault(edge.target, []).append(edge.source)
-        node = nodes[edge.target]
-        op = catalog[node.data.title]
-        i = op.inputs[edge.targetHandle]
-        if i.position in "top or bottom":
-            batch_inputs.setdefault(edge.target, []).append(edge.source)
-    stages = []
-    for bt, bss in batch_inputs.items():
-        upstream = set(bss)
-        new = set(bss)
-        while new:
-            n = new.pop()
-            for i in inputs.get(n, []):
-                if i not in upstream:
-                    upstream.add(i)
-                    new.add(i)
-        stages.append(upstream)
-    stages.sort(key=lambda s: len(s))
-    stages.append(set(nodes))
-    return stages
-
-
-def _default_serializer(obj):
-    if isinstance(obj, pydantic.BaseModel):
-        return obj.dict()
-    return {"__nonserializable__": id(obj)}
-
-
-def make_cache_key(obj):
-    return orjson.dumps(obj, default=_default_serializer)
-
-
-EXECUTOR_OUTPUT_CACHE = {}
-
-
-async def await_if_needed(obj):
-    if inspect.isawaitable(obj):
-        return await obj
-    return obj
-
-
-async def execute(ws, catalog, cache=None):
-    nodes = {n.id: n for n in ws.nodes}
-    contexts = {n.id: Context(node=n) for n in ws.nodes}
-    edges = {n.id: [] for n in ws.nodes}
-    for e in ws.edges:
-        edges[e.source].append(e)
-    tasks = {}
-    NO_INPUT = object()  # Marker for initial tasks.
-    for node in ws.nodes:
-        node.data.error = None
-        op = catalog.get(node.data.title)
-        if op is None:
-            node.data.error = f'Operation "{node.data.title}" not found.'
-            continue
-        # Start tasks for nodes that have no non-batch inputs.
-        if all([i.position in "top or bottom" for i in op.inputs.values()]):
-            tasks[node.id] = [NO_INPUT]
-    batch_inputs = {}
-    # Run the rest until we run out of tasks.
-    stages = get_stages(ws, catalog)
-    for stage in stages:
-        next_stage = {}
-        while tasks:
-            n, ts = tasks.popitem()
-            if n not in stage:
-                next_stage.setdefault(n, []).extend(ts)
-                continue
-            node = nodes[n]
-            data = node.data
-            op = catalog[data.title]
-            params = {**data.params}
-            if has_ctx(op):
-                params["_ctx"] = contexts[node.id]
-            results = []
-            for task in ts:
-                try:
-                    inputs = []
-                    for i in op.inputs.values():
-                        if i.position in "top or bottom":
-                            assert (n, i.name) in batch_inputs, f"{i.name} is missing"
-                            inputs.append(batch_inputs[(n, i.name)])
-                        else:
-                            inputs.append(task)
-                    if cache is not None:
-                        key = make_cache_key((inputs, params))
-                        if key not in cache:
-                            cache[key] = await await_if_needed(op(*inputs, **params))
-                        result = cache[key]
-                    else:
-                        result = await await_if_needed(op(*inputs, **params))
-                except Exception as e:
-                    traceback.print_exc()
-                    data.error = str(e)
-                    break
-                contexts[node.id].last_result = result
-                # Returned lists and DataFrames are considered multiple tasks.
-                if isinstance(result, pd.DataFrame):
-                    result = df_to_list(result)
-                elif not isinstance(result, list):
-                    result = [result]
-                results.extend(result)
-            else:  # Finished all tasks without errors.
-                if (
-                    op.type == "visualization"
-                    or op.type == "table_view"
-                    or op.type == "image"
-                ):
-                    data.display = results[0]
-                for edge in edges[node.id]:
-                    t = nodes[edge.target]
-                    op = catalog[t.data.title]
-                    i = op.inputs[edge.targetHandle]
-                    if i.position in "top or bottom":
-                        batch_inputs.setdefault(
-                            (edge.target, edge.targetHandle), []
-                        ).extend(results)
-                    else:
-                        tasks.setdefault(edge.target, []).extend(results)
-        tasks = next_stage
-    return contexts

lynxkite-core/build/lib/lynxkite/ops.py

@@ -1,224 +0,0 @@
-"""API for implementing LynxKite operations."""
-
-from __future__ import annotations
-import enum
-import functools
-import inspect
-import pydantic
-import typing
-from typing_extensions import Annotated
-
-CATALOGS = {}
-EXECUTORS = {}
-
-typeof = type  # We have some arguments called "type".
-
-
-def type_to_json(t):
-    if isinstance(t, type) and issubclass(t, enum.Enum):
-        return {"enum": list(t.__members__.keys())}
-    if getattr(t, "__metadata__", None):
-        return t.__metadata__[-1]
-    return {"type": str(t)}
-
-
-Type = Annotated[typing.Any, pydantic.PlainSerializer(type_to_json, return_type=dict)]
-LongStr = Annotated[str, {"format": "textarea"}]
-PathStr = Annotated[str, {"format": "path"}]
-CollapsedStr = Annotated[str, {"format": "collapsed"}]
-NodeAttribute = Annotated[str, {"format": "node attribute"}]
-EdgeAttribute = Annotated[str, {"format": "edge attribute"}]
-
-
-class BaseConfig(pydantic.BaseModel):
-    model_config = pydantic.ConfigDict(
-        arbitrary_types_allowed=True,
-    )
-
-
-class Parameter(BaseConfig):
-    """Defines a parameter for an operation."""
-
-    name: str
-    default: typing.Any
-    type: Type = None
-
-    @staticmethod
-    def options(name, options, default=None):
-        e = enum.Enum(f"OptionsFor_{name}", options)
-        return Parameter.basic(name, e[default or options[0]], e)
-
-    @staticmethod
-    def collapsed(name, default, type=None):
-        return Parameter.basic(name, default, CollapsedStr)
-
-    @staticmethod
-    def basic(name, default=None, type=None):
-        if default is inspect._empty:
-            default = None
-        if type is None or type is inspect._empty:
-            type = typeof(default) if default is not None else None
-        return Parameter(name=name, default=default, type=type)
-
-
-class Input(BaseConfig):
-    name: str
-    type: Type
-    position: str = "left"
-
-
-class Output(BaseConfig):
-    name: str
-    type: Type
-    position: str = "right"
-
-
-MULTI_INPUT = Input(name="multi", type="*")
-
-
-def basic_inputs(*names):
-    return {name: Input(name=name, type=None) for name in names}
-
-
-def basic_outputs(*names):
-    return {name: Output(name=name, type=None) for name in names}
-
-
-class Op(BaseConfig):
-    func: typing.Callable = pydantic.Field(exclude=True)
-    name: str
-    params: dict[str, Parameter]
-    inputs: dict[str, Input]
-    outputs: dict[str, Output]
-    type: str = "basic"  # The UI to use for this operation.
-
-    def __call__(self, *inputs, **params):
-        # Convert parameters.
-        for p in params:
-            if p in self.params:
-                if self.params[p].type == int:
-                    params[p] = int(params[p])
-                elif self.params[p].type == float:
-                    params[p] = float(params[p])
-                elif isinstance(self.params[p].type, enum.EnumMeta):
-                    params[p] = self.params[p].type[params[p]]
-        res = self.func(*inputs, **params)
-        return res
-
-
-def op(env: str, name: str, *, view="basic", outputs=None):
-    """Decorator for defining an operation."""
-
-    def decorator(func):
-        sig = inspect.signature(func)
-        # Positional arguments are inputs.
-        inputs = {
-            name: Input(name=name, type=param.annotation)
-            for name, param in sig.parameters.items()
-            if param.kind != param.KEYWORD_ONLY
-        }
-        params = {}
-        for n, param in sig.parameters.items():
-            if param.kind == param.KEYWORD_ONLY and not n.startswith("_"):
-                params[n] = Parameter.basic(n, param.default, param.annotation)
-        if outputs:
-            _outputs = {name: Output(name=name, type=None) for name in outputs}
-        else:
-            _outputs = (
-                {"output": Output(name="output", type=None)} if view == "basic" else {}
-            )
-        op = Op(
-            func=func,
-            name=name,
-            params=params,
-            inputs=inputs,
-            outputs=_outputs,
-            type=view,
-        )
-        CATALOGS.setdefault(env, {})
-        CATALOGS[env][name] = op
-        func.__op__ = op
-        return func
-
-    return decorator
-
-
-def input_position(**kwargs):
-    """Decorator for specifying unusual positions for the inputs."""
-
-    def decorator(func):
-        op = func.__op__
-        for k, v in kwargs.items():
-            op.inputs[k].position = v
-        return func
-
-    return decorator
-
-
-def output_position(**kwargs):
-    """Decorator for specifying unusual positions for the outputs."""
-
-    def decorator(func):
-        op = func.__op__
-        for k, v in kwargs.items():
-            op.outputs[k].position = v
-        return func
-
-    return decorator
-
-
-def no_op(*args, **kwargs):
-    if args:
-        return args[0]
-    return None
-
-
-def register_passive_op(env: str, name: str, inputs=[], outputs=["output"], params=[]):
-    """A passive operation has no associated code."""
-    op = Op(
-        func=no_op,
-        name=name,
-        params={p.name: p for p in params},
-        inputs=dict(
-            (i, Input(name=i, type=None)) if isinstance(i, str) else (i.name, i)
-            for i in inputs
-        ),
-        outputs=dict(
-            (o, Output(name=o, type=None)) if isinstance(o, str) else (o.name, o)
-            for o in outputs
-        ),
-    )
-    CATALOGS.setdefault(env, {})
-    CATALOGS[env][name] = op
-    return op
-
-
-def register_executor(env: str):
-    """Decorator for registering an executor."""
-
-    def decorator(func):
-        EXECUTORS[env] = func
-        return func
-
-    return decorator
-
-
-def op_registration(env: str):
-    return functools.partial(op, env)
-
-
-def passive_op_registration(env: str):
-    return functools.partial(register_passive_op, env)
-
-
-def register_area(env, name, params=[]):
-    """A node that represents an area. It can contain other nodes, but does not restrict movement in any way."""
-    op = Op(
-        func=no_op,
-        name=name,
-        params={p.name: p for p in params},
-        inputs={},
-        outputs={},
-        type="area",
-    )
-    CATALOGS[env][name] = op

lynxkite-core/build/lib/lynxkite/workspace.py

@@ -1,96 +0,0 @@
-"""For working with LynxKite workspaces."""
-
-from typing import Optional
-import dataclasses
-import os
-import pydantic
-import tempfile
-from . import ops
-
-
-class BaseConfig(pydantic.BaseModel):
-    model_config = pydantic.ConfigDict(
-        extra="allow",
-    )
-
-
-class Position(BaseConfig):
-    x: float
-    y: float
-
-
-class WorkspaceNodeData(BaseConfig):
-    title: str
-    params: dict
-    display: Optional[object] = None
-    error: Optional[str] = None
-    # Also contains a "meta" field when going out.
-    # This is ignored when coming back from the frontend.
-
-
-class WorkspaceNode(BaseConfig):
-    id: str
-    type: str
-    data: WorkspaceNodeData
-    position: Position
-
-
-class WorkspaceEdge(BaseConfig):
-    id: str
-    source: str
-    target: str
-    sourceHandle: str
-    targetHandle: str
-
-
-class Workspace(BaseConfig):
-    env: str = ""
-    nodes: list[WorkspaceNode] = dataclasses.field(default_factory=list)
-    edges: list[WorkspaceEdge] = dataclasses.field(default_factory=list)
-
-
-async def execute(ws: Workspace):
-    if ws.env in ops.EXECUTORS:
-        await ops.EXECUTORS[ws.env](ws)
-
-
-def save(ws: Workspace, path: str):
-    j = ws.model_dump_json(indent=2)
-    dirname, basename = os.path.split(path)
-    # Create temp file in the same directory to make sure it's on the same filesystem.
-    with tempfile.NamedTemporaryFile(
-        "w", prefix=f".{basename}.", dir=dirname, delete=False
-    ) as f:
-        temp_name = f.name
-        f.write(j)
-    os.replace(temp_name, path)
-
-
-def load(path: str):
-    with open(path) as f:
-        j = f.read()
-    ws = Workspace.model_validate_json(j)
-    # Metadata is added after loading. This way code changes take effect on old boxes too.
-    _update_metadata(ws)
-    return ws
-
-
-def _update_metadata(ws):
-    catalog = ops.CATALOGS.get(ws.env, {})
-    nodes = {node.id: node for node in ws.nodes}
-    done = set()
-    while len(done) < len(nodes):
-        for node in ws.nodes:
-            if node.id in done:
-                continue
-            data = node.data
-            op = catalog.get(data.title)
-            if op:
-                data.meta = op
-                node.type = op.type
-                if data.error == "Unknown operation.":
-                    data.error = None
-            else:
-                data.error = "Unknown operation."
-            done.add(node.id)
-    return ws