Merge pull request #1 from biggraph/darabos-lynxscribe

requirements.txt

@@ -2,6 +2,7 @@ fastapi
 matplotlib
 networkx
 numpy
+orjson
 pandas
 scipy
 uvicorn[standard]

server/executors/one_by_one.py

@@ -1,8 +1,8 @@
 from .. import ops
 from .. import workspace
-import fastapi
-import json
+import orjson
 import pandas as pd
+import pydantic
 import traceback
 import inspect
 import typing
@@ -37,7 +37,7 @@ def register(env: str, cache: bool = True):
   ops.EXECUTORS[env] = lambda ws: execute(ws, ops.CATALOGS[env], cache=cache)
 
 def get_stages(ws, catalog):
-  '''Inputs on top are batch inputs. We decompose the graph into a DAG of components along these edges.'''
+  '''Inputs on top/bottom are batch inputs. We decompose the graph into a DAG of components along these edges.'''
   nodes = {n.id: n for n in ws.nodes}
   batch_inputs = {}
   inputs = {}
@@ -46,7 +46,7 @@ def get_stages(ws, catalog):
     node = nodes[edge.target]
     op = catalog[node.data.title]
     i = op.inputs[edge.targetHandle]
-    if i.position == 'top':
+    if i.position in 'top or bottom':
       batch_inputs.setdefault(edge.target, []).append(edge.source)
   stages = []
   for bt, bss in batch_inputs.items():
@@ -63,6 +63,15 @@ def get_stages(ws, catalog):
   stages.append(set(nodes))
   return stages
 
+
+def _default_serializer(obj):
+  if isinstance(obj, pydantic.BaseModel):
+    return obj.dict()
+  return {"__nonserializable__": id(obj)}
+
+def make_cache_key(obj):
+  return orjson.dumps(obj, default=_default_serializer)
+
 EXECUTOR_OUTPUT_CACHE = {}
 
 def execute(ws, catalog, cache=None):
@@ -77,7 +86,7 @@ def execute(ws, catalog, cache=None):
     node.data.error = None
     op = catalog[node.data.title]
     # Start tasks for nodes that have no non-batch inputs.
-    if all([i.position == 'top' for i in op.inputs.values()]):
+    if all([i.position in 'top or bottom' for i in op.inputs.values()]):
       tasks[node.id] = [NO_INPUT]
   batch_inputs = {}
   # Run the rest until we run out of tasks.
@@ -99,12 +108,12 @@ def execute(ws, catalog, cache=None):
       for task in ts:
         try:
           inputs = [
-            batch_inputs[(n, i.name)] if i.position == 'top' else task
+            batch_inputs[(n, i.name)] if i.position in 'top or bottom' else task
             for i in op.inputs.values()]
-          if cache:
-            key = json.dumps(fastapi.encoders.jsonable_encoder((inputs, params)))
+          if cache is not None:
+            key = make_cache_key((inputs, params))
             if key not in cache:
-              cache[key] = op.func(*inputs, **params)
+              cache[key] = op(*inputs, **params)
             result = cache[key]
           else:
             result = op(*inputs, **params)
@@ -126,8 +135,9 @@ def execute(ws, catalog, cache=None):
           t = nodes[edge.target]
           op = catalog[t.data.title]
           i = op.inputs[edge.targetHandle]
-          if i.position == 'top':
+          if i.position in 'top or bottom':
             batch_inputs.setdefault((edge.target, edge.targetHandle), []).extend(results)
           else:
             tasks.setdefault(edge.target, []).extend(results)
     tasks = next_stage
+  return contexts

server/main.py

@@ -1,15 +1,18 @@
 import dataclasses
 import fastapi
+import importlib
 import pathlib
 import pkgutil
 from . import ops
 from . import workspace
 
 here = pathlib.Path(__file__).parent
+lynxkite_modules = {}
 for _, name, _ in pkgutil.iter_modules([str(here)]):
     if name.endswith('_ops') and not name.startswith('test_'):
         print(f'Importing {name}')
-        __import__(f'server.{name}')
+        name = f'server.{name}'
+        lynxkite_modules[name] = importlib.import_module(name)
 
 app = fastapi.FastAPI()
 
@@ -67,3 +70,9 @@ def make_dir(req: dict):
     assert not path.exists()
     path.mkdir()
     return list_dir(path.parent)
+
+@app.post("/api/service")
+async def service(req: dict):
+    '''Executors can provide extra HTTP APIs through the /api/service endpoint.'''
+    module = lynxkite_modules[req['module']]
+    return await module.api_service(req)

web/package-lock.json

@@ -25,6 +25,7 @@
         "sass": "^1.77.2",
         "svelte": "^4.2.12",
         "svelte-check": "^3.6.9",
+        "svelte-markdown": "^0.4.1",
         "tslib": "^2.6.2",
         "typescript": "^5.4.4",
         "unplugin-icons": "^0.18.5",
@@ -1081,6 +1082,13 @@
       "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.14.tgz",
       "integrity": "sha512-WCfD5Ht3ZesJUsONdhvm84dmzWOiOzOAqOncN0++w0lBw1o8OuDNJF2McvvCef/yBqb/HYRahp1BYtODFQ8bRg=="
     },
+    "node_modules/@types/marked": {
+      "version": "5.0.2",
+      "resolved": "https://registry.npmjs.org/@types/marked/-/marked-5.0.2.tgz",
+      "integrity": "sha512-OucS4KMHhFzhz27KxmWg7J+kIYqyqoW5kdIEI319hqARQQUTqhao3M/F+uFnDXD0Rg72iDDZxZNxq5gvctmLlg==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/@types/pug": {
       "version": "2.0.10",
       "resolved": "https://registry.npmjs.org/@types/pug/-/pug-2.0.10.tgz",
@@ -1844,6 +1852,19 @@
         "@jridgewell/sourcemap-codec": "^1.4.15"
       }
     },
+    "node_modules/marked": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/marked/-/marked-5.1.2.tgz",
+      "integrity": "sha512-ahRPGXJpjMjwSOlBoTMZAK7ATXkli5qCPxZ21TG44rx1KEo44bii4ekgTDQPNRQ4Kh7JMb9Ub1PVk1NxRSsorg==",
+      "dev": true,
+      "license": "MIT",
+      "bin": {
+        "marked": "bin/marked.js"
+      },
+      "engines": {
+        "node": ">= 16"
+      }
+    },
     "node_modules/mdn-data": {
       "version": "2.0.30",
       "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.30.tgz",
@@ -2460,6 +2481,20 @@
         "svelte": "^3.19.0 || ^4.0.0"
       }
     },
+    "node_modules/svelte-markdown": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/svelte-markdown/-/svelte-markdown-0.4.1.tgz",
+      "integrity": "sha512-pOlLY6EruKJaWI9my/2bKX8PdTeP5CM0s4VMmwmC2prlOkjAf+AOmTM4wW/l19Y6WZ87YmP8+ZCJCCwBChWjYw==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@types/marked": "^5.0.1",
+        "marked": "^5.1.2"
+      },
+      "peerDependencies": {
+        "svelte": "^4.0.0"
+      }
+    },
     "node_modules/svelte-preprocess": {
       "version": "5.1.4",
       "resolved": "https://registry.npmjs.org/svelte-preprocess/-/svelte-preprocess-5.1.4.tgz",

web/package.json

@@ -15,6 +15,7 @@
     "sass": "^1.77.2",
     "svelte": "^4.2.12",
     "svelte-check": "^3.6.9",
+    "svelte-markdown": "^0.4.1",
     "tslib": "^2.6.2",
     "typescript": "^5.4.4",
     "unplugin-icons": "^0.18.5",